Merge pull request #465 from jhawthorn/c_commonmarker

Go back to C commonmarker

Author: jhawthorn

benchmarks/lobsters/Gemfile

@@ -45,7 +45,7 @@ gem "rqrcode_core", github: "whomwah/rqrcode_core"
 gem "pdf-reader"
 gem "nokogiri", ">= 1.13.9"
 gem "htmlentities"
-gem "commonmarker", github: "ianks/commonmarker", branch: "hotfix/ruby-4-1"
+gem "commonmarker", github: "jhawthorn/commonmarker", branch: "c-api-stable" # The v1.0 Rust gem is perpetually broken on Ruby master
 
 # perf - skip for benchmarking
 group :development do

benchmarks/lobsters/Gemfile.lock

@@ -1,10 +1,9 @@
 GIT
-  remote: https://github.com/ianks/commonmarker.git
-  revision: 6ea4f5a2be0423206dbaeca7108035a70a149ab7
-  branch: hotfix/ruby-4-1
+  remote: https://github.com/jhawthorn/commonmarker.git
+  revision: 1469d28ad133cb5b9d834714e7c285a45d30da6c
+  branch: c-api-stable
   specs:
-    commonmarker (2.6.1)
-      rb_sys (~> 0.9)
+    commonmarker (0.23.12)
 
 GIT
   remote: https://github.com/whomwah/rqrcode.git
@@ -242,10 +241,7 @@ GEM
       zeitwerk (~> 2.6)
     rainbow (3.1.1)
     rake (13.3.0)
-    rake-compiler-dock (1.11.0)
     rb-readline (0.5.5)
-    rb_sys (0.9.124)
-      rake-compiler-dock (= 1.11.0)
     rdoc (6.14.2)
       erb
       psych (>= 4.0.0)
@@ -420,7 +416,7 @@ CHECKSUMS
   capybara (3.40.0) sha256=42dba720578ea1ca65fd7a41d163dd368502c191804558f6e0f71b391054aeef
   cgi (0.5.0) sha256=fe99f65bb2c146e294372ebb27602adbc3b4c008e9ea7038c6bd48c1ec9759da
   chunky_png (1.4.0) sha256=89d5b31b55c0cf4da3cf89a2b4ebc3178d8abe8cbaf116a1dba95668502fdcfe
-  commonmarker (2.6.1)
+  commonmarker (0.23.12)
   concurrent-ruby (1.3.5) sha256=813b3e37aca6df2a21a3b9f1d497f8cbab24a2b94cab325bffe65ee0f6cbebc6
   connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
   crack (1.0.0) sha256=c83aefdb428cdc7b66c7f287e488c796f055c0839e6e545fec2c7047743c4a49
@@ -485,9 +481,7 @@ CHECKSUMS
   railties (8.1.1) sha256=fb0c7038b147bea41bf6697fa443ff1c5c47d3bb1eedd9ecf1bceeb90efcb868
   rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a
   rake (13.3.0) sha256=96f5092d786ff412c62fde76f793cc0541bd84d2eb579caa529aa8a059934493
-  rake-compiler-dock (1.11.0) sha256=eab51f2cd533eb35cea6b624a75281f047123e70a64c58b607471bb49428f8c2
   rb-readline (0.5.5) sha256=9e9bd7e198bdef0822c46902f6c592b882c1f9777894a4c3dcf5b320824a8793
-  rb_sys (0.9.124) sha256=513476557b12eaf73764b3da9f8746024558fe8699bda785fb548c9aa3877ae7
   rdoc (6.14.2) sha256=9fdd44df130f856ae70cc9a264dfd659b9b40de369b16581f4ab746e42439226
   regexp_parser (2.10.0) sha256=cb6f0ddde88772cd64bff1dbbf68df66d376043fe2e66a9ef77fcb1b0c548c61
   reline (0.6.2) sha256=1dad26a6008872d59c8e05244b119347c9f2ddaf4a53dce97856cd5f30a02846

benchmarks/lobsters/extras/markdowner.rb

@@ -1,23 +1,17 @@
 class Markdowner
   # opts[:allow_images] allows <img> tags
 
-  COMMONMARKER_OPTIONS = {
-    parse: { smart: true },
-    render: { unsafe: true },
-    extension: { tagfilter: true, autolink: true, strikethrough: true }
-  }.freeze
-
   def self.to_html(text, opts = {})
     if text.blank?
       return ""
     end
 
-    # Preprocess @mentions before markdown parsing
-    processed_text = preprocess_mentions(text.to_s)
+    exts = [:tagfilter, :autolink, :strikethrough]
+    root = CommonMarker.render_doc(text.to_s, [:SMART], exts)
 
-    html = Commonmarker.to_html(processed_text, options: COMMONMARKER_OPTIONS)
+    walk_text_nodes(root) {|n| postprocess_text_node(n) }
 
-    ng = Nokogiri::HTML(html)
+    ng = Nokogiri::HTML(root.to_html([:DEFAULT], exts))
 
     # change <h1>, <h2>, etc. headings to just bold tags
     ng.css("h1, h2, h3, h4, h5, h6").each do |h|
@@ -39,13 +33,44 @@ def self.to_html(text, opts = {})
     end
   end
 
-  def self.preprocess_mentions(text)
-    text.gsub(/\B(@#{User::VALID_USERNAME})/) do |match|
-      user = match[1..-1]
-      if User.exists?(username: user)
-        "[#{match}](#{Rails.application.root_url}u/#{user})"
+  def self.walk_text_nodes(node, &block)
+    return if node.type == :link
+    return block.call(node) if node.type == :text
+
+    node.each do |child|
+      walk_text_nodes(child, &block)
+    end
+  end
+
+  def self.postprocess_text_node(node)
+    while node
+      return unless node.string_content =~ /\B(@#{User::VALID_USERNAME})/
+      before, user, after = $`, $1, $'
+
+      node.string_content = before
+
+      if User.exists?(:username => user[1..-1])
+        link = CommonMarker::Node.new(:link)
+        link.url = Rails.application.root_url + "u/#{user[1..-1]}"
+        node.insert_after(link)
+
+        link_text = CommonMarker::Node.new(:text)
+        link_text.string_content = user
+        link.append_child(link_text)
+
+        node = link
+      else
+        node.string_content += user
+      end
+
+      if after.length > 0
+        remainder = CommonMarker::Node.new(:text)
+        remainder.string_content = after
+        node.insert_after(remainder)
+
+        node = remainder
       else
-        match
+        node = nil
       end
     end
   end