Merge branch 'cppdoc-cc:main' into main

Author: mumupika

migrate/migrate-bot.ts

@@ -5,7 +5,6 @@ import path, { join } from "path";
 import { fileURLToPath } from "url";
 import { execSync, spawnSync } from "child_process";
 import { visualizeTextDiff } from "./text-diff-visualizer";
-import { convert } from "html-to-text";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -85,11 +84,14 @@ async function fetchPageContent(
   if (!contentElement) {
     throw new Error("Could not find #mw-content-text");
   }
+
+  const title = headingElement?.textContent?.trim() || "";
+
   return {
     html: contentElement.innerHTML,
-    title: headingElement?.textContent?.trim() || "",
+    title,
     url,
-    innerText: (contentElement as HTMLDivElement).innerText,
+    innerText: title + "\n" + (contentElement as HTMLDivElement).innerText,
   };
 }
 

migrate/text-diff-visualizer.ts

@@ -36,7 +36,7 @@ function parseText(text: string): LineData[] {
   return text.split("\n").map((line) => {
     const parts = line
       .trimEnd()
-      .replace(/[^\p{L}\p{N}]/gu, " ")
+      .replace(/[,，。""''"".]/gu, " ")
       .split(/\s+/)
       .filter((p) => p.length > 0);
     const tokens = parts
@@ -105,19 +105,21 @@ export function visualizeTextDiff(textA: string, textB: string) {
 
     let hasDifference = false;
 
-    const checkTokens = (tokens: WordToken[] | undefined) => {
+    const checkTokens = (tokens: WordToken[] | undefined, isRowA: boolean) => {
       if (!tokens) return false;
       for (const t of tokens) {
         if (!t.key) continue;
         const countA = freqA.get(t.key) || 0;
         const countB = freqB.get(t.key) || 0;
-        if (countA !== countB) return true;
+        if (isRowA ? countA > countB : countB > countA) {
+          return true;
+        }
       }
       return false;
     };
 
-    const diffA = checkTokens(lineRowA?.tokens);
-    const diffB = checkTokens(lineRowB?.tokens);
+    const diffA = checkTokens(lineRowA?.tokens, true);
+    const diffB = checkTokens(lineRowB?.tokens, false);
 
     const emptyA = !lineRowA || lineRowA.tokens.length === 0;
     const emptyB = !lineRowB || lineRowB.tokens.length === 0;

package-lock.json

@@ -25,7 +25,6 @@
   },
   "devDependencies": {
     "@octokit/rest": "^21.0.2",
-    "@types/html-to-text": "^9.0.4",
     "prettier": "3.6.2"
   }
 }