Merge pull request #7552 from Vaivaswat2244/snapshot-test

Tried an approch for text case in pixelmatch

Author: davepagurek

package-lock.json

@@ -35,6 +35,7 @@
     "libtess": "^1.2.2",
     "omggif": "^1.0.10",
     "pako": "^2.1.0",
+    "pixelmatch": "^7.1.0",
     "zod": "^3.23.8"
   },
   "devDependencies": {

package.json

@@ -2,6 +2,7 @@ import p5 from '../../../src/app.js';
 import { server } from '@vitest/browser/context'
 import { THRESHOLD, DIFFERENCE, ERODE } from '../../../src/core/constants.js';
 const { readFile, writeFile } = server.commands
+import pixelmatch from 'pixelmatch';
 
 // By how much can each color channel value (0-255) differ before
 // we call it a mismatch? This should be large enough to not trigger
@@ -86,57 +87,258 @@ export function visualSuite(
   });
 }
 
+/**
+ * Image Diff Algorithm for p5.js Visual Tests
+ * 
+ * This algorithm addresses the challenge of cross-platform rendering differences in p5.js visual tests.
+ * Different operating systems and browsers render graphics with subtle variations, particularly with
+ * anti-aliasing, text rendering, and sub-pixel positioning. This can cause false negatives in tests
+ * when the visual differences are acceptable rendering variations rather than actual bugs.
+ * 
+ * Key components of the approach:
+ * 
+ * 1. Initial pixel-by-pixel comparison:
+ *    - Uses pixelmatch to identify differences between expected and actual images
+ *    - Sets a moderate threshold (0.5) to filter out minor color/intensity variations
+ *    - Produces a diff image with red pixels marking differences
+ * 
+ * 2. Cluster identification using BFS (Breadth-First Search):
+ *    - Groups connected difference pixels into clusters
+ *    - Uses a queue-based BFS algorithm to find all connected pixels
+ *    - Defines connectivity based on 8-way adjacency (all surrounding pixels)
+ * 
+ * 3. Cluster categorization by type:
+ *    - Analyzes each pixel's neighborhood characteristics
+ *    - Specifically identifies "line shift" clusters - differences that likely represent
+ *      the same visual elements shifted by 1px due to platform rendering differences
+ *    - Line shifts are identified when >80% of pixels in a cluster have ≤2 neighboring diff pixels
+ * 
+ * 4. Intelligent failure criteria:
+ *    - Filters out clusters smaller than MIN_CLUSTER_SIZE pixels (noise reduction)
+ *    - Applies different thresholds for regular differences vs. line shifts
+ *    - Considers both the total number of significant pixels and number of distinct clusters
+ * 
+ * This approach balances the need to catch genuine visual bugs (like changes to shape geometry, 
+ * colors, or positioning) while tolerating acceptable cross-platform rendering variations.
+ * 
+ * Parameters:
+ * - MIN_CLUSTER_SIZE: Minimum size for a cluster to be considered significant (default: 4)
+ * - MAX_TOTAL_DIFF_PIXELS: Maximum allowed non-line-shift difference pixels (default: 40)
+ * Note: These can be adjusted for further updation
+ * 
+ * Note for contributors: When running tests locally, you may not see these differences as they
+ * mainly appear when tests run on different operating systems or browser rendering engines.
+ * However, the same code may produce slightly different renderings on CI environments, particularly
+ * with text positioning, thin lines, or curved shapes. This algorithm helps distinguish between
+ * these acceptable variations and actual visual bugs.
+ */
+
 export async function checkMatch(actual, expected, p5) {
   let scale = Math.min(MAX_SIDE/expected.width, MAX_SIDE/expected.height);
-
-  // Long screenshots end up super tiny when fit to a small square, so we
-  // can double the max side length for these
   const ratio = expected.width / expected.height;
   const narrow = ratio !== 1;
   if (narrow) {
     scale *= 2;
   }
-
+  
   for (const img of [actual, expected]) {
     img.resize(
       Math.ceil(img.width * scale),
       Math.ceil(img.height * scale)
     );
   }
 
-  const expectedWithBg = p5.createGraphics(expected.width, expected.height);
-  expectedWithBg.pixelDensity(1);
-  expectedWithBg.background(BG);
-  expectedWithBg.image(expected, 0, 0);
-
-  const cnv = p5.createGraphics(actual.width, actual.height);
-  cnv.pixelDensity(1);
-  cnv.background(BG);
-  cnv.image(actual, 0, 0);
-  cnv.blendMode(DIFFERENCE);
-  cnv.image(expectedWithBg, 0, 0);
-  for (let i = 0; i < shiftThreshold; i++) {
-    cnv.filter(ERODE, false);
+  // Ensure both images have the same dimensions
+  const width = expected.width;
+  const height = expected.height;
+  
+  // Create canvases with background color
+  const actualCanvas = p5.createGraphics(width, height);
+  const expectedCanvas = p5.createGraphics(width, height);
+  actualCanvas.pixelDensity(1);
+  expectedCanvas.pixelDensity(1);
+  
+  actualCanvas.background(BG);
+  expectedCanvas.background(BG);
+  
+  actualCanvas.image(actual, 0, 0);
+  expectedCanvas.image(expected, 0, 0);
+  
+  // Load pixel data
+  actualCanvas.loadPixels();
+  expectedCanvas.loadPixels();
+  
+  // Create diff output canvas
+  const diffCanvas = p5.createGraphics(width, height);
+  diffCanvas.pixelDensity(1);
+  diffCanvas.loadPixels();
+  
+  // Run pixelmatch
+  const diffCount = pixelmatch(
+    actualCanvas.pixels,
+    expectedCanvas.pixels,
+    diffCanvas.pixels,
+    width,
+    height,
+    { 
+      threshold: 0.5,
+      includeAA: false,
+      alpha: 0.1
+    }
+  );
+  
+  // If no differences, return early
+  if (diffCount === 0) {
+    actualCanvas.remove();
+    expectedCanvas.remove();
+    diffCanvas.updatePixels();
+    return { ok: true, diff: diffCanvas };
   }
-  const diff = cnv.get();
-  cnv.remove();
-  diff.loadPixels();
-  expectedWithBg.remove();
-
-  let ok = true;
-  for (let i = 0; i < diff.pixels.length; i += 4) {
-    let diffSum = 0;
-    for (let off = 0; off < 3; off++) {
-      diffSum += diff.pixels[i+off]
+  
+  // Post-process to identify and filter out isolated differences
+  const visited = new Set();
+  const clusterSizes = [];
+  
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const pos = (y * width + x) * 4;
+      
+      // If this is a diff pixel (red in pixelmatch output) and not yet visited
+      if (
+        diffCanvas.pixels[pos] === 255 && 
+        diffCanvas.pixels[pos + 1] === 0 && 
+        diffCanvas.pixels[pos + 2] === 0 &&
+        !visited.has(pos)
+      ) {
+        // Find the connected cluster size using BFS
+        const clusterSize = findClusterSize(diffCanvas.pixels, x, y, width, height, 1, visited);
+        clusterSizes.push(clusterSize);
+      }
     }
-    diffSum /= 3;
-    if (diffSum > COLOR_THRESHOLD) {
-      ok = false;
-      break;
+  }
+  
+  // Define significance thresholds
+  const MIN_CLUSTER_SIZE = 4;  // Minimum pixels in a significant cluster
+  const MAX_TOTAL_DIFF_PIXELS = 40;  // Maximum total different pixels
+
+  // Determine if the differences are significant
+  const nonLineShiftClusters = clusterSizes.filter(c => !c.isLineShift && c.size >= MIN_CLUSTER_SIZE);
+  
+  // Calculate significant differences excluding line shifts
+  const significantDiffPixels = nonLineShiftClusters.reduce((sum, c) => sum + c.size, 0);
+
+  // Update the diff canvas
+  diffCanvas.updatePixels();
+  
+  // Clean up canvases
+  actualCanvas.remove();
+  expectedCanvas.remove();
+  
+  // Determine test result
+  const ok = (
+    diffCount === 0 ||  
+    (
+      significantDiffPixels === 0 ||  
+      (
+        (significantDiffPixels <= MAX_TOTAL_DIFF_PIXELS) &&  
+        (nonLineShiftClusters.length <= 2)  // Not too many significant clusters
+      )
+    )
+  );
+
+  return { 
+    ok,
+    diff: diffCanvas,
+    details: {
+      totalDiffPixels: diffCount,
+      significantDiffPixels,
+      clusters: clusterSizes
+    }
+  };
+}
+
+/**
+ * Find the size of a connected cluster of diff pixels using BFS
+ */
+function findClusterSize(pixels, startX, startY, width, height, radius, visited) {
+  const queue = [{x: startX, y: startY}];
+  let size = 0;
+  const clusterPixels = [];
+  
+  while (queue.length > 0) {
+    const {x, y} = queue.shift();
+    const pos = (y * width + x) * 4;
+    
+    // Skip if already visited
+    if (visited.has(pos)) continue;
+    
+    // Skip if not a diff pixel
+    if (pixels[pos] !== 255 || pixels[pos + 1] !== 0 || pixels[pos + 2] !== 0) continue;
+    
+    // Mark as visited
+    visited.add(pos);
+    size++;
+    clusterPixels.push({x, y});
+    
+    // Add neighbors to queue
+    for (let dy = -radius; dy <= radius; dy++) {
+      for (let dx = -radius; dx <= radius; dx++) {
+        const nx = x + dx;
+        const ny = y + dy;
+        
+        // Skip if out of bounds
+        if (nx < 0 || nx >= width || ny < 0 || ny >= height) continue;
+        
+        // Skip if already visited
+        const npos = (ny * width + nx) * 4;
+        if (!visited.has(npos)) {
+          queue.push({x: nx, y: ny});
+        }
+      }
+    }
+  }
+
+  let isLineShift = false;
+  if (clusterPixels.length > 0) {
+    // Count pixels with limited neighbors (line-like characteristic)
+    let linelikePixels = 0;
+    
+    for (const {x, y} of clusterPixels) {
+      // Count neighbors
+      let neighbors = 0;
+      for (let dy = -1; dy <= 1; dy++) {
+        for (let dx = -1; dx <= 1; dx++) {
+          if (dx === 0 && dy === 0) continue; // Skip self
+          
+          const nx = x + dx;
+          const ny = y + dy;
+          
+          // Skip if out of bounds
+          if (nx < 0 || nx >= width || ny < 0 || ny >= height) continue;
+          
+          const npos = (ny * width + nx) * 4;
+          // Check if neighbor is a diff pixel
+          if (pixels[npos] === 255 && pixels[npos + 1] === 0 && pixels[npos + 2] === 0) {
+            neighbors++;
+          }
+        }
+      }
+      
+      // Line-like pixels typically have 1-2 neighbors
+      if (neighbors <= 2) {
+        linelikePixels++;
+      }
     }
+    
+    // If most pixels (>80%) in the cluster have ≤2 neighbors, it's likely a line shift
+    isLineShift = linelikePixels / clusterPixels.length > 0.8;
   }
 
-  return { ok, diff };
+  return {
+    size,
+    pixels: clusterPixels,
+    isLineShift
+  };
 }
 
 /**