Implement comprehensive stepByStep reporter fix for all command scenarios

Co-authored-by: kobenguyent <7845001+kobenguyent@users.noreply.github.com>

Author: Copilot

lib/plugin/stepByStepReport.js

@@ -46,6 +46,12 @@ const templates = {}
  * npx codeceptjs run-workers 2 --plugins stepByStepReport
  * ```
  *
+ * Run tests with multiple configurations:
+ *
+ * ```
+ * npx codeceptjs run-multiple --all --plugins stepByStepReport
+ * ```
+ *
  * #### Configuration
  *
  * ```js
@@ -66,10 +72,17 @@ const templates = {}
  * * `screenshotsForAllureReport`: If Allure plugin is enabled this plugin attaches each saved screenshot to allure report. Default: false.
  * * `disableScreenshotOnFail : Disables the capturing of screeshots after the failed step. Default: true.
  *
- * #### Worker Support
+ * #### Worker and Multiple Run Support
  *
- * When using `run-workers`, screenshots from all workers are automatically consolidated into a shared directory
- * to ensure the final step-by-step report contains all screenshots from all workers.
+ * When using `run-workers`, `run-multiple`, or combinations thereof, screenshots from all workers and runs 
+ * are automatically consolidated into a shared directory to ensure the final step-by-step report contains 
+ * all screenshots from all processes.
+ *
+ * The plugin automatically detects:
+ * - **run-workers**: Consolidates screenshots from multiple worker processes  
+ * - **run-multiple**: Consolidates screenshots from multiple run configurations
+ * - **Mixed scenarios**: Handles combinations of both workers and multiple runs
+ * - **Custom output directories**: Works with both user-specified and default output directories
  *
  * @param {*} config
  */
@@ -99,19 +112,39 @@ module.exports = function (config) {
   const recordedTests = {}
   const pad = '0000'
 
-  // When running with workers, use the shared output directory instead of worker-specific directories
-  // This ensures that all workers save their screenshots to the same location so the final consolidated
-  // report can find all screenshots in one place
+  // Determine if we're running in a multi-process scenario (workers or run-multiple)
+  // and need to consolidate screenshots from multiple directories
   let reportDir
-  if (process.env.RUNS_WITH_WORKERS === 'true' && global.codecept_dir) {
-    // Extract the base output directory from the worker-specific path
-    // Worker paths are typically like: /project/output/worker_name
-    // We want to extract: /project/output and create: /project/output/stepByStepReport
+  
+  const isRunningWithWorkers = process.env.RUNS_WITH_WORKERS === 'true'
+  const isRunMultipleChild = process.argv.some(arg => arg === '--child')
+  const needsConsolidation = isRunningWithWorkers || isRunMultipleChild
+  
+  if (needsConsolidation && global.codecept_dir) {
+    // Extract the base output directory and create a shared location for screenshots
     const currentOutputDir = config.output ? path.resolve(global.codecept_dir, config.output) : defaultConfig.output
-    const workerDirPattern = /[/\\][^/\\]+$/ // Match the last directory segment (worker name)
-    const baseOutputDir = currentOutputDir.replace(workerDirPattern, '')
+    
+    let baseOutputDir = currentOutputDir
+    
+    // For mixed scenario (run-multiple + workers), we need to strip both worker and run directory segments
+    // For run-workers only, strip worker directory segment  
+    // For run-multiple only, strip run directory segment
+    if (isRunningWithWorkers) {
+      // Strip worker directory: /output/smoke_chrome_hash_1/worker1 -> /output/smoke_chrome_hash_1 or /output/worker1 -> /output
+      const workerDirPattern = /[/\\][^/\\]+$/ // Match the last directory segment (worker name)
+      baseOutputDir = baseOutputDir.replace(workerDirPattern, '')
+    }
+    
+    if (isRunMultipleChild) {
+      // Strip run directory: /output/smoke_chrome_hash_1 -> /output
+      const runDirPattern = /[/\\][^/\\]+$/ // Match the last directory segment (run name)
+      baseOutputDir = baseOutputDir.replace(runDirPattern, '')
+    }
+    
+    // Create a shared directory for step-by-step screenshots
     reportDir = path.join(baseOutputDir, 'stepByStepReport')
   } else {
+    // Regular run command: use the output directory as-is
     reportDir = config.output ? path.resolve(global.codecept_dir, config.output) : defaultConfig.output
   }
 

test/plugin/stepByStepReport_test.js

@@ -0,0 +1,119 @@
+const path = require('path')
+const fs = require('fs')
+const { exec } = require('child_process')
+const { expect } = require('expect')
+
+const runner = path.join(__dirname, '../../bin/codecept.js')
+const codecept_dir = path.join(__dirname, '../data/sandbox')
+
+describe('stepByStepReport plugin with different run commands', function () {
+  this.timeout(60000)
+
+  const outputDir = path.join(codecept_dir, 'output')
+  const stepByStepDir = path.join(outputDir, 'stepByStepReport')
+
+  beforeEach(() => {
+    // Clean up output directory before each test
+    if (fs.existsSync(outputDir)) {
+      fs.rmSync(outputDir, { recursive: true, force: true })
+    }
+  })
+
+  it('should create stepByStepReport directory when using run-workers', function (done) {
+    const config = `
+exports.config = {
+  tests: './*_test.js',
+  timeout: 10000,
+  output: './output',
+  helpers: {
+    FakeDriver: {
+      require: '../fake_driver',
+      browser: 'dummy',
+    },
+  },
+  plugins: {
+    stepByStepReport: {
+      enabled: true,
+      deleteSuccessful: false,
+    },
+  },
+  include: {},
+  bootstrap: false,
+  mocha: {},
+  name: 'stepByStepTest',
+}
+`
+    
+    const configPath = path.join(codecept_dir, 'codecept.stepbystep.js')
+    fs.writeFileSync(configPath, config)
+
+    const command = `${runner} run-workers 2 --config ${configPath} --grep "@stepbystep"`
+    
+    exec(command, (err, stdout, stderr) => {
+      console.log('STDOUT:', stdout)
+      console.log('STDERR:', stderr)
+      
+      // The step by step directory should exist
+      const exists = fs.existsSync(stepByStepDir)
+      expect(exists).toBe(true)
+      
+      // Clean up
+      fs.unlinkSync(configPath)
+      
+      done()
+    })
+  })
+
+  it('should consolidate screenshots from run-multiple', function (done) {
+    const config = `
+exports.config = {
+  tests: './*_test.multiple.js',
+  timeout: 10000,
+  output: './output',
+  helpers: {
+    FakeDriver: {
+      require: '../fake_driver',
+      browser: 'dummy',
+    },
+  },
+  plugins: {
+    stepByStepReport: {
+      enabled: true,
+      deleteSuccessful: false,
+    },
+  },
+  multiple: {
+    test1: {
+      browsers: ['chrome'],
+    },
+    test2: {
+      browsers: ['firefox'],
+    },
+  },
+  include: {},
+  bootstrap: false,
+  mocha: {},
+  name: 'stepByStepMultipleTest',
+}
+`
+    
+    const configPath = path.join(codecept_dir, 'codecept.stepbystep.multiple.js')
+    fs.writeFileSync(configPath, config)
+
+    const command = `${runner} run-multiple --config ${configPath} test1`
+    
+    exec(command, (err, stdout, stderr) => {
+      console.log('STDOUT:', stdout)
+      console.log('STDERR:', stderr)
+      
+      // The step by step directory should exist
+      const exists = fs.existsSync(stepByStepDir)
+      expect(exists).toBe(true)
+      
+      // Clean up
+      fs.unlinkSync(configPath)
+      
+      done()
+    })
+  })
+})

test/unit/worker_test.js

@@ -269,29 +269,73 @@ describe('Workers', function () {
     const path = require('path')
 
     // Mock the stepByStep directory resolution logic
-    function getStepByStepReportDir(config, isWorker, globalCodeceptDir) {
+    function getStepByStepReportDir(config, isRunningWithWorkers, isRunMultipleChild, globalCodeceptDir) {
+      const needsConsolidation = isRunningWithWorkers || isRunMultipleChild
       let reportDir
-      if (isWorker && globalCodeceptDir) {
+
+      if (needsConsolidation && globalCodeceptDir) {
         const currentOutputDir = config.output ? path.resolve(globalCodeceptDir, config.output) : '/default-output'
-        const workerDirPattern = /[/\\][^/\\]+$/
-        const baseOutputDir = currentOutputDir.replace(workerDirPattern, '')
+        
+        let baseOutputDir = currentOutputDir
+        
+        // For mixed scenario (run-multiple + workers), we need to strip both worker and run directory segments
+        // For run-workers only, strip worker directory segment  
+        // For run-multiple only, strip run directory segment
+        if (isRunningWithWorkers) {
+          // Strip worker directory: /output/smoke_chrome_hash_1/worker1 -> /output/smoke_chrome_hash_1 or /output/worker1 -> /output
+          const workerDirPattern = /[/\\][^/\\]+$/ // Match the last directory segment (worker name)
+          baseOutputDir = baseOutputDir.replace(workerDirPattern, '')
+        }
+        
+        if (isRunMultipleChild) {
+          // Strip run directory: /output/smoke_chrome_hash_1 -> /output
+          const runDirPattern = /[/\\][^/\\]+$/ // Match the last directory segment (run name)
+          baseOutputDir = baseOutputDir.replace(runDirPattern, '')
+        }
+        
         reportDir = path.join(baseOutputDir, 'stepByStepReport')
       } else {
         reportDir = config.output ? path.resolve(globalCodeceptDir, config.output) : '/default-output'
       }
+      
       return reportDir
     }
 
     const globalCodeceptDir = '/tmp/test'
 
-    // Test regular (non-worker) mode
+    // Test regular (non-worker) mode with default directory
     const regularConfig = { output: './output' }
-    const regularDir = getStepByStepReportDir(regularConfig, false, globalCodeceptDir)
+    const regularDir = getStepByStepReportDir(regularConfig, false, false, globalCodeceptDir)
     expect(regularDir).equal('/tmp/test/output')
 
-    // Test worker mode
+    // Test regular (non-worker) mode with custom directory
+    const customConfig = { output: './custom-output' }
+    const customDir = getStepByStepReportDir(customConfig, false, false, globalCodeceptDir)
+    expect(customDir).equal('/tmp/test/custom-output')
+
+    // Test run-workers mode with default directory
     const workerConfig = { output: './output/worker1' }
-    const workerDir = getStepByStepReportDir(workerConfig, true, globalCodeceptDir)
+    const workerDir = getStepByStepReportDir(workerConfig, true, false, globalCodeceptDir)
     expect(workerDir).equal('/tmp/test/output/stepByStepReport')
+
+    // Test run-workers mode with custom directory
+    const workerCustomConfig = { output: './custom-output/worker2' }
+    const workerCustomDir = getStepByStepReportDir(workerCustomConfig, true, false, globalCodeceptDir)
+    expect(workerCustomDir).equal('/tmp/test/custom-output/stepByStepReport')
+
+    // Test run-multiple mode with default directory
+    const multipleConfig = { output: './output/smoke_chrome_hash_1' }
+    const multipleDir = getStepByStepReportDir(multipleConfig, false, true, globalCodeceptDir)
+    expect(multipleDir).equal('/tmp/test/output/stepByStepReport')
+
+    // Test run-multiple mode with custom directory
+    const multipleCustomConfig = { output: './custom-output/regression_firefox_hash_2' }
+    const multipleCustomDir = getStepByStepReportDir(multipleCustomConfig, false, true, globalCodeceptDir)
+    expect(multipleCustomDir).equal('/tmp/test/custom-output/stepByStepReport')
+
+    // Test mixed run-multiple + workers mode
+    const mixedConfig = { output: './output/smoke_chrome_hash_1/worker1' }
+    const mixedDir = getStepByStepReportDir(mixedConfig, true, true, globalCodeceptDir)
+    expect(mixedDir).equal('/tmp/test/output/stepByStepReport')
   })
 })