added notes & metadata for tests, improved output

Author: DavertMik

docs/ai.md

@@ -30,6 +30,7 @@ module.exports = async function (workerCount, selectedRuns, options) {
   output.print(`CodeceptJS v${require('../codecept').version()} ${output.standWithUkraine()}`)
   output.print(`Running tests in ${output.styles.bold(numberOfWorkers)} workers...`)
   output.print()
+  store.hasWorkers = true
 
   const workers = new Workers(numberOfWorkers, config)
   workers.overrideConfig(overrideConfigs)
@@ -100,7 +101,6 @@ module.exports = async function (workerCount, selectedRuns, options) {
     if (options.verbose || options.debug) store.debugMode = true
 
     if (options.verbose) {
-      global.debugMode = true
       const { getMachineInfo } = require('./info')
       await getMachineInfo()
     }

lib/command/run-workers.js

@@ -130,12 +130,13 @@ class Heal {
           })
 
           if (failureContext?.test) {
+            const test = failureContext.test
             let note = `This test was healed by '${suggestion.name}'`
             note += `\n\nReplace the failed code:\n\n`
             note += colors.red(`- ${failedStep.toCode()}\n`)
             note += colors.green(`+ ${codeSnippet}\n`)
-            failureContext.notes.push(note)
-            failureContext.test.meta.healed = true
+            test.addNote('heal', note)
+            test.meta.healed = true
           }
 
           recorder.add('healed', () => output.print(colors.bold.green(`  Code healed successfully by ${suggestion.name}`), colors.gray('(no errors thrown)')))

lib/heal.js

@@ -482,6 +482,7 @@ class Playwright extends Helper {
 
   async _before(test) {
     this.currentRunningTest = test
+
     recorder.retry({
       retries: process.env.FAILED_STEP_RETRIES || 3,
       when: err => {
@@ -552,6 +553,15 @@ class Playwright extends Helper {
 
     await this._setPage(mainPage)
 
+    try {
+      // set metadata for reporting
+      test.meta.browser = this.browser.browserType().name()
+      test.meta.browserVersion = this.browser.version()
+      test.meta.windowSize = `${this.page.viewportSize().width}x${this.page.viewportSize().height}`
+    } catch (e) {
+      this.debug('Failed to set metadata for reporting')
+    }
+
     if (this.options.trace) await this.browserContext.tracing.start({ screenshots: true, snapshots: true })
 
     return this.browser

lib/helper/Playwright.js

@@ -1,6 +1,7 @@
 const {
   reporters: { Base },
 } = require('mocha')
+const figures = require('figures')
 const ms = require('ms')
 const event = require('../event')
 const AssertionFailedError = require('../assert/error')
@@ -163,33 +164,49 @@ class Cli extends Base {
         const err = test.err
 
         let log = ''
+        let originalMessage = err.message
 
         if (err instanceof AssertionFailedError) {
           err.message = err.inspect()
         }
 
+        // multi-line error messages
+        err.message = '\n  ' + (err.message || '').replace(/^/gm, '  ').trim()
+
         const steps = test.steps || (test.ctx && test.ctx.test.steps)
 
         if (steps && steps.length) {
           let scenarioTrace = ''
           steps.reverse().forEach(step => {
-            const line = `- ${step.toCode()} ${step.line()}`
-            // if (step.status === 'failed') line = '' + line;
+            const hasFailed = step.status === 'failed'
+            let line = `${hasFailed ? output.styles.bold(figures.cross) : figures.tick} ${step.toCode()} ${step.line()}`
+            if (hasFailed) line = output.styles.bold(line)
             scenarioTrace += `\n${line}`
           })
-          log += `${output.styles.bold('Scenario Steps')}:${scenarioTrace}\n`
+          log += `${output.styles.basic(figures.circle)} ${output.styles.section('Scenario Steps')}:${scenarioTrace}\n`
         }
 
         // display artifacts in debug mode
         if (test?.artifacts && Object.keys(test.artifacts).length) {
-          log += `\n${output.styles.bold('Artifacts:')}`
+          log += `\n${output.styles.basic(figures.circle)} ${output.styles.section('Artifacts:')}`
           for (const artifact of Object.keys(test.artifacts)) {
             log += `\n- ${artifact}: ${test.artifacts[artifact]}`
           }
         }
 
+        // display metadata
+        if (test.meta && Object.keys(test.meta).length) {
+          log += `\n\n${output.styles.basic(figures.circle)} ${output.styles.section('Metadata:')}`
+          for (const [key, value] of Object.entries(test.meta)) {
+            log += `\n- ${key}: ${value}`
+          }
+        }
+
         try {
-          let stack = err.stack ? err.stack.split('\n') : []
+          let stack = err.stack
+          stack = stack.replace(originalMessage, '')
+          stack = stack ? stack.split('\n') : []
+
           if (stack[0] && stack[0].includes(err.message)) {
             stack.shift()
           }

lib/mocha/cli.js

@@ -7,6 +7,19 @@ class FeatureConfig {
     this.suite = suite
   }
 
+  /**
+   * Set metadata for this suite
+   * @param {string} key
+   * @param {string} value
+   * @returns {this}
+   */
+  meta(key, value) {
+    this.suite.tests.forEach(test => {
+      test.meta[key] = value
+    })
+    return this
+  }
+
   /**
    * Retry this test for number of times
    *

lib/mocha/featureConfig.js

@@ -42,6 +42,17 @@ class ScenarioConfig {
     return this
   }
 
+  /**
+   * Set metadata for this test
+   * @param {string} key
+   * @param {string} value
+   * @returns {this}
+   */
+  meta(key, value) {
+    this.test.meta[key] = value
+    return this
+  }
+
   /**
    * Set timeout for this test
    * @param {number} timeout

lib/mocha/scenarioConfig.js

@@ -1,4 +1,5 @@
 const Test = require('mocha/lib/test')
+const Suite = require('mocha/lib/suite')
 const { test: testWrapper } = require('./asyncWrapper')
 const { enhanceMochaSuite } = require('./suite')
 
@@ -31,6 +32,11 @@ function enhanceMochaTest(test) {
   test.inject = {}
   test.opts = {}
 
+  test.notes = []
+  test.addNote = (type, note) => {
+    test.notes.push({ type, text: note })
+  }
+
   // Add new methods
   /**
    * @param {Mocha.Suite} suite - The Mocha suite to add this test to
@@ -41,20 +47,27 @@ function enhanceMochaTest(test) {
     test.tags = [...(test.tags || []), ...(suite.tags || [])]
     test.fullTitle = () => `${suite.title}: ${test.title}`
     test.meta = {}
-    test.notes = []
   }
 
   test.applyOptions = function (opts) {
     if (!opts) opts = {}
     test.opts = opts
+    test.meta = opts.meta || {}
     test.totalTimeout = opts.timeout
     if (opts.retries) this.retries(opts.retries)
   }
 
   return test
 }
 
+function repackTestForWorkersTransport(test) {
+  test = Object.assign(new Test(test.title || '', () => {}), test)
+  test.parent = Object.assign(new Suite(test.parent.title), test.parent)
+  return test
+}
+
 module.exports = {
   createTest,
   enhanceMochaTest,
+  repackTestForWorkersTransport,
 }

lib/mocha/test.js

@@ -8,7 +8,10 @@ declare global {
       tags: string[]
       steps: string[]
       meta: Record<string, any>
-      notes: string[]
+      notes: Array<{
+        type: string
+        text: string
+      }>
       config: Record<string, any>
       artifacts: string[]
       inject: Record<string, any>
@@ -17,6 +20,7 @@ declare global {
       totalTimeout?: number
       addToSuite(suite: Mocha.Suite): void
       applyOptions(opts: Record<string, any>): void
+      addNote(type: string, note: string): void
       codeceptjs: boolean
     }