From 09a7b340363c0855161144784760e2145f4f9185 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 25 Jan 2025 23:45:11 +0200 Subject: [PATCH 1/5] better hooks handling by submitting where test.failed occur --- lib/helper/Playwright.js | 8 ++- lib/listener/store.js | 10 +++- lib/mocha/asyncWrapper.js | 6 +-- lib/plugin/analyze.js | 20 ++++---- lib/plugin/autoLogin.js | 94 +++++++++++++++++++++++++++++++++- lib/plugin/screenshotOnFail.js | 7 +-- lib/plugin/stepByStepReport.js | 9 ++-- lib/step.js | 6 +++ lib/store.js | 2 + package.json | 2 +- 10 files changed, 139 insertions(+), 25 deletions(-) diff --git a/lib/helper/Playwright.js b/lib/helper/Playwright.js index 979cc743a..61bc4f752 100644 --- a/lib/helper/Playwright.js +++ b/lib/helper/Playwright.js @@ -523,12 +523,17 @@ class Playwright extends Helper { this.currentRunningTest.artifacts.har = fileName contextOptions.recordHar = this.options.recordHar } + + // load pre-saved cookies + if (test.opts.cookies) contextOptions.storageState = { cookies: test.opts.cookies } + if (this.storageState) contextOptions.storageState = this.storageState if (this.options.userAgent) contextOptions.userAgent = this.options.userAgent if (this.options.locale) contextOptions.locale = this.options.locale if (this.options.colorScheme) contextOptions.colorScheme = this.options.colorScheme this.contextOptions = contextOptions if (!this.browserContext || !restartsSession()) { + this.debugSection('New Session', JSON.stringify(this.contextOptions)) this.browserContext = await this.browser.newContext(this.contextOptions) // Adding the HTTPSError ignore in the context so that we can ignore those errors } } @@ -938,7 +943,8 @@ class Playwright extends Helper { throw new Error('Cannot open pages inside an Electron container') } if (!/^\w+\:(\/\/|.+)/.test(url)) { - url = this.options.url + (url.startsWith('/') ? url : `/${url}`) + url = this.options.url + (!this.options.url.endsWith('/') && url.startsWith('/') ? url : `/${url}`) + this.debug(`Changed URL to base url + relative path: ${url}`) } if (this.options.basicAuth && this.isAuthenticated !== true) { diff --git a/lib/listener/store.js b/lib/listener/store.js index 763aa1edc..3539e59ce 100644 --- a/lib/listener/store.js +++ b/lib/listener/store.js @@ -2,11 +2,19 @@ const event = require('../event') const store = require('../store') module.exports = function () { + event.dispatcher.on(event.suite.before, suite => { + store.currentSuite = suite + }) + + event.dispatcher.on(event.suite.after, () => { + store.currentSuite = null + }) + event.dispatcher.on(event.test.before, test => { store.currentTest = test }) - event.dispatcher.on(event.test.finished, test => { + event.dispatcher.on(event.test.finished, () => { store.currentTest = null }) } diff --git a/lib/mocha/asyncWrapper.js b/lib/mocha/asyncWrapper.js index 560776ed6..9be59ddb3 100644 --- a/lib/mocha/asyncWrapper.js +++ b/lib/mocha/asyncWrapper.js @@ -19,10 +19,10 @@ const injectHook = function (inject, suite) { return recorder.promise() } -function suiteTestFailedHookError(suite, err) { +function suiteTestFailedHookError(suite, err, hookName) { suite.eachTest(test => { test.err = err - event.emit(event.test.failed, test, err) + event.emit(event.test.failed, test, err, ucfirst(hookName)) }) } @@ -120,7 +120,7 @@ module.exports.injected = function (fn, suite, hookName) { const errHandler = err => { recorder.session.start('teardown') recorder.cleanAsyncErr() - if (hookName == 'before' || hookName == 'beforeSuite') suiteTestFailedHookError(suite, err) + if (hookName == 'before' || hookName == 'beforeSuite') suiteTestFailedHookError(suite, err, hookName) if (hookName === 'after') event.emit(event.test.after, suite) if (hookName === 'afterSuite') event.emit(event.suite.after, suite) recorder.add(() => doneFn(err)) diff --git a/lib/plugin/analyze.js b/lib/plugin/analyze.js index e92e38a3a..e977d6351 100644 --- a/lib/plugin/analyze.js +++ b/lib/plugin/analyze.js @@ -12,8 +12,8 @@ const { ansiRegExp, base64EncodeFile, markdownToAnsi } = require('../utils') const MAX_DATA_LENGTH = 5000 const defaultConfig = { - clusterize: 2, - analyze: 3, + clusterize: 5, + analyze: 2, vision: false, categories: [ 'Browser connection error / browser crash', @@ -64,17 +64,18 @@ const defaultConfig = { If you identify that all tests in the group have the same tag, add this tag to the group report, otherwise ignore TAG section. If you identify that all tests in the group have the same suite, add this suite to the group report, otherwise ignore SUITE section. Pick different emojis for each group. - Do not include group into report if it has only one test in affected tests section. + Order groups by the number of tests in the group. + If group has one test, skip that group. Provide list of groups in following format: _______________________________ - ## Group + ## Group + * SUMMARY * CATEGORY * ERROR , , ... - * SUMMARY * STEP (use CodeceptJS format I.click(), I.see(), etc; if all failures happend on the same step) * SUITE , (if SUITE is present, and if all tests in the group have the same suite or suites) * TAG (if TAG is present, and if all tests in the group have the same tag) @@ -126,14 +127,16 @@ const defaultConfig = { Do not get to details, be concise. If there is failed step, just write it in STEPS section. If you have suggestions for the test, write them in SUMMARY section. + Do not be too technical in SUMMARY section. Inside SUMMARY write exact values, if you have suggestions, explain which information you used to suggest. Be concise, each section should not take more than one sentence. Response format: + * SUMMARY + * ERROR , , ... * CATEGORY * STEPS - * SUMMARY Do not add any other sections or explanations. Only CATEGORY, SUMMARY, STEPS. ${config.vision ? 'Also a screenshot of the page is attached to the prompt.' : ''} @@ -153,11 +156,6 @@ const defaultConfig = { }) } - messages.push({ - role: 'assistant', - content: `## `, - }) - return messages }, }, diff --git a/lib/plugin/autoLogin.js b/lib/plugin/autoLogin.js index 805585510..3624091ba 100644 --- a/lib/plugin/autoLogin.js +++ b/lib/plugin/autoLogin.js @@ -1,8 +1,12 @@ const fs = require('fs') const path = require('path') const { fileExists } = require('../utils') +const FuncStep = require('../step/func') +const Section = require('../step/section') +const recordStep = require('../step/record') const container = require('../container') const store = require('../store') +const event = require('../event') const recorder = require('../recorder') const { debug } = require('../output') const isAsyncFunction = require('../utils').isAsyncFunction @@ -276,8 +280,42 @@ module.exports = function (config) { } const loginFunction = async name => { - const userSession = config.users[name] const I = container.support('I') + const test = store.currentTest + + // we are in BeforeSuite hook + if (!test) { + enableAuthBeforeEachTest(name) + return + } + + if (config.saveToFile && !store[`${name}_session`]) { + // loading from file + for (const name in config.users) { + const fileName = path.join(global.output_dir, `${name}_session.json`) + if (!fileExists(fileName)) continue + const data = fs.readFileSync(fileName).toString() + try { + store[`${name}_session`] = JSON.parse(data) + } catch (err) { + throw new Error(`Could not load session from ${fileName}\n${err}`) + } + debug(`Loaded user session for ${name}`) + } + } + + if (isPlaywrightSession() && test?.opts?.cookies) { + if (test.opts.user == name) { + debug(`Cookies already loaded for ${name}`) + // alreadyLoggedIn(name); + return + } else { + debug(`Cookies already loaded for ${test.opts.user}, but not for ${name}`) + await I.deleteCookie() + } + } + + const userSession = config.users[name] const cookies = store[`${name}_session`] const shouldAwait = isAsyncFunction(userSession.login) || isAsyncFunction(userSession.restore) || isAsyncFunction(userSession.check) @@ -332,8 +370,62 @@ module.exports = function (config) { return recorder.promise() } + function enableAuthBeforeEachTest(name) { + const suite = store.currentSuite + if (!suite) return + + debug(`enabling auth as ${name} for each test of suite ${suite.title}`) + + // we are setting test opts so they can be picked up by Playwright if it starts browser for this test + suite.eachTest(test => { + // preload from store + if (store[`${name}_session`]) { + test.opts.cookies = store[`${name}_session`] + test.opts.user = name + return + } + + if (!config.saveToFile) return + const cookieFile = path.join(global.output_dir, `${name}_session.json`) + + if (!fileExists(cookieFile)) { + return + } + + const context = fs.readFileSync(cookieFile).toString() + test.opts.cookies = JSON.parse(context) + test.opts.user = name + }) + + function runLoginFunctionForTest(test) { + if (!suite.tests.includes(test)) return + // let's call this function to ensure that authorization happened + // if no cookies, it will login and save them + loginFunction(name) + } + + // we are in BeforeSuite hook + event.dispatcher.on(event.test.started, runLoginFunctionForTest) + event.dispatcher.on(event.suite.after, () => { + event.dispatcher.off(event.test.started, runLoginFunctionForTest) + }) + } + // adding this to DI container const support = {} support[config.inject] = loginFunction container.append({ support }) + + return loginFunction +} + +function isPlaywrightSession() { + return !!container.helpers('Playwright') +} + +function alreadyLoggedIn(name) { + const step = new FuncStep('am logged in as') + step.actor = 'I' + step.setCallable(() => {}) + return recordStep(step, [name]) } diff --git a/lib/plugin/screenshotOnFail.js b/lib/plugin/screenshotOnFail.js index f35b9d052..b44692722 100644 --- a/lib/plugin/screenshotOnFail.js +++ b/lib/plugin/screenshotOnFail.js @@ -72,11 +72,12 @@ module.exports = function (config) { return } - event.dispatcher.on(event.test.failed, test => { - if (test.ctx?._runnable.title.includes('hook: ')) { - output.plugin('screenshotOnFail', 'BeforeSuite/AfterSuite do not have any access to the browser, hence it could not take screenshot.') + event.dispatcher.on(event.test.failed, (test, _err, hookName) => { + if (hookName == 'BeforeSuite' || hookName == 'AfterSuite') { + // no browser here return } + recorder.add( 'screenshot of failed test', async () => { diff --git a/lib/plugin/stepByStepReport.js b/lib/plugin/stepByStepReport.js index 9072c86d8..180ae6c38 100644 --- a/lib/plugin/stepByStepReport.js +++ b/lib/plugin/stepByStepReport.js @@ -121,12 +121,13 @@ module.exports = function (config) { deleteDir(dir) }) - event.dispatcher.on(event.test.failed, (test, err) => { - if (test.ctx._runnable.title.includes('hook: ')) { - output.plugin('stepByStepReport', 'BeforeSuite/AfterSuite do not have any access to the browser, hence it could not take screenshot.') + event.dispatcher.on(event.test.failed, (test, _err, hookName) => { + if (hookName == 'BeforeSuite' || hookName == 'AfterSuite') { + // no browser here return } - persist(test, err) + + persist(test) }) event.dispatcher.on(event.all.result, () => { diff --git a/lib/step.js b/lib/step.js index 9295a0835..0c92c8f36 100644 --- a/lib/step.js +++ b/lib/step.js @@ -14,7 +14,13 @@ const Step = require('./step/helper') */ const MetaStep = require('./step/meta') +/** + * Step used to execute a single function + */ +const FuncStep = require('./step/func') + module.exports = Step module.exports.MetaStep = MetaStep module.exports.BaseStep = BaseStep module.exports.StepConfig = StepConfig +module.exports.FuncStep = FuncStep diff --git a/lib/store.js b/lib/store.js index 352b2d27c..18f918ae7 100644 --- a/lib/store.js +++ b/lib/store.js @@ -15,6 +15,8 @@ const store = { currentTest: null, /** @type {any} */ currentStep: null, + /** @type {CodeceptJS.Suite | null} */ + currentSuite: null, } module.exports = store diff --git a/package.json b/package.json index 51ba4f82b..ef38a3ef0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeceptjs", - "version": "3.7.0-beta.1", + "version": "3.7.0-beta.8", "description": "Supercharged End 2 End Testing Framework for NodeJS", "keywords": [ "acceptance", From 5b4a9b79a70cb61d6b5ca83d567b897107047a12 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 26 Jan 2025 00:53:22 +0200 Subject: [PATCH 2/5] reverted autologin plugin changes --- lib/plugin/autoLogin.js | 94 +---------------------------------------- 1 file changed, 1 insertion(+), 93 deletions(-) diff --git a/lib/plugin/autoLogin.js b/lib/plugin/autoLogin.js index 3624091ba..805585510 100644 --- a/lib/plugin/autoLogin.js +++ b/lib/plugin/autoLogin.js @@ -1,12 +1,8 @@ const fs = require('fs') const path = require('path') const { fileExists } = require('../utils') -const FuncStep = require('../step/func') -const Section = require('../step/section') -const recordStep = require('../step/record') const container = require('../container') const store = require('../store') -const event = require('../event') const recorder = require('../recorder') const { debug } = require('../output') const isAsyncFunction = require('../utils').isAsyncFunction @@ -280,42 +276,8 @@ module.exports = function (config) { } const loginFunction = async name => { - const I = container.support('I') - const test = store.currentTest - - // we are in BeforeSuite hook - if (!test) { - enableAuthBeforeEachTest(name) - return - } - - if (config.saveToFile && !store[`${name}_session`]) { - // loading from file - for (const name in config.users) { - const fileName = path.join(global.output_dir, `${name}_session.json`) - if (!fileExists(fileName)) continue - const data = fs.readFileSync(fileName).toString() - try { - store[`${name}_session`] = JSON.parse(data) - } catch (err) { - throw new Error(`Could not load session from ${fileName}\n${err}`) - } - debug(`Loaded user session for ${name}`) - } - } - - if (isPlaywrightSession() && test?.opts?.cookies) { - if (test.opts.user == name) { - debug(`Cookies already loaded for ${name}`) - // alreadyLoggedIn(name); - return - } else { - debug(`Cookies already loaded for ${test.opts.user}, but not for ${name}`) - await I.deleteCookie() - } - } - const userSession = config.users[name] + const I = container.support('I') const cookies = store[`${name}_session`] const shouldAwait = isAsyncFunction(userSession.login) || isAsyncFunction(userSession.restore) || isAsyncFunction(userSession.check) @@ -370,62 +332,8 @@ module.exports = function (config) { return recorder.promise() } - function enableAuthBeforeEachTest(name) { - const suite = store.currentSuite - if (!suite) return - - debug(`enabling auth as ${name} for each test of suite ${suite.title}`) - - // we are setting test opts so they can be picked up by Playwright if it starts browser for this test - suite.eachTest(test => { - // preload from store - if (store[`${name}_session`]) { - test.opts.cookies = store[`${name}_session`] - test.opts.user = name - return - } - - if (!config.saveToFile) return - const cookieFile = path.join(global.output_dir, `${name}_session.json`) - - if (!fileExists(cookieFile)) { - return - } - - const context = fs.readFileSync(cookieFile).toString() - test.opts.cookies = JSON.parse(context) - test.opts.user = name - }) - - function runLoginFunctionForTest(test) { - if (!suite.tests.includes(test)) return - // let's call this function to ensure that authorization happened - // if no cookies, it will login and save them - loginFunction(name) - } - - // we are in BeforeSuite hook - event.dispatcher.on(event.test.started, runLoginFunctionForTest) - event.dispatcher.on(event.suite.after, () => { - event.dispatcher.off(event.test.started, runLoginFunctionForTest) - }) - } - // adding this to DI container const support = {} support[config.inject] = loginFunction container.append({ support }) - - return loginFunction -} - -function isPlaywrightSession() { - return !!container.helpers('Playwright') -} - -function alreadyLoggedIn(name) { - const step = new FuncStep('am logged in as') - step.actor = 'I' - step.setCallable(() => {}) - return recordStep(step, [name]) } From 06311e990e12cc684b27000e8a380d0c71b5dc7b Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 27 Jan 2025 00:32:36 +0200 Subject: [PATCH 3/5] small fix to screenshot failures --- lib/plugin/screenshotOnFail.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/plugin/screenshotOnFail.js b/lib/plugin/screenshotOnFail.js index b44692722..62dc8658f 100644 --- a/lib/plugin/screenshotOnFail.js +++ b/lib/plugin/screenshotOnFail.js @@ -140,12 +140,8 @@ module.exports = function (config) { }) function _getUUID(test) { - if (test.uuid) { - return test.uuid - } - - if (test.ctx && test.ctx.test.uuid) { - return test.ctx.test.uuid + if (test.uid) { + return test.uid } return Math.floor(new Date().getTime() / 1000) From d6a25ce140082f545a615939ffe064fa2c9570a8 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 27 Jan 2025 01:27:36 +0200 Subject: [PATCH 4/5] fixed screenshot tests --- lib/mocha/cli.js | 22 +++++++++++++++------- lib/plugin/pageInfo.js | 3 --- lib/plugin/screenshotOnFail.js | 4 ++-- lib/plugin/stepByStepReport.js | 2 +- test/unit/plugin/screenshotOnFail_test.js | 13 ++++++++----- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/lib/mocha/cli.js b/lib/mocha/cli.js index 313bb8834..a89e59023 100644 --- a/lib/mocha/cli.js +++ b/lib/mocha/cli.js @@ -198,17 +198,25 @@ class Cli extends Base { // add new line before the message err.message = '\n ' + err.message + // explicitly show file with error + if (test.file) { + log += `\n${output.styles.basic(figures.circle)} ${output.styles.section('File:')} ${output.styles.basic(test.file)}\n` + } + const steps = test.steps || (test.ctx && test.ctx.test.steps) if (steps && steps.length) { let scenarioTrace = '' - steps.reverse().forEach(step => { - const hasFailed = step.status === 'failed' - let line = `${hasFailed ? output.styles.bold(figures.cross) : figures.tick} ${step.toCode()} ${step.line()}` - if (hasFailed) line = output.styles.bold(line) - scenarioTrace += `\n${line}` - }) - log += `${output.styles.basic(figures.circle)} ${output.styles.section('Scenario Steps')}:${scenarioTrace}\n` + steps + .reverse() + .slice(0, 10) + .forEach(step => { + const hasFailed = step.status === 'failed' + let line = `${hasFailed ? output.styles.bold(figures.cross) : figures.tick} ${step.toCode()} ${step.line()}` + if (hasFailed) line = output.styles.bold(line) + scenarioTrace += `\n${line}` + }) + log += `\n${output.styles.basic(figures.circle)} ${output.styles.section('Scenario Steps')}:${scenarioTrace}\n` } // display artifacts in debug mode diff --git a/lib/plugin/pageInfo.js b/lib/plugin/pageInfo.js index 950b500aa..2a1dd3117 100644 --- a/lib/plugin/pageInfo.js +++ b/lib/plugin/pageInfo.js @@ -62,10 +62,7 @@ module.exports = function (config = {}) { }) recorder.add('HTML snapshot failed test', async () => { try { - const currentOutputLevel = output.level() - output.level(0) const html = await helper.grabHTMLFrom('body') - output.level(currentOutputLevel) if (!html) return diff --git a/lib/plugin/screenshotOnFail.js b/lib/plugin/screenshotOnFail.js index 62dc8658f..059a55159 100644 --- a/lib/plugin/screenshotOnFail.js +++ b/lib/plugin/screenshotOnFail.js @@ -64,7 +64,7 @@ module.exports = function (config) { } if (Codeceptjs.container.mocha()) { - options.reportDir = Codeceptjs.container.mocha().options.reporterOptions && Codeceptjs.container.mocha().options.reporterOptions.reportDir + options.reportDir = Codeceptjs.container.mocha()?.options?.reporterOptions && Codeceptjs.container.mocha()?.options?.reporterOptions?.reportDir } if (options.disableScreenshots) { @@ -73,7 +73,7 @@ module.exports = function (config) { } event.dispatcher.on(event.test.failed, (test, _err, hookName) => { - if (hookName == 'BeforeSuite' || hookName == 'AfterSuite') { + if (hookName === 'BeforeSuite' || hookName === 'AfterSuite') { // no browser here return } diff --git a/lib/plugin/stepByStepReport.js b/lib/plugin/stepByStepReport.js index 180ae6c38..52dc64902 100644 --- a/lib/plugin/stepByStepReport.js +++ b/lib/plugin/stepByStepReport.js @@ -122,7 +122,7 @@ module.exports = function (config) { }) event.dispatcher.on(event.test.failed, (test, _err, hookName) => { - if (hookName == 'BeforeSuite' || hookName == 'AfterSuite') { + if (hookName === 'BeforeSuite' || hookName === 'AfterSuite') { // no browser here return } diff --git a/test/unit/plugin/screenshotOnFail_test.js b/test/unit/plugin/screenshotOnFail_test.js index 70d07e501..4b88d84f7 100644 --- a/test/unit/plugin/screenshotOnFail_test.js +++ b/test/unit/plugin/screenshotOnFail_test.js @@ -49,14 +49,17 @@ describe('screenshotOnFail', () => { it('should create screenshot with unique name', async () => { screenshotOnFail({ uniqueScreenshotNames: true }) - event.dispatcher.emit(event.test.failed, { title: 'test1', uuid: 1 }) + + const test = { title: 'test1', uid: 1 } + event.dispatcher.emit(event.test.failed, test) await recorder.promise() expect(screenshotSaved.called).is.ok - expect('test1_1.failed.png').is.equal(screenshotSaved.getCall(0).args[0]) + expect(`test1_${test.uid}.failed.png`).is.equal(screenshotSaved.getCall(0).args[0]) }) - it('should create screenshot with unique name when uuid is null', async () => { + it('should create screenshot with unique name when uid is null', async () => { screenshotOnFail({ uniqueScreenshotNames: true }) + event.dispatcher.emit(event.test.failed, { title: 'test1' }) await recorder.promise() expect(screenshotSaved.called).is.ok @@ -67,14 +70,14 @@ describe('screenshotOnFail', () => { it('should not save screenshot in BeforeSuite', async () => { screenshotOnFail({ uniqueScreenshotNames: true }) - event.dispatcher.emit(event.test.failed, { title: 'test1', ctx: { _runnable: { title: 'hook: BeforeSuite' } } }) + event.dispatcher.emit(event.test.failed, { title: 'test1' }, null, 'BeforeSuite') await recorder.promise() expect(!screenshotSaved.called).is.ok }) it('should not save screenshot in AfterSuite', async () => { screenshotOnFail({ uniqueScreenshotNames: true }) - event.dispatcher.emit(event.test.failed, { title: 'test1', ctx: { _runnable: { title: 'hook: AfterSuite' } } }) + event.dispatcher.emit(event.test.failed, { title: 'test1' }, null, 'AfterSuite') await recorder.promise() expect(!screenshotSaved.called).is.ok }) From ed9b476c6d94ecb6106c6817c9b2de6772044032 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 27 Jan 2025 02:23:26 +0200 Subject: [PATCH 5/5] fixed PW tests --- lib/helper/Playwright.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/helper/Playwright.js b/lib/helper/Playwright.js index 61bc4f752..8bb30d1fb 100644 --- a/lib/helper/Playwright.js +++ b/lib/helper/Playwright.js @@ -525,7 +525,7 @@ class Playwright extends Helper { } // load pre-saved cookies - if (test.opts.cookies) contextOptions.storageState = { cookies: test.opts.cookies } + if (test?.opts?.cookies) contextOptions.storageState = { cookies: test.opts.cookies } if (this.storageState) contextOptions.storageState = this.storageState if (this.options.userAgent) contextOptions.userAgent = this.options.userAgent