From da8351928f29a8c045235574685923caa3f67de1 Mon Sep 17 00:00:00 2001 From: Jaromir Obr Date: Mon, 9 Feb 2026 14:34:47 +0100 Subject: [PATCH] fix: aggregate custom reporter results in run-workers --by suite (#5411) When using run-workers with --by suite, custom reporters (mochawesome, mocha-junit-reporter, etc.) only contained data from the last worker because each worker independently ran the reporter and overwrote the same output files. Strip custom reporters from worker configs and replay aggregated results through the reporter in the main thread after all workers complete. Handles both single-output and config.multiple (per-browser) scenarios with correct path resolution for nested reporter options. Co-Authored-By: Claude Opus 4.6 --- lib/workers.js | 214 ++++++++++++++++-- .../codecept.workers.mochawesome.conf.js | 24 ++ ...ecept.workers.mochawesome.multiple.conf.js | 29 +++ test/runner/run_workers_test.js | 70 ++++++ test/unit/worker_test.js | 54 +++++ 5 files changed, 367 insertions(+), 24 deletions(-) create mode 100644 test/data/sandbox/codecept.workers.mochawesome.conf.js create mode 100644 test/data/sandbox/codecept.workers.mochawesome.multiple.conf.js diff --git a/lib/workers.js b/lib/workers.js index 0ed3a71b3..e9920e82a 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -102,7 +102,7 @@ const simplifyObject = object => { }, {}) } -const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns) => { +const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns, workerOutputGroups) => { selectedRuns = options && options.all && config.multiple ? Object.keys(config.multiple) : selectedRuns if (selectedRuns === undefined || !selectedRuns.length || config.multiple === undefined) { return testGroups.map((tests, index) => { @@ -111,36 +111,23 @@ const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns workerObj.addTests(tests) workerObj.setTestRoot(testRoot) workerObj.addOptions(options) + if (workerOutputGroups) { + workerOutputGroups.set(index, { outputDir: config.output }) + } return workerObj }) } const workersToExecute = [] + const workerOutputDirs = [] const currentOutputFolder = config.output - let currentMochawesomeReportDir - let currentMochaJunitReporterFile - - if (config.mocha && config.mocha.reporterOptions) { - currentMochawesomeReportDir = config.mocha.reporterOptions?.mochawesome.options.reportDir - currentMochaJunitReporterFile = config.mocha.reporterOptions['mocha-junit-reporter'].options.mochaFile - } createRuns(selectedRuns, config).forEach(worker => { const separator = path.sep const _config = { ...config } let workerName = worker.name.replace(':', '_') _config.output = `${currentOutputFolder}${separator}${workerName}` - if (config.mocha && config.mocha.reporterOptions) { - _config.mocha.reporterOptions.mochawesome.options.reportDir = `${currentMochawesomeReportDir}${separator}${workerName}` - - const _tempArray = currentMochaJunitReporterFile.split(separator) - _tempArray.splice( - _tempArray.findIndex(item => item.includes('.xml')), - 0, - workerName, - ) - _config.mocha.reporterOptions['mocha-junit-reporter'].options.mochaFile = _tempArray.join(separator) - } + workerOutputDirs.push(_config.output) workerName = worker.getOriginalName() || worker.getName() const workerConfig = worker.getConfig() workersToExecute.push(getOverridenConfig(workerName, workerConfig, _config)) @@ -149,12 +136,16 @@ const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns let index = 0 testGroups.forEach(tests => { const testWorkerArray = [] - workersToExecute.forEach(finalConfig => { - const workerObj = new WorkerObject(index++) + workersToExecute.forEach((finalConfig, runIndex) => { + const workerObj = new WorkerObject(index) workerObj.addConfig(finalConfig) workerObj.addTests(tests) workerObj.setTestRoot(testRoot) workerObj.addOptions(options) + if (workerOutputGroups) { + workerOutputGroups.set(index, { outputDir: workerOutputDirs[runIndex] }) + } + index++ testWorkerArray.push(workerObj) }) workers.push(...testWorkerArray) @@ -293,6 +284,9 @@ class Workers extends EventEmitter { this.isPoolMode = config.by === 'pool' this.activeWorkers = new Map() this.maxWorkers = numberOfWorkers // Track original worker count for pool mode + this._savedReporterConfig = null + this._workerOutputGroups = new Map() + this._reporterTests = [] createOutputDir(config.testConfig) // Defer worker initialization until codecept is ready @@ -316,7 +310,36 @@ class Workers extends EventEmitter { this.splitTestsByGroups(numberOfWorkers, config) // For function-based grouping, use the actual number of test groups created const actualNumberOfWorkers = isFunction(config.by) ? this.testGroups.length : numberOfWorkers - this.workers = createWorkerObjects(this.testGroups, this.codecept.config, getTestRoot(config.testConfig), config.options, config.selectedRuns) + + let codeceptConfig = this.codecept.config + if (codeceptConfig.mocha && codeceptConfig.mocha.reporter) { + const testRoot = getTestRoot(config.testConfig) + const resolvedOutput = path.isAbsolute(codeceptConfig.output) ? codeceptConfig.output : path.join(testRoot, codeceptConfig.output) + const reporterOpts = codeceptConfig.mocha.reporterOptions ? deepClone(codeceptConfig.mocha.reporterOptions) : {} + const pathKeys = ['reportDir', 'output', 'mochaFile', 'stdout'] + const resolvePathsDeep = (obj) => { + if (!obj || typeof obj !== 'object') return + for (const key of Object.keys(obj)) { + if (pathKeys.includes(key) && typeof obj[key] === 'string' && !path.isAbsolute(obj[key])) { + obj[key] = path.join(testRoot, obj[key]) + } else if (typeof obj[key] === 'object' && obj[key] !== null) { + resolvePathsDeep(obj[key]) + } + } + } + resolvePathsDeep(reporterOpts) + this._savedReporterConfig = { + reporter: codeceptConfig.mocha.reporter, + reporterOptions: reporterOpts, + outputDir: resolvedOutput, + } + codeceptConfig = deepClone(codeceptConfig) + codeceptConfig.mocha.reporter = null + codeceptConfig.mocha.reporterOptions = {} + codeceptConfig.output = resolvedOutput + } + + this.workers = createWorkerObjects(this.testGroups, codeceptConfig, getTestRoot(config.testConfig), config.options, config.selectedRuns, this._workerOutputGroups) this.numberOfWorkers = this.workers.length } @@ -618,8 +641,16 @@ class Workers extends EventEmitter { } if (message.data.tests) { + const outputGroup = this._workerOutputGroups.get(message.workerIndex - 1) message.data.tests.forEach(test => { - Container.result().addTest(deserializeTest(test)) + const deserialized = deserializeTest(test) + if (outputGroup) { + deserialized._outputGroup = outputGroup + } + Container.result().addTest(deserialized) + if (this._savedReporterConfig) { + this._reporterTests.push(deserialized) + } }) } @@ -777,7 +808,142 @@ class Workers extends EventEmitter { this.emit(event.all.result, Container.result()) event.dispatcher.emit(event.workers.result, Container.result()) - this.emit('end') // internal event + + this._runReporters().then(() => { + this.emit('end') + }).catch(err => { + output.error(`Reporter error: ${err.message}`) + this.emit('end') + }) + } + + async _runReporters() { + if (!this._savedReporterConfig) return + + const { reporter: reporterName, reporterOptions: savedReporterOptions, outputDir: defaultOutputDir } = this._savedReporterConfig + + let ReporterClass + try { + if (typeof reporterName === 'function') { + ReporterClass = reporterName + } else { + try { + const mod = await import(reporterName) + ReporterClass = mod.default || mod + } catch { + const { createRequire } = await import('module') + const require = createRequire(import.meta.url) + ReporterClass = require(reporterName) + } + } + } catch (err) { + output.error(`Could not load reporter "${reporterName}": ${err.message}`) + return + } + + const createStatsCollector = (await import('mocha/lib/stats-collector.js')).default + const MochaSuite = (await import('mocha/lib/suite.js')).default + const MochaTest = (await import('mocha/lib/test.js')).default + + const tests = this._reporterTests + const groups = new Map() + + for (const test of tests) { + const groupKey = test._outputGroup?.outputDir || defaultOutputDir + if (!groups.has(groupKey)) { + groups.set(groupKey, { tests: [], outputDir: groupKey }) + } + groups.get(groupKey).tests.push(test) + } + + for (const [, group] of groups) { + try { + const rootSuite = new MochaSuite('', null, true) + rootSuite.root = true + + const suiteMap = new Map() + for (const test of group.tests) { + const suiteTitle = test.parent?.title || 'Suite' + if (!suiteMap.has(suiteTitle)) { + suiteMap.set(suiteTitle, []) + } + suiteMap.get(suiteTitle).push(test) + } + + for (const [suiteTitle, suiteTests] of suiteMap) { + const childSuite = MochaSuite.create(rootSuite, suiteTitle) + childSuite.root = false + for (const test of suiteTests) { + const mochaTest = new MochaTest(test.title, () => {}) + mochaTest.state = test.state + mochaTest.duration = test.duration || 0 + mochaTest.speed = test.duration > 75 ? 'slow' : (test.duration > 37 ? 'medium' : 'fast') + mochaTest.file = test.file + mochaTest.pending = test.state === 'pending' || test.state === 'skipped' + if (test.err) { + const err = new Error(test.err.message || '') + err.stack = test.err.stack || '' + err.name = test.err.name || 'Error' + err.actual = test.err.actual + err.expected = test.err.expected + mochaTest.err = err + } + childSuite.addTest(mochaTest) + } + } + + let reporterOptions = savedReporterOptions ? deepClone(savedReporterOptions) : {} + if (group.outputDir && group.outputDir !== defaultOutputDir) { + mkdirp.sync(group.outputDir) + reporterOptions = replaceValueDeep(reporterOptions, 'reportDir', group.outputDir) + reporterOptions = replaceValueDeep(reporterOptions, 'output', group.outputDir) + reporterOptions = replaceValueDeep(reporterOptions, 'mochaFile', path.join(group.outputDir, 'report.xml')) + reporterOptions = replaceValueDeep(reporterOptions, 'stdout', path.join(group.outputDir, 'console.log')) + } + + const runner = new EventEmitter() + runner.suite = rootSuite + runner.total = group.tests.length + runner.failures = 0 + createStatsCollector(runner) + + const cliKey = 'codeceptjs-cli-reporter' + const internalCliKey = 'codeceptjs/lib/mocha/cli' + const filteredOptions = { ...reporterOptions } + delete filteredOptions[cliKey] + delete filteredOptions[internalCliKey] + + const reporterInstance = new ReporterClass(runner, { reporterOption: filteredOptions, reporterOptions: filteredOptions }) + + runner.emit('start') + + for (const childSuite of rootSuite.suites) { + runner.emit('suite', childSuite) + for (const test of childSuite.tests) { + runner.emit('test', test) + if (test.pending) { + runner.emit('pending', test) + } else if (test.state === 'passed') { + runner.emit('pass', test) + } else if (test.state === 'failed') { + runner.emit('fail', test, test.err) + } + runner.emit('test end', test) + } + runner.emit('suite end', childSuite) + } + + runner.emit('end') + + if (typeof reporterInstance.done === 'function') { + await new Promise((resolve) => { + reporterInstance.done(runner.failures || 0, resolve) + }) + } + } catch (err) { + output.error(`Reporter error: ${err.message}`) + } + } } printResults() { diff --git a/test/data/sandbox/codecept.workers.mochawesome.conf.js b/test/data/sandbox/codecept.workers.mochawesome.conf.js new file mode 100644 index 000000000..d927100e0 --- /dev/null +++ b/test/data/sandbox/codecept.workers.mochawesome.conf.js @@ -0,0 +1,24 @@ +export const config = { + tests: './workers/*.js', + timeout: 10000, + output: './output/workers_mochawesome', + helpers: { + FileSystem: {}, + Workers: { + require: './workers_helper', + }, + }, + include: {}, + mocha: { + reporter: 'mochawesome', + reporterOptions: { + reportDir: './output/workers_mochawesome', + reportFilename: 'report', + quiet: true, + overwrite: true, + json: true, + html: true, + }, + }, + name: 'sandbox', +}; diff --git a/test/data/sandbox/codecept.workers.mochawesome.multiple.conf.js b/test/data/sandbox/codecept.workers.mochawesome.multiple.conf.js new file mode 100644 index 000000000..c234c55fd --- /dev/null +++ b/test/data/sandbox/codecept.workers.mochawesome.multiple.conf.js @@ -0,0 +1,29 @@ +export const config = { + tests: './workers/*.js', + timeout: 10000, + output: './output/workers_mochawesome', + helpers: { + FileSystem: {}, + Workers: { + require: './workers_helper', + }, + }, + include: {}, + mocha: { + reporter: 'mochawesome', + reporterOptions: { + reportDir: './output/workers_mochawesome', + reportFilename: 'report', + quiet: true, + overwrite: true, + json: true, + html: false, + }, + }, + multiple: { + parallel: { + browsers: ['chrome', 'firefox'], + }, + }, + name: 'sandbox', +}; diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index e7b03f268..18fa8ebcf 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -530,4 +530,74 @@ describe('CodeceptJS Workers Runner', function () { done() }) }) + + it('should aggregate results from all suites in mochawesome report when using --by suite (#5411)', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + + const outputDir = path.join(codecept_dir, 'output/workers_mochawesome') + if (fs.existsSync(outputDir)) { + fs.rmSync(outputDir, { recursive: true }) + } + + const configPath = path.join(codecept_dir, 'codecept.workers.mochawesome.conf.js') + exec(`${runner} run-workers --config ${configPath} 2 --by suite`, (_, stdout) => { + expect(stdout).toContain('Running tests in 2 workers') + + const reportPath = path.join(outputDir, 'report.json') + expect(fs.existsSync(reportPath)).toBe(true) + + const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')) + const suiteNames = report.results[0].suites.map(s => s.title) + + expect(suiteNames).toContain('Workers') + expect(suiteNames).toContain('@feature_grep in worker') + expect(suiteNames.length).toBeGreaterThanOrEqual(3) + + const totalTests = report.results[0].suites.reduce((sum, s) => sum + s.tests.length, 0) + expect(totalTests).toBeGreaterThanOrEqual(5) + + const htmlPath = path.join(outputDir, 'report.html') + expect(fs.existsSync(htmlPath)).toBe(true) + const html = fs.readFileSync(htmlPath, 'utf8') + expect(html).toContain('Workers') + expect(html).toContain('@feature_grep in worker') + expect(html).toContain('Retry Workers') + + fs.rmSync(outputDir, { recursive: true }) + done() + }) + }) + + it('should aggregate results per browser in mochawesome report with config.multiple (#5411)', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + + const outputDir = path.join(codecept_dir, 'output/workers_mochawesome') + if (fs.existsSync(outputDir)) { + fs.rmSync(outputDir, { recursive: true }) + } + + const configPath = path.join(codecept_dir, 'codecept.workers.mochawesome.multiple.conf.js') + exec(`${runner} run-workers --config ${configPath} 2 --by suite parallel`, (_, stdout) => { + expect(stdout).toContain('Running tests in') + + for (const browser of ['parallel_chrome1', 'parallel_firefox1']) { + const browserDir = path.join(outputDir, browser) + const reportPath = path.join(browserDir, 'report.json') + expect(fs.existsSync(reportPath)).toBe(true) + + const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')) + const suiteNames = report.results[0].suites.map(s => s.title) + + expect(suiteNames).toContain('Workers') + expect(suiteNames).toContain('@feature_grep in worker') + expect(suiteNames.length).toBeGreaterThanOrEqual(3) + + const totalTests = report.results[0].suites.reduce((sum, s) => sum + s.tests.length, 0) + expect(totalTests).toBeGreaterThanOrEqual(5) + } + + fs.rmSync(outputDir, { recursive: true }) + done() + }) + }) }) diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index 635f00b72..f8870cdc4 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -382,4 +382,58 @@ describe('Workers', function () { workers.run() }) + + it('should aggregate results from all suites for custom reporters when splitting by suite (issue #5411)', async () => { + const receivedTests = [] + const receivedSuites = [] + let endCalled = false + + function TestReporter(runner) { + runner.on('suite', suite => { + if (!suite.root) receivedSuites.push(suite.title) + }) + runner.on('test end', test => { + receivedTests.push(test.title) + }) + runner.on('end', () => { + endCalled = true + }) + } + + const workerConfig = { + by: 'suite', + testConfig: './test/data/sandbox/codecept.workers.conf.js', + } + + const workers = new Workers(2, workerConfig) + await workers._ensureInitialized() + + workers._savedReporterConfig = { + reporter: TestReporter, + reporterOptions: {}, + outputDir: workers.codecept.config.output, + } + + const { deserializeTest } = await import('../../lib/mocha/test.js') + const fakeTests = [ + { title: 'Scenario 1', state: 'passed', duration: 5, parent: { title: 'SuiteA' }, uid: 'suiteA-1' }, + { title: 'Scenario 2', state: 'passed', duration: 3, parent: { title: 'SuiteA' }, uid: 'suiteA-2' }, + { title: 'Scenario 1', state: 'passed', duration: 4, parent: { title: 'SuiteB' }, uid: 'suiteB-1' }, + { title: 'Scenario 2', state: 'failed', duration: 2, parent: { title: 'SuiteB' }, uid: 'suiteB-2', err: { message: 'test failed', stack: 'Error: test failed' } }, + ] + + for (const test of fakeTests) { + const deserialized = deserializeTest(test) + Container.result().addTest(deserialized) + workers._reporterTests.push(deserialized) + } + Container.result().addStats({ passes: 3, failures: 1, tests: 4, pending: 0 }) + + await workers._runReporters() + + expect(endCalled).to.equal(true, 'Reporter end event should be called') + expect(receivedSuites).to.include('SuiteA') + expect(receivedSuites).to.include('SuiteB') + expect(receivedTests.length).to.equal(4, 'Reporter should receive all 4 tests') + }) })