diff --git a/lib/workers.js b/lib/workers.js index 0ed3a71b3..e9920e82a 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -102,7 +102,7 @@ const simplifyObject = object => { }, {}) } -const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns) => { +const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns, workerOutputGroups) => { selectedRuns = options && options.all && config.multiple ? Object.keys(config.multiple) : selectedRuns if (selectedRuns === undefined || !selectedRuns.length || config.multiple === undefined) { return testGroups.map((tests, index) => { @@ -111,36 +111,23 @@ const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns workerObj.addTests(tests) workerObj.setTestRoot(testRoot) workerObj.addOptions(options) + if (workerOutputGroups) { + workerOutputGroups.set(index, { outputDir: config.output }) + } return workerObj }) } const workersToExecute = [] + const workerOutputDirs = [] const currentOutputFolder = config.output - let currentMochawesomeReportDir - let currentMochaJunitReporterFile - - if (config.mocha && config.mocha.reporterOptions) { - currentMochawesomeReportDir = config.mocha.reporterOptions?.mochawesome.options.reportDir - currentMochaJunitReporterFile = config.mocha.reporterOptions['mocha-junit-reporter'].options.mochaFile - } createRuns(selectedRuns, config).forEach(worker => { const separator = path.sep const _config = { ...config } let workerName = worker.name.replace(':', '_') _config.output = `${currentOutputFolder}${separator}${workerName}` - if (config.mocha && config.mocha.reporterOptions) { - _config.mocha.reporterOptions.mochawesome.options.reportDir = `${currentMochawesomeReportDir}${separator}${workerName}` - - const _tempArray = currentMochaJunitReporterFile.split(separator) - _tempArray.splice( - _tempArray.findIndex(item => item.includes('.xml')), - 0, - workerName, - ) - _config.mocha.reporterOptions['mocha-junit-reporter'].options.mochaFile = _tempArray.join(separator) - } + workerOutputDirs.push(_config.output) workerName = worker.getOriginalName() || worker.getName() const workerConfig = worker.getConfig() workersToExecute.push(getOverridenConfig(workerName, workerConfig, _config)) @@ -149,12 +136,16 @@ const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns let index = 0 testGroups.forEach(tests => { const testWorkerArray = [] - workersToExecute.forEach(finalConfig => { - const workerObj = new WorkerObject(index++) + workersToExecute.forEach((finalConfig, runIndex) => { + const workerObj = new WorkerObject(index) workerObj.addConfig(finalConfig) workerObj.addTests(tests) workerObj.setTestRoot(testRoot) workerObj.addOptions(options) + if (workerOutputGroups) { + workerOutputGroups.set(index, { outputDir: workerOutputDirs[runIndex] }) + } + index++ testWorkerArray.push(workerObj) }) workers.push(...testWorkerArray) @@ -293,6 +284,9 @@ class Workers extends EventEmitter { this.isPoolMode = config.by === 'pool' this.activeWorkers = new Map() this.maxWorkers = numberOfWorkers // Track original worker count for pool mode + this._savedReporterConfig = null + this._workerOutputGroups = new Map() + this._reporterTests = [] createOutputDir(config.testConfig) // Defer worker initialization until codecept is ready @@ -316,7 +310,36 @@ class Workers extends EventEmitter { this.splitTestsByGroups(numberOfWorkers, config) // For function-based grouping, use the actual number of test groups created const actualNumberOfWorkers = isFunction(config.by) ? this.testGroups.length : numberOfWorkers - this.workers = createWorkerObjects(this.testGroups, this.codecept.config, getTestRoot(config.testConfig), config.options, config.selectedRuns) + + let codeceptConfig = this.codecept.config + if (codeceptConfig.mocha && codeceptConfig.mocha.reporter) { + const testRoot = getTestRoot(config.testConfig) + const resolvedOutput = path.isAbsolute(codeceptConfig.output) ? codeceptConfig.output : path.join(testRoot, codeceptConfig.output) + const reporterOpts = codeceptConfig.mocha.reporterOptions ? deepClone(codeceptConfig.mocha.reporterOptions) : {} + const pathKeys = ['reportDir', 'output', 'mochaFile', 'stdout'] + const resolvePathsDeep = (obj) => { + if (!obj || typeof obj !== 'object') return + for (const key of Object.keys(obj)) { + if (pathKeys.includes(key) && typeof obj[key] === 'string' && !path.isAbsolute(obj[key])) { + obj[key] = path.join(testRoot, obj[key]) + } else if (typeof obj[key] === 'object' && obj[key] !== null) { + resolvePathsDeep(obj[key]) + } + } + } + resolvePathsDeep(reporterOpts) + this._savedReporterConfig = { + reporter: codeceptConfig.mocha.reporter, + reporterOptions: reporterOpts, + outputDir: resolvedOutput, + } + codeceptConfig = deepClone(codeceptConfig) + codeceptConfig.mocha.reporter = null + codeceptConfig.mocha.reporterOptions = {} + codeceptConfig.output = resolvedOutput + } + + this.workers = createWorkerObjects(this.testGroups, codeceptConfig, getTestRoot(config.testConfig), config.options, config.selectedRuns, this._workerOutputGroups) this.numberOfWorkers = this.workers.length } @@ -618,8 +641,16 @@ class Workers extends EventEmitter { } if (message.data.tests) { + const outputGroup = this._workerOutputGroups.get(message.workerIndex - 1) message.data.tests.forEach(test => { - Container.result().addTest(deserializeTest(test)) + const deserialized = deserializeTest(test) + if (outputGroup) { + deserialized._outputGroup = outputGroup + } + Container.result().addTest(deserialized) + if (this._savedReporterConfig) { + this._reporterTests.push(deserialized) + } }) } @@ -777,7 +808,142 @@ class Workers extends EventEmitter { this.emit(event.all.result, Container.result()) event.dispatcher.emit(event.workers.result, Container.result()) - this.emit('end') // internal event + + this._runReporters().then(() => { + this.emit('end') + }).catch(err => { + output.error(`Reporter error: ${err.message}`) + this.emit('end') + }) + } + + async _runReporters() { + if (!this._savedReporterConfig) return + + const { reporter: reporterName, reporterOptions: savedReporterOptions, outputDir: defaultOutputDir } = this._savedReporterConfig + + let ReporterClass + try { + if (typeof reporterName === 'function') { + ReporterClass = reporterName + } else { + try { + const mod = await import(reporterName) + ReporterClass = mod.default || mod + } catch { + const { createRequire } = await import('module') + const require = createRequire(import.meta.url) + ReporterClass = require(reporterName) + } + } + } catch (err) { + output.error(`Could not load reporter "${reporterName}": ${err.message}`) + return + } + + const createStatsCollector = (await import('mocha/lib/stats-collector.js')).default + const MochaSuite = (await import('mocha/lib/suite.js')).default + const MochaTest = (await import('mocha/lib/test.js')).default + + const tests = this._reporterTests + const groups = new Map() + + for (const test of tests) { + const groupKey = test._outputGroup?.outputDir || defaultOutputDir + if (!groups.has(groupKey)) { + groups.set(groupKey, { tests: [], outputDir: groupKey }) + } + groups.get(groupKey).tests.push(test) + } + + for (const [, group] of groups) { + try { + const rootSuite = new MochaSuite('', null, true) + rootSuite.root = true + + const suiteMap = new Map() + for (const test of group.tests) { + const suiteTitle = test.parent?.title || 'Suite' + if (!suiteMap.has(suiteTitle)) { + suiteMap.set(suiteTitle, []) + } + suiteMap.get(suiteTitle).push(test) + } + + for (const [suiteTitle, suiteTests] of suiteMap) { + const childSuite = MochaSuite.create(rootSuite, suiteTitle) + childSuite.root = false + for (const test of suiteTests) { + const mochaTest = new MochaTest(test.title, () => {}) + mochaTest.state = test.state + mochaTest.duration = test.duration || 0 + mochaTest.speed = test.duration > 75 ? 'slow' : (test.duration > 37 ? 'medium' : 'fast') + mochaTest.file = test.file + mochaTest.pending = test.state === 'pending' || test.state === 'skipped' + if (test.err) { + const err = new Error(test.err.message || '') + err.stack = test.err.stack || '' + err.name = test.err.name || 'Error' + err.actual = test.err.actual + err.expected = test.err.expected + mochaTest.err = err + } + childSuite.addTest(mochaTest) + } + } + + let reporterOptions = savedReporterOptions ? deepClone(savedReporterOptions) : {} + if (group.outputDir && group.outputDir !== defaultOutputDir) { + mkdirp.sync(group.outputDir) + reporterOptions = replaceValueDeep(reporterOptions, 'reportDir', group.outputDir) + reporterOptions = replaceValueDeep(reporterOptions, 'output', group.outputDir) + reporterOptions = replaceValueDeep(reporterOptions, 'mochaFile', path.join(group.outputDir, 'report.xml')) + reporterOptions = replaceValueDeep(reporterOptions, 'stdout', path.join(group.outputDir, 'console.log')) + } + + const runner = new EventEmitter() + runner.suite = rootSuite + runner.total = group.tests.length + runner.failures = 0 + createStatsCollector(runner) + + const cliKey = 'codeceptjs-cli-reporter' + const internalCliKey = 'codeceptjs/lib/mocha/cli' + const filteredOptions = { ...reporterOptions } + delete filteredOptions[cliKey] + delete filteredOptions[internalCliKey] + + const reporterInstance = new ReporterClass(runner, { reporterOption: filteredOptions, reporterOptions: filteredOptions }) + + runner.emit('start') + + for (const childSuite of rootSuite.suites) { + runner.emit('suite', childSuite) + for (const test of childSuite.tests) { + runner.emit('test', test) + if (test.pending) { + runner.emit('pending', test) + } else if (test.state === 'passed') { + runner.emit('pass', test) + } else if (test.state === 'failed') { + runner.emit('fail', test, test.err) + } + runner.emit('test end', test) + } + runner.emit('suite end', childSuite) + } + + runner.emit('end') + + if (typeof reporterInstance.done === 'function') { + await new Promise((resolve) => { + reporterInstance.done(runner.failures || 0, resolve) + }) + } + } catch (err) { + output.error(`Reporter error: ${err.message}`) + } + } } printResults() { diff --git a/test/data/sandbox/codecept.workers.mochawesome.conf.js b/test/data/sandbox/codecept.workers.mochawesome.conf.js new file mode 100644 index 000000000..d927100e0 --- /dev/null +++ b/test/data/sandbox/codecept.workers.mochawesome.conf.js @@ -0,0 +1,24 @@ +export const config = { + tests: './workers/*.js', + timeout: 10000, + output: './output/workers_mochawesome', + helpers: { + FileSystem: {}, + Workers: { + require: './workers_helper', + }, + }, + include: {}, + mocha: { + reporter: 'mochawesome', + reporterOptions: { + reportDir: './output/workers_mochawesome', + reportFilename: 'report', + quiet: true, + overwrite: true, + json: true, + html: true, + }, + }, + name: 'sandbox', +}; diff --git a/test/data/sandbox/codecept.workers.mochawesome.multiple.conf.js b/test/data/sandbox/codecept.workers.mochawesome.multiple.conf.js new file mode 100644 index 000000000..c234c55fd --- /dev/null +++ b/test/data/sandbox/codecept.workers.mochawesome.multiple.conf.js @@ -0,0 +1,29 @@ +export const config = { + tests: './workers/*.js', + timeout: 10000, + output: './output/workers_mochawesome', + helpers: { + FileSystem: {}, + Workers: { + require: './workers_helper', + }, + }, + include: {}, + mocha: { + reporter: 'mochawesome', + reporterOptions: { + reportDir: './output/workers_mochawesome', + reportFilename: 'report', + quiet: true, + overwrite: true, + json: true, + html: false, + }, + }, + multiple: { + parallel: { + browsers: ['chrome', 'firefox'], + }, + }, + name: 'sandbox', +}; diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index e7b03f268..18fa8ebcf 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -530,4 +530,74 @@ describe('CodeceptJS Workers Runner', function () { done() }) }) + + it('should aggregate results from all suites in mochawesome report when using --by suite (#5411)', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + + const outputDir = path.join(codecept_dir, 'output/workers_mochawesome') + if (fs.existsSync(outputDir)) { + fs.rmSync(outputDir, { recursive: true }) + } + + const configPath = path.join(codecept_dir, 'codecept.workers.mochawesome.conf.js') + exec(`${runner} run-workers --config ${configPath} 2 --by suite`, (_, stdout) => { + expect(stdout).toContain('Running tests in 2 workers') + + const reportPath = path.join(outputDir, 'report.json') + expect(fs.existsSync(reportPath)).toBe(true) + + const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')) + const suiteNames = report.results[0].suites.map(s => s.title) + + expect(suiteNames).toContain('Workers') + expect(suiteNames).toContain('@feature_grep in worker') + expect(suiteNames.length).toBeGreaterThanOrEqual(3) + + const totalTests = report.results[0].suites.reduce((sum, s) => sum + s.tests.length, 0) + expect(totalTests).toBeGreaterThanOrEqual(5) + + const htmlPath = path.join(outputDir, 'report.html') + expect(fs.existsSync(htmlPath)).toBe(true) + const html = fs.readFileSync(htmlPath, 'utf8') + expect(html).toContain('Workers') + expect(html).toContain('@feature_grep in worker') + expect(html).toContain('Retry Workers') + + fs.rmSync(outputDir, { recursive: true }) + done() + }) + }) + + it('should aggregate results per browser in mochawesome report with config.multiple (#5411)', function (done) { + if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version') + + const outputDir = path.join(codecept_dir, 'output/workers_mochawesome') + if (fs.existsSync(outputDir)) { + fs.rmSync(outputDir, { recursive: true }) + } + + const configPath = path.join(codecept_dir, 'codecept.workers.mochawesome.multiple.conf.js') + exec(`${runner} run-workers --config ${configPath} 2 --by suite parallel`, (_, stdout) => { + expect(stdout).toContain('Running tests in') + + for (const browser of ['parallel_chrome1', 'parallel_firefox1']) { + const browserDir = path.join(outputDir, browser) + const reportPath = path.join(browserDir, 'report.json') + expect(fs.existsSync(reportPath)).toBe(true) + + const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')) + const suiteNames = report.results[0].suites.map(s => s.title) + + expect(suiteNames).toContain('Workers') + expect(suiteNames).toContain('@feature_grep in worker') + expect(suiteNames.length).toBeGreaterThanOrEqual(3) + + const totalTests = report.results[0].suites.reduce((sum, s) => sum + s.tests.length, 0) + expect(totalTests).toBeGreaterThanOrEqual(5) + } + + fs.rmSync(outputDir, { recursive: true }) + done() + }) + }) }) diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index 635f00b72..f8870cdc4 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -382,4 +382,58 @@ describe('Workers', function () { workers.run() }) + + it('should aggregate results from all suites for custom reporters when splitting by suite (issue #5411)', async () => { + const receivedTests = [] + const receivedSuites = [] + let endCalled = false + + function TestReporter(runner) { + runner.on('suite', suite => { + if (!suite.root) receivedSuites.push(suite.title) + }) + runner.on('test end', test => { + receivedTests.push(test.title) + }) + runner.on('end', () => { + endCalled = true + }) + } + + const workerConfig = { + by: 'suite', + testConfig: './test/data/sandbox/codecept.workers.conf.js', + } + + const workers = new Workers(2, workerConfig) + await workers._ensureInitialized() + + workers._savedReporterConfig = { + reporter: TestReporter, + reporterOptions: {}, + outputDir: workers.codecept.config.output, + } + + const { deserializeTest } = await import('../../lib/mocha/test.js') + const fakeTests = [ + { title: 'Scenario 1', state: 'passed', duration: 5, parent: { title: 'SuiteA' }, uid: 'suiteA-1' }, + { title: 'Scenario 2', state: 'passed', duration: 3, parent: { title: 'SuiteA' }, uid: 'suiteA-2' }, + { title: 'Scenario 1', state: 'passed', duration: 4, parent: { title: 'SuiteB' }, uid: 'suiteB-1' }, + { title: 'Scenario 2', state: 'failed', duration: 2, parent: { title: 'SuiteB' }, uid: 'suiteB-2', err: { message: 'test failed', stack: 'Error: test failed' } }, + ] + + for (const test of fakeTests) { + const deserialized = deserializeTest(test) + Container.result().addTest(deserialized) + workers._reporterTests.push(deserialized) + } + Container.result().addStats({ passes: 3, failures: 1, tests: 4, pending: 0 }) + + await workers._runReporters() + + expect(endCalled).to.equal(true, 'Reporter end event should be called') + expect(receivedSuites).to.include('SuiteA') + expect(receivedSuites).to.include('SuiteB') + expect(receivedTests.length).to.equal(4, 'Reporter should receive all 4 tests') + }) })