fix: aggregate custom reporter results in run-workers --by suite (#5411)

lib/workers.js

@@ -102,7 +102,7 @@ const simplifyObject = object => {
     }, {})
 }
 
-const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns) => {
+const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns, workerOutputGroups) => {
   selectedRuns = options && options.all && config.multiple ? Object.keys(config.multiple) : selectedRuns
   if (selectedRuns === undefined || !selectedRuns.length || config.multiple === undefined) {
     return testGroups.map((tests, index) => {
@@ -111,36 +111,23 @@ const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns
       workerObj.addTests(tests)
       workerObj.setTestRoot(testRoot)
       workerObj.addOptions(options)
+      if (workerOutputGroups) {
+        workerOutputGroups.set(index, { outputDir: config.output })
+      }
       return workerObj
     })
   }
   const workersToExecute = []
+  const workerOutputDirs = []
 
   const currentOutputFolder = config.output
-  let currentMochawesomeReportDir
-  let currentMochaJunitReporterFile
-
-  if (config.mocha && config.mocha.reporterOptions) {
-    currentMochawesomeReportDir = config.mocha.reporterOptions?.mochawesome.options.reportDir
-    currentMochaJunitReporterFile = config.mocha.reporterOptions['mocha-junit-reporter'].options.mochaFile
-  }
 
   createRuns(selectedRuns, config).forEach(worker => {
     const separator = path.sep
     const _config = { ...config }
     let workerName = worker.name.replace(':', '_')
     _config.output = `${currentOutputFolder}${separator}${workerName}`
-    if (config.mocha && config.mocha.reporterOptions) {
-      _config.mocha.reporterOptions.mochawesome.options.reportDir = `${currentMochawesomeReportDir}${separator}${workerName}`
-
-      const _tempArray = currentMochaJunitReporterFile.split(separator)
-      _tempArray.splice(
-        _tempArray.findIndex(item => item.includes('.xml')),
-        0,
-        workerName,
-      )
-      _config.mocha.reporterOptions['mocha-junit-reporter'].options.mochaFile = _tempArray.join(separator)
-    }
+    workerOutputDirs.push(_config.output)
     workerName = worker.getOriginalName() || worker.getName()
     const workerConfig = worker.getConfig()
     workersToExecute.push(getOverridenConfig(workerName, workerConfig, _config))
@@ -149,12 +136,16 @@ const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns
   let index = 0
   testGroups.forEach(tests => {
     const testWorkerArray = []
-    workersToExecute.forEach(finalConfig => {
-      const workerObj = new WorkerObject(index++)
+    workersToExecute.forEach((finalConfig, runIndex) => {
+      const workerObj = new WorkerObject(index)
       workerObj.addConfig(finalConfig)
       workerObj.addTests(tests)
       workerObj.setTestRoot(testRoot)
       workerObj.addOptions(options)
+      if (workerOutputGroups) {
+        workerOutputGroups.set(index, { outputDir: workerOutputDirs[runIndex] })
+      }
+      index++
       testWorkerArray.push(workerObj)
     })
     workers.push(...testWorkerArray)
@@ -293,6 +284,9 @@ class Workers extends EventEmitter {
     this.isPoolMode = config.by === 'pool'
     this.activeWorkers = new Map()
     this.maxWorkers = numberOfWorkers // Track original worker count for pool mode
+    this._savedReporterConfig = null
+    this._workerOutputGroups = new Map()
+    this._reporterTests = []
 
     createOutputDir(config.testConfig)
     // Defer worker initialization until codecept is ready
@@ -316,7 +310,36 @@ class Workers extends EventEmitter {
     this.splitTestsByGroups(numberOfWorkers, config)
     // For function-based grouping, use the actual number of test groups created
     const actualNumberOfWorkers = isFunction(config.by) ? this.testGroups.length : numberOfWorkers
-    this.workers = createWorkerObjects(this.testGroups, this.codecept.config, getTestRoot(config.testConfig), config.options, config.selectedRuns)
+
+    let codeceptConfig = this.codecept.config
+    if (codeceptConfig.mocha && codeceptConfig.mocha.reporter) {
+      const testRoot = getTestRoot(config.testConfig)
+      const resolvedOutput = path.isAbsolute(codeceptConfig.output) ? codeceptConfig.output : path.join(testRoot, codeceptConfig.output)
+      const reporterOpts = codeceptConfig.mocha.reporterOptions ? deepClone(codeceptConfig.mocha.reporterOptions) : {}
+      const pathKeys = ['reportDir', 'output', 'mochaFile', 'stdout']
+      const resolvePathsDeep = (obj) => {
+        if (!obj || typeof obj !== 'object') return
+        for (const key of Object.keys(obj)) {
+          if (pathKeys.includes(key) && typeof obj[key] === 'string' && !path.isAbsolute(obj[key])) {
+            obj[key] = path.join(testRoot, obj[key])
+          } else if (typeof obj[key] === 'object' && obj[key] !== null) {
+            resolvePathsDeep(obj[key])
+          }
+        }
+      }
+      resolvePathsDeep(reporterOpts)
+      this._savedReporterConfig = {
+        reporter: codeceptConfig.mocha.reporter,
+        reporterOptions: reporterOpts,
+        outputDir: resolvedOutput,
+      }
+      codeceptConfig = deepClone(codeceptConfig)
+      codeceptConfig.mocha.reporter = null
+      codeceptConfig.mocha.reporterOptions = {}
+      codeceptConfig.output = resolvedOutput
+    }
+
+    this.workers = createWorkerObjects(this.testGroups, codeceptConfig, getTestRoot(config.testConfig), config.options, config.selectedRuns, this._workerOutputGroups)
     this.numberOfWorkers = this.workers.length
   }
 
@@ -618,8 +641,16 @@ class Workers extends EventEmitter {
           }
 
           if (message.data.tests) {
+            const outputGroup = this._workerOutputGroups.get(message.workerIndex - 1)
             message.data.tests.forEach(test => {
-              Container.result().addTest(deserializeTest(test))
+              const deserialized = deserializeTest(test)
+              if (outputGroup) {
+                deserialized._outputGroup = outputGroup
+              }
+              Container.result().addTest(deserialized)
+              if (this._savedReporterConfig) {
+                this._reporterTests.push(deserialized)
+              }
             })
           }
 
@@ -777,7 +808,142 @@ class Workers extends EventEmitter {
 
     this.emit(event.all.result, Container.result())
     event.dispatcher.emit(event.workers.result, Container.result())
-    this.emit('end') // internal event
+
+    this._runReporters().then(() => {
+      this.emit('end')
+    }).catch(err => {
+      output.error(`Reporter error: ${err.message}`)
+      this.emit('end')
+    })
+  }
+
+  async _runReporters() {
+    if (!this._savedReporterConfig) return
+
+    const { reporter: reporterName, reporterOptions: savedReporterOptions, outputDir: defaultOutputDir } = this._savedReporterConfig
+
+    let ReporterClass
+    try {
+      if (typeof reporterName === 'function') {
+        ReporterClass = reporterName
+      } else {
+        try {
+          const mod = await import(reporterName)
+          ReporterClass = mod.default || mod
+        } catch {
+          const { createRequire } = await import('module')
+          const require = createRequire(import.meta.url)
+          ReporterClass = require(reporterName)
+        }
+      }
+    } catch (err) {
+      output.error(`Could not load reporter "${reporterName}": ${err.message}`)
+      return
+    }
+
+    const createStatsCollector = (await import('mocha/lib/stats-collector.js')).default
+    const MochaSuite = (await import('mocha/lib/suite.js')).default
+    const MochaTest = (await import('mocha/lib/test.js')).default
+
+    const tests = this._reporterTests
+    const groups = new Map()
+
+    for (const test of tests) {
+      const groupKey = test._outputGroup?.outputDir || defaultOutputDir
+      if (!groups.has(groupKey)) {
+        groups.set(groupKey, { tests: [], outputDir: groupKey })
+      }
+      groups.get(groupKey).tests.push(test)
+    }
+
+    for (const [, group] of groups) {
+      try {
+        const rootSuite = new MochaSuite('', null, true)
+        rootSuite.root = true
+
+        const suiteMap = new Map()
+        for (const test of group.tests) {
+          const suiteTitle = test.parent?.title || 'Suite'
+          if (!suiteMap.has(suiteTitle)) {
+            suiteMap.set(suiteTitle, [])
+          }
+          suiteMap.get(suiteTitle).push(test)
+        }
+
+        for (const [suiteTitle, suiteTests] of suiteMap) {
+          const childSuite = MochaSuite.create(rootSuite, suiteTitle)
+          childSuite.root = false
+          for (const test of suiteTests) {
+            const mochaTest = new MochaTest(test.title, () => {})
+            mochaTest.state = test.state
+            mochaTest.duration = test.duration || 0
+            mochaTest.speed = test.duration > 75 ? 'slow' : (test.duration > 37 ? 'medium' : 'fast')
+            mochaTest.file = test.file
+            mochaTest.pending = test.state === 'pending' || test.state === 'skipped'
+            if (test.err) {
+              const err = new Error(test.err.message || '')
+              err.stack = test.err.stack || ''
+              err.name = test.err.name || 'Error'
+              err.actual = test.err.actual
+              err.expected = test.err.expected
+              mochaTest.err = err
+            }
+            childSuite.addTest(mochaTest)
+          }
+        }
+
+        let reporterOptions = savedReporterOptions ? deepClone(savedReporterOptions) : {}
+        if (group.outputDir && group.outputDir !== defaultOutputDir) {
+          mkdirp.sync(group.outputDir)
+          reporterOptions = replaceValueDeep(reporterOptions, 'reportDir', group.outputDir)
+          reporterOptions = replaceValueDeep(reporterOptions, 'output', group.outputDir)
+          reporterOptions = replaceValueDeep(reporterOptions, 'mochaFile', path.join(group.outputDir, 'report.xml'))
+          reporterOptions = replaceValueDeep(reporterOptions, 'stdout', path.join(group.outputDir, 'console.log'))
+        }
+
+        const runner = new EventEmitter()
+        runner.suite = rootSuite
+        runner.total = group.tests.length
+        runner.failures = 0
+        createStatsCollector(runner)
+
+        const cliKey = 'codeceptjs-cli-reporter'
+        const internalCliKey = 'codeceptjs/lib/mocha/cli'
+        const filteredOptions = { ...reporterOptions }
+        delete filteredOptions[cliKey]
+        delete filteredOptions[internalCliKey]
+
+        const reporterInstance = new ReporterClass(runner, { reporterOption: filteredOptions, reporterOptions: filteredOptions })
+
+        runner.emit('start')
+
+        for (const childSuite of rootSuite.suites) {
+          runner.emit('suite', childSuite)
+          for (const test of childSuite.tests) {
+            runner.emit('test', test)
+            if (test.pending) {
+              runner.emit('pending', test)
+            } else if (test.state === 'passed') {
+              runner.emit('pass', test)
+            } else if (test.state === 'failed') {
+              runner.emit('fail', test, test.err)
+            }
+            runner.emit('test end', test)
+          }
+          runner.emit('suite end', childSuite)
+        }
+
+        runner.emit('end')
+
+        if (typeof reporterInstance.done === 'function') {
+          await new Promise((resolve) => {
+            reporterInstance.done(runner.failures || 0, resolve)
+          })
+        }
+      } catch (err) {
+        output.error(`Reporter error: ${err.message}`)
+      }
+    }
   }
 
   printResults() {

test/data/sandbox/codecept.workers.mochawesome.conf.js

@@ -0,0 +1,24 @@
+export const config = {
+  tests: './workers/*.js',
+  timeout: 10000,
+  output: './output/workers_mochawesome',
+  helpers: {
+    FileSystem: {},
+    Workers: {
+      require: './workers_helper',
+    },
+  },
+  include: {},
+  mocha: {
+    reporter: 'mochawesome',
+    reporterOptions: {
+      reportDir: './output/workers_mochawesome',
+      reportFilename: 'report',
+      quiet: true,
+      overwrite: true,
+      json: true,
+      html: true,
+    },
+  },
+  name: 'sandbox',
+};

test/data/sandbox/codecept.workers.mochawesome.multiple.conf.js

@@ -0,0 +1,29 @@
+export const config = {
+  tests: './workers/*.js',
+  timeout: 10000,
+  output: './output/workers_mochawesome',
+  helpers: {
+    FileSystem: {},
+    Workers: {
+      require: './workers_helper',
+    },
+  },
+  include: {},
+  mocha: {
+    reporter: 'mochawesome',
+    reporterOptions: {
+      reportDir: './output/workers_mochawesome',
+      reportFilename: 'report',
+      quiet: true,
+      overwrite: true,
+      json: true,
+      html: false,
+    },
+  },
+  multiple: {
+    parallel: {
+      browsers: ['chrome', 'firefox'],
+    },
+  },
+  name: 'sandbox',
+};