diff --git a/lib/codecept.js b/lib/codecept.js index 05203d241..5f6a48b08 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -284,6 +284,9 @@ class Codecept { // Ignore if gherkin module not available } + // Sort test files alphabetically for consistent execution order + this.testFiles.sort() + return new Promise((resolve, reject) => { const mocha = container.mocha() mocha.files = this.testFiles diff --git a/test/unit/alphabetical_order_test.js b/test/unit/alphabetical_order_test.js new file mode 100644 index 000000000..0f1f55680 --- /dev/null +++ b/test/unit/alphabetical_order_test.js @@ -0,0 +1,95 @@ +import { expect } from 'chai' +import Codecept from '../../lib/codecept.js' +import Container from '../../lib/container.js' +import path from 'path' +import fs from 'fs' +import { fileURLToPath } from 'url' +import { dirname } from 'path' + +const __filename = fileURLToPath(import.meta.url) +const __dirname = dirname(__filename) + +describe('Test Files Alphabetical Order', () => { + let codecept + const tempDir = path.join(__dirname, '../data/sandbox/configs/alphabetical_order') + const config = { + tests: `${tempDir}/*_test.js`, + gherkin: { features: null }, + output: './output', + hooks: [], + } + + before(() => { + if (!fs.existsSync(tempDir)) { + fs.mkdirSync(tempDir, { recursive: true }) + } + + fs.writeFileSync( + path.join(tempDir, 'zzz_test.js'), + ` + Feature('Test'); + Scenario('zzz test', () => {}); + `, + ) + fs.writeFileSync( + path.join(tempDir, 'aaa_test.js'), + ` + Feature('Test'); + Scenario('aaa test', () => {}); + `, + ) + fs.writeFileSync( + path.join(tempDir, 'mmm_test.js'), + ` + Feature('Test'); + Scenario('mmm test', () => {}); + `, + ) + }) + + after(() => { + const files = ['zzz_test.js', 'aaa_test.js', 'mmm_test.js'] + files.forEach(file => { + const filePath = path.join(tempDir, file) + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath) + } + }) + }) + + beforeEach(async () => { + codecept = new Codecept(config, {}) + await codecept.init(path.join(__dirname, '../data/sandbox')) + }) + + afterEach(() => { + Container.clear() + }) + + it('should not sort test files in loadTests (sorting happens in run)', () => { + codecept.loadTests() + + if (codecept.testFiles.length === 0) { + console.log('No test files found, skipping test') + return + } + + // After loadTests(), files should be in glob order (NOT sorted). + // Sorting should only happen later in run(). + const filenames = codecept.testFiles.map(filePath => path.basename(filePath)) + + // Verify all 3 test files were loaded + expect(filenames).to.include('aaa_test.js') + expect(filenames).to.include('mmm_test.js') + expect(filenames).to.include('zzz_test.js') + expect(filenames.length).to.equal(3) + + // Verify that sorting the files produces alphabetical order + const sortedFiles = [...codecept.testFiles].sort() + const sortedFilenames = sortedFiles.map(filePath => path.basename(filePath)) + + expect(sortedFilenames[0]).to.include('aaa_test.js') + expect(sortedFilenames[1]).to.include('mmm_test.js') + expect(sortedFilenames[2]).to.include('zzz_test.js') + }) +}) diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index 635f00b72..6c0a51978 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -382,4 +382,35 @@ describe('Workers', function () { workers.run() }) + + it('should preserve original file order in loadTests for worker distribution (issue #5412)', async () => { + // This test verifies the fix for issue #5412: + // Test files should NOT be sorted in loadTests() because that affects worker distribution. + // Sorting should only happen in run() for execution order. + // + // The bug was: sorting in loadTests() changed the order of suites during distribution, + // causing all workers to receive the same tests instead of different suites. + + const workerConfig = { + by: 'suite', + testConfig: './test/data/sandbox/codecept.customworker.js', + } + + const workers = new Workers(3, workerConfig) + await workers._ensureInitialized() + + // Verify that test files were loaded + const testFiles = workers.codecept.testFiles + expect(testFiles.length).to.be.greaterThan(1, 'Multiple test files should be loaded') + + // loadTests() must preserve the original glob order (not sort files). + // Verify by comparing with a fresh glob call — the order should match. + const { globSync } = await import('glob') + const expectedFiles = globSync('./custom-worker/*.js', { cwd: path.join(__dirname, '/../data/sandbox') }) + .filter(f => !f.includes('node_modules')) + .map(f => path.resolve(path.join(__dirname, '/../data/sandbox'), f)) + + const actualFiles = testFiles.map(f => path.resolve(f)) + expect(actualFiles).to.deep.equal(expectedFiles, 'loadTests() should preserve original glob order without sorting') + }) })