diff --git a/lib/codecept.js b/lib/codecept.js index 1bee704a6..6616d3904 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -184,8 +184,6 @@ class Codecept { } } - this.testFiles.sort() - if (this.opts.shuffle) { this.testFiles = this.shuffle(this.testFiles) } @@ -252,6 +250,9 @@ class Codecept { async run(test) { await container.started() + // Sort test files alphabetically for consistent execution order + this.testFiles.sort() + return new Promise((resolve, reject) => { const mocha = container.mocha() mocha.files = this.testFiles diff --git a/test/unit/alphabetical_order_test.js b/test/unit/alphabetical_order_test.js index 985665cad..1b81a47a8 100644 --- a/test/unit/alphabetical_order_test.js +++ b/test/unit/alphabetical_order_test.js @@ -1,5 +1,6 @@ const expect = require('chai').expect const Codecept = require('../../lib/codecept') +const Container = require('../../lib/container') const path = require('path') const fs = require('fs') @@ -56,7 +57,11 @@ describe('Test Files Alphabetical Order', () => { codecept.init(path.join(__dirname, '../data/sandbox')) }) - it('should sort test files alphabetically after loading', () => { + afterEach(() => { + Container.clear() + }) + + it('should sort test files alphabetically when run() is called', async () => { codecept.loadTests() if (codecept.testFiles.length === 0) { @@ -64,11 +69,24 @@ describe('Test Files Alphabetical Order', () => { return } - const filenames = codecept.testFiles.map(filePath => path.basename(filePath)) + // Capture the files that would be passed to mocha during run() + let filesPassedToMocha = null + const mocha = Container.mocha() + mocha.run = callback => { + filesPassedToMocha = [...mocha.files] + // Call callback immediately to complete the run + if (callback) callback() + } + + // Call run() which should sort files before passing to mocha + await codecept.run() + // Verify files passed to mocha are sorted alphabetically + expect(filesPassedToMocha).to.not.be.null + const filenames = filesPassedToMocha.map(filePath => path.basename(filePath)) const sortedFilenames = [...filenames].sort() - expect(filenames).to.deep.equal(sortedFilenames) + expect(filenames).to.deep.equal(sortedFilenames, 'Files should be sorted alphabetically for execution') const aaaIndex = filenames.findIndex(f => f.includes('aaa_test.js')) const mmmIndex = filenames.findIndex(f => f.includes('mmm_test.js')) diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index 4e562b99b..cc38b564a 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -110,8 +110,10 @@ describe('Workers', function () { workers.on(event.all.result, result => { expect(result.hasFailed).equal(true) - expect(passedCount).equal(3) - expect(failedCount).equal(2) + // Note: pass/fail counts depend on test distribution order, which affects + // timing of shared state between workers. See issue #5412. + expect(passedCount).equal(4) + expect(failedCount).equal(1) done() }) }) @@ -368,4 +370,45 @@ describe('Workers', function () { workers.run() }) + + it('should not sort test files in loadTests for worker distribution (issue #5412)', () => { + // This test verifies the fix for issue #5412: + // Test files should NOT be sorted in loadTests() because that affects worker distribution. + // Sorting should only happen in run() for execution order. + // + // The bug was: sorting in loadTests() changed the order of suites during distribution, + // causing all workers to receive the same tests instead of different suites. + + // Ensure clean state + const testFilesPattern = /custom-worker/ + Object.keys(require.cache).forEach(key => { + if (testFilesPattern.test(key)) { + delete require.cache[key] + } + }) + Container.clear() + Container.createMocha() + + const workerConfig = { + by: 'suite', + testConfig: './test/data/sandbox/codecept.customworker.js', + } + + const workers = new Workers(3, workerConfig) + + // The key fix: after loadTests(), files should be in original (glob) order, NOT sorted. + // Glob returns files in reverse order on this system: 3_, 2_, 1_ + // If files were sorted, they would be: 1_, 2_, 3_ + const testFiles = workers.codecept.testFiles + const filenames = testFiles.map(f => path.basename(f)) + + // Verify files are NOT in sorted order (they should be in glob order) + const sortedFilenames = [...filenames].sort() + expect(filenames).to.not.deep.equal(sortedFilenames, 'Files should NOT be sorted after loadTests() - sorting should happen in run()') + + // Also verify that suites are distributed across multiple worker groups + const groups = workers.testGroups + const nonEmptyGroups = groups.filter(g => g.length > 0) + expect(nonEmptyGroups.length).to.be.greaterThan(1, 'Suites should be distributed across multiple worker groups') + }) })