(wdio-config): expand and simplify the --multi-run feature (#12372)

* (wdio-config): expand and simplify the --multi-run feature * (wdio-config): revert some changes and updated the --multi-run * (wdio-config): revert some changes and updated the --multi-run * (wdio-config): revert some changes and updated the --multi-run * (wdio-config): revert some changes and updated the --multi-run * (wdio-config): revert some changes and updated the --multi-run * (wdio-config): revert some changes and updated the --multi-run * (wdio-config): resolve PR feedback
webdriverio · Feb 29, 2024 · 6512e2d · 6512e2d
1 parent 776ced4
commit 6512e2d
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 7 deletions.
diff --git a/packages/wdio-cli/src/commands/run.ts b/packages/wdio-cli/src/commands/run.ts
@@ -105,7 +105,7 @@ export const cmdArgs = {
         type: 'array'
     },
     'multi-run': {
-        desc: 'Run individual spec files x amount of times',
+        desc: 'Repeat specific specs and/or suites N times',
         type: 'number'
     },
     mochaOpts: {

diff --git a/packages/wdio-config/src/node/ConfigParser.ts b/packages/wdio-config/src/node/ConfigParser.ts
@@ -271,7 +271,7 @@ export default class ConfigParser {
      * attributes from CLI, config and capabilities
      */
     getSpecs(capSpecs?: Spec[], capExclude?: Spec[]) {
-        const isSpecParamPassed = Array.isArray(this._config.spec) && this._config.spec.length >= 1
+        const isSpecParamPassed = Array.isArray(this._config.spec) && this._config.spec.length > 0
         const multiRun = this._config.multiRun
         // when CLI --spec is explicitly specified, this._config.specs contains the filtered
         // specs matching the passed pattern else the specs defined inside the config are returned
@@ -316,11 +316,11 @@ export default class ConfigParser {
         // Remove any duplicate tests from the final specs array
         specs = [...new Set(specs)]
 
-        // If the --multi-run flag is set, duplicate the specs array
-        // Ensure that when --multi-run is set that either --spec or --suite is also set
+        // If the --multi-run flag is set, duplicate the specs array N times
+        // Ensure that when --multi-run is used that either --spec or --suite is also used
         const hasSubsetOfSpecsDefined = isSpecParamPassed || suites.length > 0
         if (multiRun && hasSubsetOfSpecsDefined) {
-            specs = specs.flatMap(i => Array.from({ length: multiRun }).fill(i)) as Spec[]
+            specs = Array.from({ length: multiRun }, () => specs).flat()
         } else if (multiRun && !hasSubsetOfSpecsDefined) {
             throw new Error('The --multi-run flag requires that either the --spec or --suite flag is also set')
         }

diff --git a/packages/wdio-config/tests/node/configparser.test.ts b/packages/wdio-config/tests/node/configparser.test.ts
@@ -853,14 +853,28 @@ describe('ConfigParser', () => {
             expect(specs).toHaveLength(1)
         })
 
-        it('should include specs from suite 3 times with mulit-run', async () => {
+        it('should include specs from suite 3 times with multi-run', async () => {
             const configParser = await ConfigParserForTestWithAllFiles(FIXTURES_CONF)
             await configParser.initialize({ suite: ['functional'], multiRun: 3 })
 
             const specs = configParser.getSpecs()
             expect(specs).toHaveLength(3)
         })
 
+        it('should repeat specs in specific order to fail early', async () => {
+            const spec1 = path.resolve(__dirname, '../utils.test.ts')
+            const spec2 = path.resolve(__dirname, 'configparser.test.ts')
+            const configParser = await ConfigParserForTestWithAllFiles(FIXTURES_CONF)
+            await configParser.initialize({ spec: [spec1, spec2], multiRun: 3 })
+
+            const specs = configParser.getSpecs()
+            expect(specs).toEqual([
+                spec1, spec2,
+                spec1, spec2,
+                spec1, spec2,
+            ])
+        })
+
         it('should throw an error if multi-run is set but no spec or suite is specified', async () => {
             const configParser = await ConfigParserForTestWithAllFiles(FIXTURES_CONF)
             await configParser.initialize({ multiRun: 3 })

diff --git a/website/docs/Retry.md b/website/docs/Retry.md
@@ -158,7 +158,7 @@ export const config = {
 
 ## Run a specific test multiple times
 
-This is to help prevent flaky tests from being introduced in a codebase. By adding the `--multi-run` cli option it will run the specified test(s) or suite(s) x number of times. When using this cli flag the `--spec` or `--suite` flag must also be specified.
+This is to help prevent flaky tests from being introduced in a codebase. By adding the `--multi-run` cli option it will run the specified specs or suites N times. When using this cli flag the `--spec` or `--suite` flag must also be specified.
 
 When adding new tests to a codebase, especially through a CI/CD process the tests could pass and get merged but become flaky later on. This flakiness could come from a number of things like network issues, server load, database size, etc. Using the `--multi-run` flag in your CD/CD process can help catch these flaky tests before they get merged to a main codebase.