refactor(test): adjust sandbox folder location and simplify config lo…

…gic (#3493)

This is a bit bigger change, but modifications are closely related and I think it makes sense to land them all together.

The main change is in test/e2e/support/world.js file, where all important paths are defined:

- `workDir` is the current working directory for the tested Karma process - `test/e2e/support`. Same as before.
- `sandboxDir` moved from `tmp/sandbox` in the repository root to `test/e2e/support/sandbox`. Sandbox directory is reset before every scenario (see `test/e2e/step_definitions/hooks.js`) and this is where all generated files should be (hence change of paths in `browser_console.feature`). This also makes things cleaner as now all paths in the .feature files are relative to `workDir`.
- `configFile` is path where Karma config file is generated. It no longer contains hash as it was not really needed. It may be re-introduced later if tests are run in parallel, but then it makes more sense to have it as a part of `sandboxDir` instead.
- `karmaExecutable` is an absolute path to the Karma executable.

The change from `__dirname + '/` to `_resolve('` in two .feature files is to avoid having two different hacks to resolve absolute paths.

The last change in this file is to simplify config generation methods.

Remaining changes in `test/e2e/step_definitions/core_steps.js` are adapting steps to the above changes by removing bunch of boilerplate, which is no longer necessary.

.eslintignore

@@ -1,5 +1,5 @@
+test/e2e/support/sandbox
 test/e2e/support/error/under-test.js
 test/unit/fixtures/bundled.js
 static/karma.js
 static/context.js
-tmp/*

.gitignore

@@ -4,11 +4,10 @@ static/context.js
 static/karma.js
 .idea/*
 *.iml
-tmp/*
 docs/_build
 *.swp
 *.swo
-test/e2e/console.log
+test/e2e/support/sandbox
 test/e2e/coverage/coverage
 test/e2e/coverageQunit/coverage
 test/e2e/coverageRequirejs/coverage

test/e2e/browser_console.feature

@@ -50,12 +50,12 @@ Feature: Browser Console Configuration
         'karma-chrome-launcher'
       ];
       browserConsoleLogOptions = {
-        path: 'test/e2e/console.log',
+        path: 'sandbox/console.log',
         format: '%t:%m'
       };
       """
     When I start Karma
-    Then the file at test/e2e/console.log contains:
+    Then the file at sandbox/console.log contains:
       """
       log:'foo'
       debug:'bar'
@@ -74,12 +74,12 @@ Feature: Browser Console Configuration
         'karma-chrome-launcher'
       ];
       browserConsoleLogOptions = {
-        path: 'test/e2e/console.log',
+        path: 'sandbox/console.log',
         format: '%t:%T:%m'
       };
       """
     When I start Karma
-    Then the file at test/e2e/console.log contains:
+    Then the file at sandbox/console.log contains:
       """
       log:LOG:'foo'
       debug:DEBUG:'bar'
@@ -98,13 +98,13 @@ Feature: Browser Console Configuration
         'karma-chrome-launcher'
       ];
       browserConsoleLogOptions = {
-        path: 'test/e2e/console.log',
+        path: 'sandbox/console.log',
         format: '%t:%T:%m',
         level: 'warn'
       };
       """
     When I start Karma
-    Then the file at test/e2e/console.log contains:
+    Then the file at sandbox/console.log contains:
       """
       log:LOG:'foo'
       warn:WARN:'foobar'
@@ -121,12 +121,12 @@ Feature: Browser Console Configuration
         'karma-chrome-launcher'
       ];
       browserConsoleLogOptions = {
-        path: 'test/e2e/console.log',
+        path: 'sandbox/console.log',
         format: '%b'
       };
       """
     When I start Karma
-    Then the file at test/e2e/console.log contains:
+    Then the file at sandbox/console.log contains:
       """
       Chrome Headless
       """
@@ -141,7 +141,7 @@ Feature: Browser Console Configuration
         'karma-chrome-launcher'
       ];
       browserConsoleLogOptions = {
-        path: 'test/e2e/console.log',
+        path: 'sandbox/console.log',
         format: '%b',
         terminal: false
       };

test/e2e/launcher-error.feature

@@ -7,7 +7,7 @@ Feature: Launcher error
     Given a configuration with:
       """
       files = ['launcher-error/specs.js'];
-      browsers = [__dirname + '/launcher-error/fake-browser.sh'];
+      browsers = [_resolve('launcher-error/fake-browser.sh')];
       plugins = [
         'karma-jasmine',
         'karma-script-launcher'

test/e2e/step_definitions/core_steps.js

@@ -2,105 +2,81 @@ const { defineParameterType, Given, Then, When } = require('cucumber')
 const fs = require('fs')
 const path = require('path')
 const { exec, spawn } = require('child_process')
-const rimraf = require('rimraf')
 const stopper = require('../../../lib/stopper')
 
-const baseDir = fs.realpathSync(path.join(__dirname, '/../../..'))
-const tmpDir = path.join(baseDir, 'tmp', 'sandbox')
-const tmpConfigFile = 'karma.conf.js'
-let cleansingNeeded = true
 let additionalArgs = []
 
-function cleanseIfNeeded () {
-  if (cleansingNeeded) {
-    try {
-      rimraf.sync(tmpDir)
-    } catch (e) {
-    }
+function execKarma (command, level, callback) {
+  level = level || 'warn'
 
-    cleansingNeeded = false
+  this.writeConfigFile()
 
-    return cleansingNeeded
-  }
-}
+  const configFile = this.configFile
+  const runtimePath = this.karmaExecutable
+  const baseDir = this.workDir
 
-function execKarma (command, level, callback) {
-  level = level || 'warn'
+  const executor = (done) => {
+    const cmd = runtimePath + ' ' + command + ' --log-level ' + level + ' ' + configFile + ' ' + additionalArgs
 
-  this.writeConfigFile(tmpDir, tmpConfigFile, (err, hash) => {
-    if (err) {
-      return callback.fail(new Error(err))
+    return exec(cmd, {
+      cwd: baseDir
+    }, done)
+  }
+
+  const runOut = command === 'runOut'
+  if (command === 'run' || command === 'runOut') {
+    let isRun = false
+    this.child = spawn('' + runtimePath, ['start', '--log-level', 'warn', configFile])
+    const done = () => {
+      this.child && this.child.kill()
+      callback()
     }
-    const configFile = path.join(tmpDir, hash + '.' + tmpConfigFile)
-    const runtimePath = path.join(baseDir, 'bin', 'karma')
 
-    const executor = (done) => {
-      const cmd = runtimePath + ' ' + command + ' --log-level ' + level + ' ' + configFile + ' ' + additionalArgs
+    this.child.on('error', (error) => {
+      this.lastRun.error = error
+      done()
+    })
 
-      return exec(cmd, {
-        cwd: baseDir
-      }, done)
-    }
+    this.child.stderr.on('data', (chunk) => {
+      this.lastRun.stderr += chunk.toString()
+    })
 
-    const runOut = command === 'runOut'
-    if (command === 'run' || command === 'runOut') {
-      let isRun = false
-      this.child = spawn('' + runtimePath, ['start', '--log-level', 'warn', configFile])
-      const done = () => {
-        cleansingNeeded = true
-        this.child && this.child.kill()
-        callback()
+    this.child.stdout.on('data', (chunk) => {
+      this.lastRun.stdout += chunk.toString()
+      const cmd = runtimePath + ' run ' + configFile + ' ' + additionalArgs
+      if (!isRun) {
+        isRun = true
+
+        setTimeout(() => {
+          exec(cmd, {
+            cwd: baseDir
+          }, (error, stdout, stderr) => {
+            if (error) {
+              this.lastRun.error = error
+            }
+            if (runOut) {
+              this.lastRun.stdout = stdout
+              this.lastRun.stderr = stderr
+            }
+            done()
+          })
+        }, 1000)
       }
-
-      this.child.on('error', (error) => {
+    })
+  } else {
+    executor((error, stdout, stderr) => {
+      if (error) {
         this.lastRun.error = error
-        done()
-      })
-
-      this.child.stderr.on('data', (chunk) => {
-        this.lastRun.stderr += chunk.toString()
-      })
-
-      this.child.stdout.on('data', (chunk) => {
-        this.lastRun.stdout += chunk.toString()
-        const cmd = runtimePath + ' run ' + configFile + ' ' + additionalArgs
-        if (!isRun) {
-          isRun = true
-
-          setTimeout(() => {
-            exec(cmd, {
-              cwd: baseDir
-            }, (error, stdout, stderr) => {
-              if (error) {
-                this.lastRun.error = error
-              }
-              if (runOut) {
-                this.lastRun.stdout = stdout
-                this.lastRun.stderr = stderr
-              }
-              done()
-            })
-          }, 1000)
-        }
-      })
-    } else {
-      executor((error, stdout, stderr) => {
-        if (error) {
-          this.lastRun.error = error
-        }
-        this.lastRun.stdout = stdout
-        this.lastRun.stderr = stderr
-        cleansingNeeded = true
-        callback()
-      })
-    }
-  })
+      }
+      this.lastRun.stdout = stdout
+      this.lastRun.stderr = stderr
+      callback()
+    })
+  }
 }
 
-Given('a configuration with:', function (fileContent, callback) {
-  cleanseIfNeeded()
-  this.addConfigContent(fileContent)
-  return callback()
+Given('a configuration with:', function (fileContent) {
+  this.updateConfig(fileContent)
 })
 
 Given('command line arguments of: {string}', function (args, callback) {
@@ -113,34 +89,27 @@ Given('a proxy on port {int} that prepends {string} to the base path', async fun
 })
 
 When('I stop a server programmatically', function (callback) {
-  const _this = this
-  setTimeout(function () {
-    stopper.stop(_this.configFile, function (exitCode) {
-      _this.stopperExitCode = exitCode
+  setTimeout(() => {
+    stopper.stop(this.config, (exitCode) => {
+      this.stopperExitCode = exitCode
       callback()
     })
   }, 1000)
 })
 
 When('I start a server in background', function (callback) {
-  this.writeConfigFile(tmpDir, tmpConfigFile, (function (_this) {
-    return function (err, hash) {
-      if (err) {
-        return callback.fail(new Error(err))
-      }
-      const configFile = path.join(tmpDir, hash + '.' + tmpConfigFile)
-      const runtimePath = path.join(baseDir, 'bin', 'karma')
-      _this.child = spawn('' + runtimePath, ['start', '--log-level', 'debug', configFile])
-      _this.child.stdout.on('data', function () {
-        callback()
-        callback = function () {
-        }
-      })
-      _this.child.on('exit', function (exitCode) {
-        _this.childExitCode = exitCode
-      })
-    }
-  })(this))
+  this.writeConfigFile()
+
+  const configFile = this.configFile
+  const runtimePath = this.karmaExecutable
+  this.child = spawn(runtimePath, ['start', '--log-level', 'debug', configFile])
+  this.child.stdout.on('data', () => {
+    callback()
+    callback = () => null
+  })
+  this.child.on('exit', (exitCode) => {
+    this.childExitCode = exitCode
+  })
 })
 
 defineParameterType({
@@ -230,11 +199,9 @@ Then(/^The (server|stopper) is dead(:? with exit code (\d+))?$/,
     }, 4000)
   })
 
-Then(/^the file at ([a-zA-Z0-9/\\_.]+) contains:$/,
-  function (filePath, expectedOutput, callback) {
-    const data = fs.readFileSync(filePath, { encoding: 'UTF-8' })
-    if (data.match(expectedOutput)) {
-      return callback()
-    }
-    callback(new Error('Expected output to match the following:\n  ' + expectedOutput + '\nGot:\n  ' + data))
-  })
+Then(/^the file at ([a-zA-Z0-9/\\_.]+) contains:$/, function (filePath, expectedOutput) {
+  const data = fs.readFileSync(path.join(this.workDir, filePath), 'utf8')
+  if (!data.match(expectedOutput)) {
+    throw new Error('Expected output to match the following:\n  ' + expectedOutput + '\nGot:\n  ' + data)
+  }
+})

test/e2e/step_definitions/hooks.js

@@ -1,4 +1,8 @@
-const { After } = require('cucumber')
+const { After, Before } = require('cucumber')
+
+Before(function () {
+  this.ensureSandbox()
+})
 
 After(async function () {
   await this.proxy.stopIfRunning()