chore(test): error tests fixed (#5069)

- Fix a missing await on createNextTaskRunner where the recursive call should be awaited and if there are no new tasks to still resolve the promise. Things were passing previously probably because we were running tasks out of sync. - Add in errorTest portion of the test suite. - Turn on the angular2 and unit tests. - Turn on browserstack test and remove ciNg2Conf test from Travis.
angular · Mar 23, 2019 · 0f7a38a · 0f7a38a
1 parent 055be2e
commit 0f7a38a
Show file tree

Hide file tree

Showing 25 changed files with 124 additions and 120 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -16,17 +16,17 @@ env:
   matrix:
     - JOB=full
     - JOB=smoke
-    # - JOB=bstack
+    - JOB=bstack
 
 matrix:
   allow_failures:
     - env: "JOB=smoke"
-#     - env: "JOB=bstack"
+    - env: "JOB=bstack"
   exclude:
     - env: JOB=smoke
       node_js: "9"
-#     - env: JOB=bstack
-#       node_js: "8"
+    - env: JOB=bstack
+      node_js: "9"
 
 addons:
   apt:

diff --git a/lib/launcher.ts b/lib/launcher.ts
@@ -221,7 +221,7 @@ let initFn = async function(configFile: string, additionalConfig: Config) {
           }
           taskResults_.add(result);
           task.done();
-          createNextTaskRunner();
+          await createNextTaskRunner();
           // If all tasks are finished
           if (scheduler.numTasksOutstanding() === 0) {
             resolve();
@@ -232,6 +232,8 @@ let initFn = async function(configFile: string, additionalConfig: Config) {
           logger.error('Error:', (err as any).stack || err.message || err);
           await cleanUpAndExit(errorCode ? errorCode : RUNNERS_FAILED_EXIT_CODE);
         }
+      } else {
+        resolve();
       }
     });
   };

diff --git a/scripts/test.js b/scripts/test.js
@@ -38,15 +38,15 @@ const passingTests = [
   'node built/cli.js spec/controlLockConf.js',
   'node built/cli.js spec/customFramework.js',
   'node built/cli.js spec/noGlobalsConf.js',
-  // 'node built/cli.js spec/angular2Conf.js',
+  'node built/cli.js spec/angular2Conf.js',
   'node built/cli.js spec/hybridConf.js',
   'node built/cli.js spec/built/noCFBasicConf.js',
   'node built/cli.js spec/built/noCFBasicConf.js --useBlockingProxy',
   'node built/cli.js spec/built/noCFPluginConf.js',
   'node scripts/driverProviderAttachSession.js',
-  // 'node scripts/errorTest.js',
-  // // Unit tests
-  // 'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/unit_test.json',
+  'node scripts/errorTest.js',
+  // Unit tests
+  'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/unit_test.json',
   // Dependency tests
   'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/dependency_test.json',
   // Typings tests
@@ -68,7 +68,7 @@ passingTests.forEach((passing_test) => {
 executor.addCommandlineTest('node built/cli.js spec/errorTest/singleFailureConf.js')
     .expectExitCode(1)
     .expectErrors({
-      stackTrace: 'single_failure_spec1.js:5:32'
+      stackTrace: 'single_failure_spec1.js:5:38'
     });
 
 // assert timeout works
@@ -90,63 +90,63 @@ executor.addCommandlineTest('node built/cli.js spec/errorTest/multiFailureConf.j
     .expectExitCode(1)
     .expectErrors([{
       message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
-      stacktrace: 'single_failure_spec1.js:5:32'
+      stacktrace: 'single_failure_spec1.js:5:38'
     }, {
       message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
-      stacktrace: 'single_failure_spec2.js:5:32'
+      stacktrace: 'single_failure_spec2.js:5:38'
     }]);
 
 executor.addCommandlineTest('node built/cli.js spec/errorTest/shardedFailureConf.js')
     .expectExitCode(1)
     .expectErrors([{
       message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
-      stacktrace: 'single_failure_spec1.js:5:32'
+      stacktrace: 'single_failure_spec1.js:5:38'
     }, {
       message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
-      stacktrace: 'single_failure_spec2.js:5:32'
+      stacktrace: 'single_failure_spec2.js:5:38'
     }]);
 
 executor.addCommandlineTest('node built/cli.js spec/errorTest/mochaFailureConf.js')
     .expectExitCode(1)
     .expectErrors([{
       message: 'expected \'My AngularJS App\' to equal \'INTENTIONALLY INCORRECT\'',
-      stacktrace: 'mocha_failure_spec.js:11:20'
+      stacktrace: 'mocha_failure_spec.js:11:41'
     }]);
 
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/pluginsFailingConf.js')
-//     .expectExitCode(1)
-//     .expectErrors([
-//       {message: 'Expected true to be false'},
-//       {message: 'from setup'},
-//       {message: 'from postTest passing'},
-//       {message: 'from postTest failing'},
-//       {message: 'from teardown'}
-//     ]);
-
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js')
-//     .expectExitCode(1)
-//     .expectErrors([
-//       {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
-//       {message: 'The following tasks were pending:[\\s\\S]*' +
-//                 '- \\$timeout: function\\(\\) {[\\s\\S]*' +
-//                   '\\$scope\\.slowAngularTimeoutStatus = \'done\';[\\s\\S]' +
-//                 '*}'}
-//     ]);
-
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js ' +
-//                             '--untrackOutstandingTimeouts true')
-//     .expectExitCode(1)
-//     .expectErrors([
-//       {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
-//       {message: 'While waiting for element with locator - ' +
-//                 'Locator: by.binding\\(\\"slowAngularTimeoutStatus\\"\\)$'}
-//     ]);
-
-// executor.addCommandlineTest('node built/cli.js spec/angular2TimeoutConf.js')
-//     .expectExitCode(1)
-//     .expectErrors([
-//       {message: 'Timed out waiting for asynchronous Angular tasks to finish'},
-//     ]);
+executor.addCommandlineTest('node built/cli.js spec/errorTest/pluginsFailingConf.js')
+    .expectExitCode(1)
+    .expectErrors([
+      {message: 'Expected true to be false'},
+      {message: 'from setup'},
+      {message: 'from postTest passing'},
+      {message: 'from postTest failing'},
+      {message: 'from teardown'}
+    ]);
+
+executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js')
+    .expectExitCode(1)
+    .expectErrors([
+      {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
+      {message: 'The following tasks were pending:[\\s\\S]*' +
+                '- \\$timeout: function\\(\\) {[\\s\\S]*' +
+                  '\\$scope\\.slowAngularTimeoutStatus = \'done\';[\\s\\S]' +
+                '*}'}
+    ]);
+
+executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js ' +
+                            '--untrackOutstandingTimeouts true')
+    .expectExitCode(1)
+    .expectErrors([
+      {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
+      {message: 'While waiting for element with locator - ' +
+                'Locator: by.binding\\(\\"slowAngularTimeoutStatus\\"\\)$'}
+    ]);
+
+executor.addCommandlineTest('node built/cli.js spec/angular2TimeoutConf.js')
+    .expectExitCode(1)
+    .expectErrors([
+      {message: 'Timed out waiting for asynchronous Angular tasks to finish'},
+    ]);
 
 // If we're running on CircleCI, save stdout and stderr from the test run to a log file.
 if (process.env['CIRCLE_ARTIFACTS']) {

diff --git a/scripts/test_on_travis.sh b/scripts/test_on_travis.sh
@@ -5,11 +5,11 @@ if [ $JOB = "smoke" ]; then
   node bin/protractor spec/ciSmokeConf.js
 elif [ $JOB = "full" ]; then
   node bin/protractor spec/ciFullConf.js
-  if [ $? = "0" ]; then
-    node bin/protractor spec/ciNg2Conf.js
-  else
-  	exit 1
-  fi
+  # if [ $? = "0" ]; then
+  #   node bin/protractor spec/ciNg2Conf.js
+  # else
+  # 	exit 1
+  # fi
 elif [ $JOB = "bstack" ]; then
   node bin/protractor spec/ciBStackConf.js
 else

diff --git a/spec/ciFullConf.js b/spec/ciFullConf.js
@@ -9,18 +9,8 @@ exports.config = {
   framework: 'jasmine',
 
   // Spec patterns are relative to this directory.
-  // TODO(selenium4): revert back to basic/*_spec.js
   specs: [
-    'basic/lib_spec.js',
-    'basic/locators_spec.js'
-    // 'basic/elements_spec.js',
-    // 'basic/expected_conditions_spec.js',
-    // 'basic/handling_spec.js'
-    // 'basic/mockmodule_spec.js',
-    // 'basic/navigation_spec.js',
-    // 'basic/polling_spec.js',
-    // 'basic/restart_spec.js',
-    // 'basic/synchronize_spec.js',
+    'basic/*_spec.js',
   ],
 
   // Exclude patterns are relative to this directory.

diff --git a/spec/errorTest/afterLaunchChangesExitCodeConf.js b/spec/errorTest/afterLaunchChangesExitCodeConf.js
@@ -2,6 +2,7 @@ var env = require('../environment.js');
 
 exports.config = {
   seleniumAddress: env.seleniumAddress,
+  SELENIUM_PROMISE_MANAGER: false,
 
   framework: 'jasmine',
 

diff --git a/spec/errorTest/baseCase/error_spec.js b/spec/errorTest/baseCase/error_spec.js
@@ -1,6 +1,6 @@
-describe('finding an element that does not exist', function() {
-  it('should throw an error', function() {
-    browser.get('index.html');
+describe('finding an element that does not exist', () => {
+  it('should throw an error', async () => {
+    await browser.get('index.html');
     element(by.binding('INVALID'));  // greeting
   });
 });
diff --git a/spec/errorTest/baseCase/mocha_failure_spec.js b/spec/errorTest/baseCase/mocha_failure_spec.js
@@ -1,13 +1,13 @@
 // Use the external Chai As Promised to deal with resolving promises in
 // expectations.
-var chai = require('chai');
-var chaiAsPromised = require('chai-as-promised');
+const chai = require('chai');
+const chaiAsPromised = require('chai-as-promised');
 chai.use(chaiAsPromised);
-var expect = chai.expect;
+const expect = chai.expect;
 
-describe('protractor library', function() {
-  it('should fail', function() {
-    browser.get('index.html');
-    expect(browser.getTitle()).to.eventually.equal('INTENTIONALLY INCORRECT');
+describe('protractor library', () => {
+  it('should fail', async () => {
+    await browser.get('index.html');
+    expect(await browser.getTitle()).to.equal('INTENTIONALLY INCORRECT');
   });
 });
diff --git a/spec/errorTest/baseCase/single_failure_spec1.js b/spec/errorTest/baseCase/single_failure_spec1.js
@@ -1,7 +1,7 @@
-describe('single failure spec1', function() {
-  it('should fail expectation', function() {
-    browser.get('index.html');
-    var greeting = element(by.binding('greeting'));
-    expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
+describe('single failure spec1', () => {
+  it('should fail expectation', async () => {
+    await browser.get('index.html');
+    const greeting = element(by.binding('greeting'));
+    expect(await greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
   });
 });
diff --git a/spec/errorTest/baseCase/single_failure_spec2.js b/spec/errorTest/baseCase/single_failure_spec2.js
@@ -1,7 +1,7 @@
-describe('single failure spec2', function() {
-  it('should fail expectation', function() {
-    browser.get('index.html');
-    var greeting = element(by.binding('greeting'));
-    expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
+describe('single failure spec2', () => {
+  it('should fail expectation', async () => {
+    await browser.get('index.html');
+    const greeting = element(by.binding('greeting'));
+    expect(await greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
   });
 });
diff --git a/spec/errorTest/baseCase/slow_http_and_timeout_spec.js b/spec/errorTest/baseCase/slow_http_and_timeout_spec.js
@@ -1,27 +1,27 @@
-describe('slow asynchronous events', function() {
-  beforeEach(function() {
-    browser.get('index.html#/async');
+describe('slow asynchronous events', () => {
+  beforeEach(async () => {
+    await browser.get('index.html#/async');
   });
 
-  it('waits for http calls', function() {
-    var status = element(by.binding('slowHttpStatus'));
-    var button = element(by.css('[ng-click="slowHttp()"]'));
+  it('waits for http calls', async () => {
+    const status = element(by.binding('slowHttpStatus'));
+    const button = element(by.css('[ng-click="slowHttp()"]'));
 
-    expect(status.getText()).toEqual('not started');
+    expect(await status.getText()).toEqual('not started');
 
-    button.click();
+    await button.click();
 
-    expect(status.getText()).toEqual('done');
+    expect(await status.getText()).toEqual('done');
   });
 
-  it('waits for $timeout', function() {
-    var status = element(by.binding('slowAngularTimeoutStatus'));
-    var button = element(by.css('[ng-click="slowAngularTimeout()"]'));
+  it('waits for $timeout', async () => {
+    const status = element(by.binding('slowAngularTimeoutStatus'));
+    const button = element(by.css('[ng-click="slowAngularTimeout()"]'));
 
-    expect(status.getText()).toEqual('not started');
+    expect(await status.getText()).toEqual('not started');
 
-    button.click();
+    await button.click();
 
-    expect(status.getText()).toEqual('done');
+    expect(await status.getText()).toEqual('done');
   });
 });
diff --git a/spec/errorTest/baseCase/success_spec.js b/spec/errorTest/baseCase/success_spec.js
@@ -1,7 +1,7 @@
-describe('success spec', function() {
-  it('should pass', function() {
-    browser.get('index.html');
-    var greeting = element(by.binding('greeting'));
-    expect(greeting.getText()).toEqual('Hiya');
+describe('success spec', () => {
+  it('should pass', async () => {
+    await browser.get('index.html');
+    const greeting = element(by.binding('greeting'));
+    expect(await greeting.getText()).toEqual('Hiya');
   });
 });
diff --git a/spec/errorTest/baseCase/timeout_spec.js b/spec/errorTest/baseCase/timeout_spec.js
@@ -1,5 +1,5 @@
-describe('timeout spec', function() {
-  it('should timeout due to jasmine spec limit', function() {
-    browser.get('index.html#/form');
+describe('timeout spec', () => {
+  it('should timeout due to jasmine spec limit', async () => {
+    await browser.get('index.html#/form');
   }, 1);
 });
diff --git a/spec/errorTest/browserStackAuthentication.js b/spec/errorTest/browserStackAuthentication.js
@@ -1,8 +1,9 @@
-var env = require('../environment.js');
+const env = require('../environment.js');
 
 exports.config = {
   browserstackUser: 'foobar',
   browserstackKey: 'foobar',
+  SELENIUM_PROMISE_MANAGER: false,
 
   framework: 'jasmine',
 

diff --git a/spec/errorTest/debugMultiCapabilities.js b/spec/errorTest/debugMultiCapabilities.js
@@ -1,7 +1,8 @@
-var env = require('../environment.js');
+const env = require('../environment.js');
 
 exports.config = {
   seleniumAddress: env.seleniumAddress,
+  SELENIUM_PROMISE_MANAGER: false,
   framework: 'jasmine',
   debug: true,
   specs: [

diff --git a/spec/errorTest/getMultiCapabilitiesConf.js b/spec/errorTest/getMultiCapabilitiesConf.js
@@ -1,7 +1,8 @@
-var env = require('../environment.js');
+const env = require('../environment.js');
 
 exports.config = {
   seleniumAddress: env.seleniumAddress,
+  SELENIUM_PROMISE_MANAGER: false,
 
   // Spec patterns are relative to this directory.
   specs: [

diff --git a/spec/errorTest/mochaFailureConf.js b/spec/errorTest/mochaFailureConf.js
@@ -1,7 +1,8 @@
-var env = require('../environment.js');
+const env = require('../environment.js');
 
 exports.config = {
   seleniumAddress: env.seleniumAddress,
+  SELENIUM_PROMISE_MANAGER: false,
 
   specs: [
     'baseCase/mocha_failure_spec.js'

diff --git a/spec/errorTest/multiFailureConf.js b/spec/errorTest/multiFailureConf.js
@@ -1,7 +1,8 @@
-var env = require('../environment.js');
+const env = require('../environment.js');
 
 exports.config = {
   seleniumAddress: env.seleniumAddress,
+  SELENIUM_PROMISE_MANAGER: false,
 
   framework: 'jasmine',