CircleCI: run jobs in parallel

.circleci/config.yml

@@ -40,6 +40,7 @@ jobs:
     docker:
       # need '-browsers' version to test in real (xvfb-wrapped) browsers
       - image: circleci/node:10.9.0-browsers
+    parallelism: 2
     working_directory: ~/plotly.js
     steps:
       - attach_workspace:
@@ -52,6 +53,7 @@ jobs:
     docker:
       # need '-browsers' version to test in real (xvfb-wrapped) browsers
       - image: circleci/node:10.9.0-browsers
+    parallelism: 2
     working_directory: ~/plotly.js
     steps:
       - attach_workspace:
@@ -75,6 +77,7 @@ jobs:
   test-image:
     docker:
       - image: plotly/testbed:latest
+    parallelism: 4
     working_directory: /var/www/streambed/image_server/plotly.js/
     steps:
       - attach_workspace:
@@ -121,6 +124,18 @@ jobs:
           name: Run syntax tests
           command: ./.circleci/test.sh syntax
 
+  test-bundle:
+    docker:
+      # need '-browsers' version to test in real (xvfb-wrapped) browsers
+      - image: circleci/node:10.9.0-browsers
+    working_directory: ~/plotly.js
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          name: Run test-bundle
+          command: ./.circleci/test.sh bundle
+
   publish:
     docker:
       - image: circleci/node:10.9.0
@@ -157,6 +172,9 @@ workflows:
   build-and-test:
     jobs:
       - build
+      - test-bundle:
+          requires:
+            - build
       - test-jasmine:
           requires:
             - build

.circleci/test.sh

@@ -44,19 +44,19 @@ case $1 in
     jasmine)
         set_tz
 
-        npm run test-jasmine -- --skip-tags=gl,noCI,flaky || EXIT_STATE=$?
-        npm run test-bundle || EXIT_STATE=$?
+        SUITE=$(circleci tests glob "$ROOT/test/jasmine/tests/*" | circleci tests split)
+        npm run test-jasmine -- $SUITE --skip-tags=gl,noCI,flaky --showSkipped || EXIT_STATE=$?
 
         exit $EXIT_STATE
         ;;
 
     jasmine2)
         set_tz
 
-        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl))
+        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl | circleci tests split))
 
         for s in ${SHARDS[@]}; do
-            retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI
+            retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI --showSkipped
         done
 
         exit $EXIT_STATE
@@ -65,23 +65,29 @@ case $1 in
     jasmine3)
         set_tz
 
-        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=flaky))
+        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=flaky | circleci tests split))
 
         for s in ${SHARDS[@]}; do
-            retry npm run test-jasmine -- "$s" --tags=flaky --skip-tags=noCI
+            retry npm run test-jasmine -- "$s" --tags=flaky --skip-tags=noCI --showSkipped
         done
 
         exit $EXIT_STATE
         ;;
 
     image)
-        npm run test-image      || EXIT_STATE=$?
+        SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | circleci tests split)
+        npm run test-image -- $SUITE --filter || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 
     image2)
         npm run test-export     || EXIT_STATE=$?
-        npm run test-image-gl2d || EXIT_STATE=$?
+        exit $EXIT_STATE
+        ;;
+
+    bundle)
+        set_tz
+        npm run test-bundle || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 

package.json

@@ -35,7 +35,6 @@
     "pretest": "node tasks/pretest.js",
     "test-jasmine": "karma start test/jasmine/karma.conf.js",
     "test-image": "node tasks/test_image.js",
-    "test-image-gl2d": "node tasks/test_image.js gl2d_* --queue",
     "test-export": "node tasks/test_export.js",
     "test-syntax": "node tasks/test_syntax.js && npm run find-strings -- --no-output",
     "test-bundle": "node tasks/test_bundle.js",

test/image/compare_pixels_test.js

@@ -1,4 +1,5 @@
 var fs = require('fs');
+var minimist = require('minimist');
 
 var common = require('../../tasks/util/common');
 var getMockList = require('./assets/get_mock_list');
@@ -48,37 +49,51 @@ var QUEUE_WAIT = 10;
  *      npm run test-image -- gl3d_* --queue
  */
 
-var pattern = process.argv[2];
-var mockList = getMockList(pattern);
-var isInQueue = (process.argv[3] === '--queue');
+var argv = minimist(process.argv.slice(2), {boolean: ['queue', 'filter' ]});
+var isInQueue = argv.queue;
+var filter = argv.filter;
 
-if(mockList.length === 0) {
-    throw new Error('No mocks found with pattern ' + pattern);
+var allMock = false;
+// If no pattern is provided, all mocks are compared
+if(argv._.length === 0) {
+    allMock = true;
+    argv._.push('');
 }
 
-// filter out untestable mocks if no pattern is specified
-if(!pattern) {
-    console.log('Filtering out untestable mocks:');
-    mockList = mockList.filter(untestableFilter);
-    console.log('\n');
-}
+// Build list of mocks to compare
+var allMockList = [];
+argv._.forEach(function(pattern) {
+    var mockList = getMockList(pattern);
 
-// gl2d have limited image-test support
-if(pattern === 'gl2d_*') {
-    if(!isInQueue) {
-        console.log('WARN: Running gl2d image tests in batch may lead to unwanted results\n');
+    if(mockList.length === 0) {
+        throw new Error('No mocks found with pattern ' + pattern);
     }
-    console.log('\nSorting gl2d mocks to avoid gl-shader conflicts');
-    sortGl2dMockList(mockList);
-    console.log('');
+
+    allMockList = allMockList.concat(mockList);
+});
+
+// To get rid of duplicates
+Array.prototype.unique = function() {
+    return this.filter(function(value, index, self) {
+        return self.indexOf(value) === index;
+    });
+};
+allMockList = allMockList.unique();
+
+// filter out untestable mocks if no pattern is specified (ie. we're testing all mocks)
+// or if flag '--filter' is provided
+if(allMock || filter) {
+    console.log('Filtering out untestable mocks:');
+    allMockList = allMockList.filter(untestableFilter);
+    console.log('\n');
 }
 
 // main
 if(isInQueue) {
-    runInQueue(mockList);
+    runInQueue(allMockList);
 }
 else {
-    runInBatch(mockList);
+    runInBatch(allMockList);
 }
 
 /* Test cases:
@@ -95,7 +110,7 @@ else {
 function untestableFilter(mockName) {
     var cond = !(
         mockName === 'font-wishlist' ||
-        mockName.indexOf('gl2d_') !== -1 ||
+        // mockName.indexOf('gl2d_') !== -1 ||
         mockName.indexOf('mapbox_') !== -1
     );
 
@@ -104,34 +119,6 @@ function untestableFilter(mockName) {
     return cond;
 }
 
-/* gl2d pointcloud and other non-regl gl2d mock(s)
- * must be tested first on in order to work;
- * sort them here.
- *
- * gl-shader appears to conflict with regl.
- * We suspect that the lone gl context on CircleCI is
- * having issues with dealing with the two different
- * program binding algorithm.
- *
- * The problem will be solved by switching all our
- * WebGL-based trace types to regl.
- *
- * More info here:
- * https://github.com/plotly/plotly.js/pull/1037
- */
-function sortGl2dMockList(mockList) {
-    var mockNames = ['gl2d_pointcloud-basic', 'gl2d_heatmapgl'];
-    var pos = 0;
-
-    mockNames.forEach(function(m) {
-        var ind = mockList.indexOf(m);
-        var tmp = mockList[pos];
-        mockList[pos] = m;
-        mockList[ind] = tmp;
-        pos++;
-    });
-}
-
 function runInBatch(mockList) {
     var running = 0;
 

test/jasmine/karma.conf.js

@@ -59,6 +59,7 @@ if(argv.info) {
         '  - `--nowatch (dflt: `false`, `true` on CI)`: run karma w/o `autoWatch` / multiple run mode',
         '  - `--failFast` (dflt: `false`): exit karma upon first test failure',
         '  - `--verbose` (dflt: `false`): show test result using verbose reporter',
+        '  - `--showSkipped` (dflt: `false`): show tests that are skipped',
         '  - `--tags`: run only test with given tags (using the `jasmine-spec-tags` framework)',
         '  - `--width`(dflt: 1035): set width of the browser window',
         '  - `--height` (dflt: 617): set height of the browser window',
@@ -111,7 +112,7 @@ var pathToCustomMatchers = path.join(__dirname, 'assets', 'custom_matchers.js');
 var pathToUnpolyfill = path.join(__dirname, 'assets', 'unpolyfill.js');
 var pathToMathJax = path.join(constants.pathToDist, 'extras', 'mathjax');
 
-var reporters = (isFullSuite && !argv.tags) ? ['dots', 'spec'] : ['progress'];
+var reporters = ((isFullSuite && !argv.tags) || argv.showSkipped) ? ['dots', 'spec'] : ['progress'];
 if(argv.failFast) reporters.push('fail-fast');
 if(argv.verbose) reporters.push('verbose');
 

test/jasmine/tests/sankey_test.js

@@ -997,7 +997,7 @@ describe('sankey tests', function() {
         });
 
         ['node', 'link'].forEach(function(obj) {
-            it('should not output hover/unhover event data when ' + obj + '.hoverinfo is skip', function(done) {
+            it('@flaky should not output hover/unhover event data when ' + obj + '.hoverinfo is skip', function(done) {
                 var fig = Lib.extendDeep({}, mock);
 
                 Plotly.plot(gd, fig)