Shard gl jasmine tests

.circleci/test.sh

@@ -4,6 +4,7 @@
 set +e
 set +o pipefail
 
+ROOT=$(dirname $0)/..
 EXIT_STATE=0
 MAX_AUTO_RETRY=5
 
@@ -12,12 +13,14 @@ retry () {
     local n=0
 
     until [ $n -ge $MAX_AUTO_RETRY ]; do
+        if [ $n -ge 1 ]; then
+            echo ''
+            echo run $n of $MAX_AUTO_RETRY failed, trying again ...
+            echo ''
+            sleep 15
+        fi
         "$@" && break
         n=$[$n+1]
-        echo ''
-        echo run $n of $MAX_AUTO_RETRY failed, trying again ...
-        echo ''
-        sleep 15
     done
 
     if [ $n -eq $MAX_AUTO_RETRY ]; then
@@ -34,7 +37,12 @@ case $1 in
         ;;
 
     jasmine2)
-        retry npm run test-jasmine -- --tags=gl --skip-tags=noCI,flaky
+        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl))
+
+        for s in ${SHARDS[@]}; do
+            retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI
+        done
+
         retry npm run test-jasmine -- --tags=flaky --skip-tags=noCI
         exit $EXIT_STATE
         ;;

tasks/shard_jasmine_tests.js

@@ -0,0 +1,94 @@
+var fs = require('fs');
+var path = require('path');
+
+var falafel = require('falafel');
+var glob = require('glob');
+var minimist = require('minimist');
+
+var pathToJasmineTests = require('./util/constants').pathToJasmineTests;
+var isJasmineTestIt = require('./util/common').isJasmineTestIt;
+
+var argv = minimist(process.argv.slice(2), {
+    string: ['tag', 'limit'],
+    alias: {
+        tag: ['t'],
+        limit: ['l'],
+    },
+    default: {
+        limit: 20
+    }
+});
+
+var tag = argv.tag;
+var limit = argv.limit;
+
+glob(path.join(pathToJasmineTests, '*.js'), function(err, files) {
+    if(err) throw err;
+
+    var file2cnt = {};
+
+    files.forEach(function(file) {
+        var code = fs.readFileSync(file, 'utf-8');
+        var bn = path.basename(file);
+
+        falafel(code, function(node) {
+            if(isJasmineTestIt(node, tag)) {
+                if(file2cnt[bn]) {
+                    file2cnt[bn]++;
+                } else {
+                    file2cnt[bn] = 1;
+                }
+            }
+        });
+    });
+
+    var ranking = Object.keys(file2cnt);
+    var runs = [];
+
+    // if 'it' count in file greater than threshold,
+    // run only this file separately,
+    // don't try to shard within file
+    Object.keys(file2cnt).forEach(function(f) {
+        if(file2cnt[f] > limit) {
+            runs.push(f);
+            ranking.splice(ranking.indexOf(f), 1);
+        }
+    });
+
+    // sort ranking in decreasing order
+    ranking.sort(function(a, b) { return file2cnt[b] - file2cnt[a]; });
+
+    var runi;
+    var cnt;
+
+    function newRun() {
+        var r0 = ranking[0];
+        runi = [r0];
+        cnt = file2cnt[r0];
+        ranking.shift();
+    }
+
+    function concat() {
+        runs.push(runi.join(','));
+    }
+
+    // try to match files with many tests with files not-that-many,
+    // by matching first rank with one or multiple trailing ranks.
+    newRun();
+    while(ranking.length) {
+        var rn = ranking[ranking.length - 1];
+
+        if((cnt + file2cnt[rn]) > limit) {
+            concat();
+            newRun();
+        } else {
+            runi.push(rn);
+            cnt += file2cnt[rn];
+            ranking.pop();
+        }
+    }
+    concat();
+
+    // print result to stdout
+    console.log(runs.join('\n'));
+});

tasks/test_syntax.js

@@ -8,6 +8,11 @@ var readLastLines = require('read-last-lines');
 var eslint = require('eslint');
 var trueCasePath = require('true-case-path');
 
+var common = require('./util/common');
+var isJasmineTestIt = common.isJasmineTestIt;
+var isJasmineTestDescribe = common.isJasmineTestDescribe;
+var hasJasmineTestTag = common.hasJasmineTestTag;
+
 var constants = require('./util/constants');
 var srcGlob = path.join(constants.pathToSrc, '**/*.js');
 var libGlob = path.join(constants.pathToLib, '**/*.js');
@@ -28,26 +33,60 @@ assertES5();
 // check for for focus and exclude jasmine blocks
 function assertJasmineSuites() {
     var BLACK_LIST = ['fdescribe', 'fit', 'xdescribe', 'xit'];
+    var TAGS = ['noCI', 'noCIdep', 'gl', 'flaky'];
+    var IT_ONLY_TAGS = ['gl', 'flaky'];
     var logs = [];
 
+    var addTagPrefix = function(t) { return '@' + t; };
+
     glob(combineGlobs([testGlob, bundleTestGlob]), function(err, files) {
         files.forEach(function(file) {
             var code = fs.readFileSync(file, 'utf-8');
+            var bn = path.basename(file);
 
             falafel(code, {locations: true}, function(node) {
+                var lineInfo = '[line ' + node.loc.start.line + '] :';
+
                 if(node.type === 'Identifier' && BLACK_LIST.indexOf(node.name) !== -1) {
                     logs.push([
-                        path.basename(file),
-                        '[line ' + node.loc.start.line + '] :',
+                        bn, lineInfo,
                         'contains either a *fdescribe*, *fit*,',
                         '*xdescribe* or *xit* block.'
                     ].join(' '));
                 }
-            });
 
+                if(isJasmineTestIt(node)) {
+                    if(hasJasmineTestTag(node)) {
+                        if(TAGS.every(function(t) { return !hasJasmineTestTag(node, t); })) {
+                            logs.push([
+                                bn, lineInfo,
+                                'contains an unrecognized tag,',
+                                'not one of: ' + TAGS.map(addTagPrefix).join(', ')
+                            ].join(' '));
+                        }
+                    }
+
+                    if(hasJasmineTestTag(node, 'gl') && hasJasmineTestTag(node, 'flaky')) {
+                        logs.push([
+                            bn, lineInfo,
+                            'contains a @gl tag AND a @flaky tag, which is not allowed'
+                        ].join(' '));
+                    }
+                }
+
+                IT_ONLY_TAGS.forEach(function(t) {
+                    if(isJasmineTestDescribe(node, t)) {
+                        logs.push([
+                            bn, lineInfo,
+                            'contains a', addTagPrefix(t), 'tag is a *describe* block,',
+                            addTagPrefix(t), 'tags are only allowed in jasmine *it* blocks.'
+                        ].join(' '));
+                    }
+                });
+            });
         });
 
-        log('no jasmine suites focus/exclude blocks', logs);
+        log('no jasmine suites focus/exclude blocks or wrong tag patterns', logs);
     });
 }
 

tasks/util/common.js

@@ -105,3 +105,30 @@ exports.formatEnumeration = function(list) {
         return '`' + l + '`' + ending;
     }).join(' ');
 };
+
+exports.hasJasmineTestTag = function(node, tag) {
+    var re = tag ?
+        new RegExp('@' + tag + '\\s') :
+        new RegExp('@' + '\\w');
+    return re.test(node.source());
+};
+
+function isJasmineBase(block, node, tag) {
+    return (
+        node.type === 'Literal' &&
+        node.parent &&
+        node.parent.type === 'CallExpression' &&
+        node.parent.callee &&
+        node.parent.callee.type === 'Identifier' &&
+        node.parent.callee.name === block &&
+        (tag === undefined || exports.hasJasmineTestTag(node, tag))
+    );
+}
+
+exports.isJasmineTestIt = function(node, tag) {
+    return isJasmineBase('it', node, tag);
+};
+
+exports.isJasmineTestDescribe = function(node, tag) {
+    return isJasmineBase('describe', node, tag);
+};

test/jasmine/karma.conf.js

@@ -236,7 +236,10 @@ func.defaultConfig = {
         suppressSkipped: false,
         showSpecTiming: false,
         failFast: false
-    }
+    },
+
+    // e.g. when a test file does not container a given spec tags
+    failOnEmptyTestSuite: false
 };
 
 func.defaultConfig.preprocessors[pathToCustomMatchers] = ['browserify'];

test/jasmine/tests/cone_test.js

@@ -60,7 +60,7 @@ describe('Test cone defaults', function() {
     });
 });
 
-describe('@gl Test cone autorange:', function() {
+describe('Test cone autorange:', function() {
     var gd;
 
     beforeEach(function() {
@@ -79,7 +79,7 @@ describe('@gl Test cone autorange:', function() {
         expect(sceneLayout.zaxis.range).toBeCloseToArray(zrng, 2, 'zaxis range - ' + msg);
     }
 
-    it('should add pad around cone position to make sure they fit on the scene', function(done) {
+    it('@gl should add pad around cone position to make sure they fit on the scene', function(done) {
         var fig = Lib.extendDeep({}, require('@mocks/gl3d_cone-autorange.json'));
         var rng0 = [0.103, 3.897];
 
@@ -177,7 +177,7 @@ describe('@gl Test cone autorange:', function() {
     });
 });
 
-describe('@gl Test cone interactions', function() {
+describe('Test cone interactions', function() {
     var gd;
 
     beforeEach(function() {
@@ -189,7 +189,7 @@ describe('@gl Test cone interactions', function() {
         destroyGraphDiv();
     });
 
-    it('should add/clear gl objects correctly', function(done) {
+    it('@gl should add/clear gl objects correctly', function(done) {
         var fig = Lib.extendDeep({}, require('@mocks/gl3d_cone-simple.json'));
         // put traces on same subplot
         delete fig.data[1].scene;
@@ -219,7 +219,7 @@ describe('@gl Test cone interactions', function() {
         .then(done);
     });
 
-    it('should not pass zero or infinite `coneSize` to gl-cone3d', function(done) {
+    it('@gl should not pass zero or infinite `coneSize` to gl-cone3d', function(done) {
         var base = {
             type: 'cone',
             x: [1, 2, 3],
@@ -254,7 +254,7 @@ describe('@gl Test cone interactions', function() {
         .then(done);
     });
 
-    it('should display hover labels', function(done) {
+    it('@gl should display hover labels', function(done) {
         var fig = Lib.extendDeep({}, require('@mocks/gl3d_cone-simple.json'));
         // only one trace on one scene
         fig.data = [fig.data[0]];
@@ -302,7 +302,7 @@ describe('@gl Test cone interactions', function() {
         .then(done);
     });
 
-    it('should display hover labels (multi-trace case)', function(done) {
+    it('@gl should display hover labels (multi-trace case)', function(done) {
         function _hover() {
             mouseEvent('mouseover', 282, 240);
             return delay(20)();