Shard gl jasmine tests

.circleci/test.sh

@@ -4,23 +4,31 @@
 set +e
 set +o pipefail
 
+ROOT=$(dirname $0)/..
 EXIT_STATE=0
 MAX_AUTO_RETRY=5
 
+log () {
+    echo -e "\n$1"
+}
+
 # inspired by https://unix.stackexchange.com/a/82602
 retry () {
-    local n=0
+    local n=1
 
     until [ $n -ge $MAX_AUTO_RETRY ]; do
-        "$@" && break
+        "$@" --failFast && break
+        log "run $n of $MAX_AUTO_RETRY failed, trying again ..."
         n=$[$n+1]
-        echo ''
-        echo run $n of $MAX_AUTO_RETRY failed, trying again ...
-        echo ''
-        sleep 15
     done
 
     if [ $n -eq $MAX_AUTO_RETRY ]; then
+        log "one last time, w/o failing fast"
+        "$@" && n=0
+    fi
+
+    if [ $n -eq $MAX_AUTO_RETRY ]; then
+        log "all $n runs failed, moving on."
         EXIT_STATE=1
     fi
 }
@@ -29,13 +37,18 @@ case $1 in
 
     jasmine)
         npm run test-jasmine -- --skip-tags=gl,noCI,flaky || EXIT_STATE=$?
+        retry npm run test-jasmine -- --tags=flaky --skip-tags=noCI
         npm run test-bundle || EXIT_STATE=$?
         exit $EXIT_STATE
         ;;
 
     jasmine2)
-        retry npm run test-jasmine -- --tags=gl --skip-tags=noCI,flaky
-        retry npm run test-jasmine -- --tags=flaky --skip-tags=noCI
+        SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl))
+
+        for s in ${SHARDS[@]}; do
+            retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI
+        done
+
         exit $EXIT_STATE
         ;;
 

package.json

@@ -136,6 +136,7 @@
     "karma": "^3.0.0",
     "karma-browserify": "^5.3.0",
     "karma-chrome-launcher": "^2.0.0",
+    "karma-fail-fast-reporter": "^1.0.5",
     "karma-firefox-launcher": "^1.0.1",
     "karma-jasmine": "^1.1.2",
     "karma-jasmine-spec-tags": "^1.0.1",

tasks/shard_jasmine_tests.js

@@ -0,0 +1,94 @@
+var fs = require('fs');
+var path = require('path');
+
+var falafel = require('falafel');
+var glob = require('glob');
+var minimist = require('minimist');
+
+var pathToJasmineTests = require('./util/constants').pathToJasmineTests;
+var isJasmineTestIt = require('./util/common').isJasmineTestIt;
+
+var argv = minimist(process.argv.slice(2), {
+    string: ['tag', 'limit'],
+    alias: {
+        tag: ['t'],
+        limit: ['l'],
+    },
+    default: {
+        limit: 20
+    }
+});
+
+var tag = argv.tag;
+var limit = argv.limit;
+
+glob(path.join(pathToJasmineTests, '*.js'), function(err, files) {
+    if(err) throw err;
+
+    var file2cnt = {};
+
+    files.forEach(function(file) {
+        var code = fs.readFileSync(file, 'utf-8');
+        var bn = path.basename(file);
+
+        falafel(code, function(node) {
+            if(isJasmineTestIt(node, tag)) {
+                if(file2cnt[bn]) {
+                    file2cnt[bn]++;
+                } else {
+                    file2cnt[bn] = 1;
+                }
+            }
+        });
+    });
+
+    var ranking = Object.keys(file2cnt);
+    var runs = [];
+
+    // if 'it' count in file greater than threshold,
+    // run only this file separately,
+    // don't try to shard within file
+    Object.keys(file2cnt).forEach(function(f) {
+        if(file2cnt[f] > limit) {
+            runs.push(f);
+            ranking.splice(ranking.indexOf(f), 1);
+        }
+    });
+
+    // sort ranking in decreasing order
+    ranking.sort(function(a, b) { return file2cnt[b] - file2cnt[a]; });
+
+    var runi;
+    var cnt;
+
+    function newRun() {
+        var r0 = ranking[0];
+        runi = [r0];
+        cnt = file2cnt[r0];
+        ranking.shift();
+    }
+
+    function concat() {
+        runs.push(runi.join(','));
+    }
+
+    // try to match files with many tests with files not-that-many,
+    // by matching first rank with one or multiple trailing ranks.
+    newRun();
+    while(ranking.length) {
+        var rn = ranking[ranking.length - 1];
+
+        if((cnt + file2cnt[rn]) > limit) {
+            concat();
+            newRun();
+        } else {
+            runi.push(rn);
+            cnt += file2cnt[rn];
+            ranking.pop();
+        }
+    }
+    concat();
+
+    // print result to stdout
+    console.log(runs.join('\n'));
+});

tasks/test_syntax.js

@@ -8,6 +8,11 @@ var readLastLines = require('read-last-lines');
 var eslint = require('eslint');
 var trueCasePath = require('true-case-path');
 
+var common = require('./util/common');
+var isJasmineTestIt = common.isJasmineTestIt;
+var isJasmineTestDescribe = common.isJasmineTestDescribe;
+var hasJasmineTestTag = common.hasJasmineTestTag;
+
 var constants = require('./util/constants');
 var srcGlob = path.join(constants.pathToSrc, '**/*.js');
 var libGlob = path.join(constants.pathToLib, '**/*.js');
@@ -28,26 +33,60 @@ assertES5();
 // check for for focus and exclude jasmine blocks
 function assertJasmineSuites() {
     var BLACK_LIST = ['fdescribe', 'fit', 'xdescribe', 'xit'];
+    var TAGS = ['noCI', 'noCIdep', 'gl', 'flaky'];
+    var IT_ONLY_TAGS = ['gl', 'flaky'];
     var logs = [];
 
+    var addTagPrefix = function(t) { return '@' + t; };
+
     glob(combineGlobs([testGlob, bundleTestGlob]), function(err, files) {
         files.forEach(function(file) {
             var code = fs.readFileSync(file, 'utf-8');
+            var bn = path.basename(file);
 
             falafel(code, {locations: true}, function(node) {
+                var lineInfo = '[line ' + node.loc.start.line + '] :';
+
                 if(node.type === 'Identifier' && BLACK_LIST.indexOf(node.name) !== -1) {
                     logs.push([
-                        path.basename(file),
-                        '[line ' + node.loc.start.line + '] :',
+                        bn, lineInfo,
                         'contains either a *fdescribe*, *fit*,',
                         '*xdescribe* or *xit* block.'
                     ].join(' '));
                 }
-            });
 
+                if(isJasmineTestIt(node)) {
+                    if(hasJasmineTestTag(node)) {
+                        if(TAGS.every(function(t) { return !hasJasmineTestTag(node, t); })) {
+                            logs.push([
+                                bn, lineInfo,
+                                'contains an unrecognized tag,',
+                                'not one of: ' + TAGS.map(addTagPrefix).join(', ')
+                            ].join(' '));
+                        }
+                    }
+
+                    if(hasJasmineTestTag(node, 'gl') && hasJasmineTestTag(node, 'flaky')) {
+                        logs.push([
+                            bn, lineInfo,
+                            'contains a @gl tag AND a @flaky tag, which is not allowed'
+                        ].join(' '));
+                    }
+                }
+
+                IT_ONLY_TAGS.forEach(function(t) {
+                    if(isJasmineTestDescribe(node, t)) {
+                        logs.push([
+                            bn, lineInfo,
+                            'contains a', addTagPrefix(t), 'tag is a *describe* block,',
+                            addTagPrefix(t), 'tags are only allowed in jasmine *it* blocks.'
+                        ].join(' '));
+                    }
+                });
+            });
         });
 
-        log('no jasmine suites focus/exclude blocks', logs);
+        log('no jasmine suites focus/exclude blocks or wrong tag patterns', logs);
     });
 }
 

tasks/util/common.js

@@ -105,3 +105,30 @@ exports.formatEnumeration = function(list) {
         return '`' + l + '`' + ending;
     }).join(' ');
 };
+
+exports.hasJasmineTestTag = function(node, tag) {
+    var re = tag ?
+        new RegExp('@' + tag + '\\s') :
+        new RegExp('@' + '\\w');
+    return re.test(node.source());
+};
+
+function isJasmineBase(block, node, tag) {
+    return (
+        node.type === 'Literal' &&
+        node.parent &&
+        node.parent.type === 'CallExpression' &&
+        node.parent.callee &&
+        node.parent.callee.type === 'Identifier' &&
+        node.parent.callee.name === block &&
+        (tag === undefined || exports.hasJasmineTestTag(node, tag))
+    );
+}
+
+exports.isJasmineTestIt = function(node, tag) {
+    return isJasmineBase('it', node, tag);
+};
+
+exports.isJasmineTestDescribe = function(node, tag) {
+    return isJasmineBase('describe', node, tag);
+};

test/jasmine/karma.conf.js

@@ -7,16 +7,18 @@ var constants = require('../../tasks/util/constants');
 var isCI = !!process.env.CIRCLECI;
 var argv = minimist(process.argv.slice(4), {
     string: ['bundleTest', 'width', 'height'],
-    'boolean': ['info', 'nowatch', 'verbose', 'Chrome', 'Firefox'],
+    'boolean': ['info', 'nowatch', 'failFast', 'verbose', 'Chrome', 'Firefox'],
     alias: {
         'Chrome': 'chrome',
         'Firefox': ['firefox', 'FF'],
         'bundleTest': ['bundletest', 'bundle_test'],
-        'nowatch': 'no-watch'
+        'nowatch': 'no-watch',
+        'failFast': 'fail-fast'
     },
     'default': {
         info: false,
         nowatch: isCI,
+        failFast: false,
         verbose: false,
         width: '1035',
         height: '617'
@@ -105,6 +107,10 @@ var pathToJQuery = path.join(__dirname, 'assets', 'jquery-1.8.3.min.js');
 var pathToIE9mock = path.join(__dirname, 'assets', 'ie9_mock.js');
 var pathToCustomMatchers = path.join(__dirname, 'assets', 'custom_matchers.js');
 
+var reporters = (isFullSuite && !argv.tags) ? ['dots', 'spec'] : ['progress'];
+if(argv.failFast) reporters.push('fail-fast');
+if(argv.verbose) reporters.push('verbose');
+
 function func(config) {
     // level of logging
     // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG
@@ -154,7 +160,7 @@ func.defaultConfig = {
     // See note in CONTRIBUTING.md about more verbose reporting via karma-verbose-reporter:
     // https://www.npmjs.com/package/karma-verbose-reporter ('verbose')
     //
-    reporters: (isFullSuite && !argv.tags) ? ['dots', 'spec'] : ['progress'],
+    reporters: reporters,
 
     // web server port
     port: 9876,
@@ -235,8 +241,13 @@ func.defaultConfig = {
         suppressPassed: true,
         suppressSkipped: false,
         showSpecTiming: false,
+        // use 'karma-fail-fast-reporter' to fail fast w/o conflicting
+        // with other karma plugins
         failFast: false
-    }
+    },
+
+    // e.g. when a test file does not container a given spec tags
+    failOnEmptyTestSuite: false
 };
 
 func.defaultConfig.preprocessors[pathToCustomMatchers] = ['browserify'];
@@ -291,9 +302,4 @@ if(argv.Chrome) browsers.push('_Chrome');
 if(argv.Firefox) browsers.push('_Firefox');
 if(browsers.length === 0) browsers.push('_Chrome');
 
-// add verbose reporter if specified
-if(argv.verbose) {
-    func.defaultConfig.reporters.push('verbose');
-}
-
 module.exports = func;