diff --git a/.circleci/config.yml b/.circleci/config.yml index 33d2638b085..2630b369840 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,6 +40,7 @@ jobs: docker: # need '-browsers' version to test in real (xvfb-wrapped) browsers - image: circleci/node:10.9.0-browsers + parallelism: 2 working_directory: ~/plotly.js steps: - attach_workspace: @@ -52,6 +53,7 @@ jobs: docker: # need '-browsers' version to test in real (xvfb-wrapped) browsers - image: circleci/node:10.9.0-browsers + parallelism: 2 working_directory: ~/plotly.js steps: - attach_workspace: @@ -75,6 +77,7 @@ jobs: test-image: docker: - image: plotly/testbed:latest + parallelism: 4 working_directory: /var/www/streambed/image_server/plotly.js/ steps: - attach_workspace: @@ -121,6 +124,18 @@ jobs: name: Run syntax tests command: ./.circleci/test.sh syntax + test-bundle: + docker: + # need '-browsers' version to test in real (xvfb-wrapped) browsers + - image: circleci/node:10.9.0-browsers + working_directory: ~/plotly.js + steps: + - attach_workspace: + at: ~/ + - run: + name: Run test-bundle + command: ./.circleci/test.sh bundle + publish: docker: - image: circleci/node:10.9.0 @@ -157,6 +172,9 @@ workflows: build-and-test: jobs: - build + - test-bundle: + requires: + - build - test-jasmine: requires: - build diff --git a/.circleci/test.sh b/.circleci/test.sh index 62fd478837f..8094323e6d5 100755 --- a/.circleci/test.sh +++ b/.circleci/test.sh @@ -44,8 +44,8 @@ case $1 in jasmine) set_tz - npm run test-jasmine -- --skip-tags=gl,noCI,flaky || EXIT_STATE=$? - npm run test-bundle || EXIT_STATE=$? + SUITE=$(circleci tests glob "$ROOT/test/jasmine/tests/*" | circleci tests split) + npm run test-jasmine -- $SUITE --skip-tags=gl,noCI,flaky --showSkipped || EXIT_STATE=$? exit $EXIT_STATE ;; @@ -53,10 +53,10 @@ case $1 in jasmine2) set_tz - SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl)) + SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=gl | circleci tests split)) for s in ${SHARDS[@]}; do - retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI + retry npm run test-jasmine -- "$s" --tags=gl --skip-tags=noCI --showSkipped done exit $EXIT_STATE @@ -65,23 +65,29 @@ case $1 in jasmine3) set_tz - SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=flaky)) + SHARDS=($(node $ROOT/tasks/shard_jasmine_tests.js --tag=flaky | circleci tests split)) for s in ${SHARDS[@]}; do - retry npm run test-jasmine -- "$s" --tags=flaky --skip-tags=noCI + retry npm run test-jasmine -- "$s" --tags=flaky --skip-tags=noCI --showSkipped done exit $EXIT_STATE ;; image) - npm run test-image || EXIT_STATE=$? + SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | circleci tests split) + npm run test-image -- $SUITE --filter || EXIT_STATE=$? exit $EXIT_STATE ;; image2) npm run test-export || EXIT_STATE=$? - npm run test-image-gl2d || EXIT_STATE=$? + exit $EXIT_STATE + ;; + + bundle) + set_tz + npm run test-bundle || EXIT_STATE=$? exit $EXIT_STATE ;; diff --git a/package.json b/package.json index 034b53eecff..32ba40583b0 100644 --- a/package.json +++ b/package.json @@ -35,11 +35,10 @@ "pretest": "node tasks/pretest.js", "test-jasmine": "karma start test/jasmine/karma.conf.js", "test-image": "node tasks/test_image.js", - "test-image-gl2d": "node tasks/test_image.js gl2d_* --queue", "test-export": "node tasks/test_export.js", "test-syntax": "node tasks/test_syntax.js && npm run find-strings -- --no-output", "test-bundle": "node tasks/test_bundle.js", - "test": "npm run test-jasmine && npm run test-bundle && npm run test-image && npm run test-image-gl2d && npm run test-syntax && npm run lint", + "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint", "start-test_dashboard": "node devtools/test_dashboard/server.js", "start-image_viewer": "node devtools/image_viewer/server.js", "start": "npm run start-test_dashboard", diff --git a/test/image/compare_pixels_test.js b/test/image/compare_pixels_test.js index e5a5a1f43f1..8ed7fa7ff03 100644 --- a/test/image/compare_pixels_test.js +++ b/test/image/compare_pixels_test.js @@ -1,4 +1,5 @@ var fs = require('fs'); +var minimist = require('minimist'); var common = require('../../tasks/util/common'); var getMockList = require('./assets/get_mock_list'); @@ -48,45 +49,57 @@ var QUEUE_WAIT = 10; * npm run test-image -- gl3d_* --queue */ -var pattern = process.argv[2]; -var mockList = getMockList(pattern); -var isInQueue = (process.argv[3] === '--queue'); +var argv = minimist(process.argv.slice(2), {boolean: ['queue', 'filter' ]}); +var isInQueue = argv.queue; +var filter = argv.filter; -if(mockList.length === 0) { - throw new Error('No mocks found with pattern ' + pattern); +var allMock = false; +// If no pattern is provided, all mocks are compared +if(argv._.length === 0) { + allMock = true; + argv._.push(''); } -// filter out untestable mocks if no pattern is specified -if(!pattern) { +// Build list of mocks to compare +var allMockList = []; +argv._.forEach(function(pattern) { + var mockList = getMockList(pattern); + + if(mockList.length === 0) { + throw new Error('No mocks found with pattern ' + pattern); + } + + allMockList = allMockList.concat(mockList); +}); + +// To get rid of duplicates +function unique(value, index, self) { + return self.indexOf(value) === index; +} +allMockList = allMockList.filter(unique); + +// filter out untestable mocks if no pattern is specified (ie. we're testing all mocks) +// or if flag '--filter' is provided +if(allMock || filter) { console.log('Filtering out untestable mocks:'); - mockList = mockList.filter(untestableFilter); + allMockList = allMockList.filter(untestableFilter); console.log('\n'); } -// gl2d have limited image-test support -if(pattern === 'gl2d_*') { - if(!isInQueue) { - console.log('WARN: Running gl2d image tests in batch may lead to unwanted results\n'); - } - console.log('\nSorting gl2d mocks to avoid gl-shader conflicts'); - sortGl2dMockList(mockList); - console.log(''); -} +sortGl2dMockList(allMockList); // main if(isInQueue) { - runInQueue(mockList); + runInQueue(allMockList); } else { - runInBatch(mockList); + runInBatch(allMockList); } /* Test cases: * * - font-wishlist - * - all gl2d * - all mapbox - * - gl3d_cone-* * * don't behave consistently from run-to-run and/or * machine-to-machine; skip over them for now. @@ -95,7 +108,6 @@ else { function untestableFilter(mockName) { var cond = !( mockName === 'font-wishlist' || - mockName.indexOf('gl2d_') !== -1 || mockName.indexOf('mapbox_') !== -1 ); diff --git a/test/jasmine/karma.conf.js b/test/jasmine/karma.conf.js index 77bc53c0cea..818fd2e1f2b 100644 --- a/test/jasmine/karma.conf.js +++ b/test/jasmine/karma.conf.js @@ -59,6 +59,7 @@ if(argv.info) { ' - `--nowatch (dflt: `false`, `true` on CI)`: run karma w/o `autoWatch` / multiple run mode', ' - `--failFast` (dflt: `false`): exit karma upon first test failure', ' - `--verbose` (dflt: `false`): show test result using verbose reporter', + ' - `--showSkipped` (dflt: `false`): show tests that are skipped', ' - `--tags`: run only test with given tags (using the `jasmine-spec-tags` framework)', ' - `--width`(dflt: 1035): set width of the browser window', ' - `--height` (dflt: 617): set height of the browser window', @@ -111,7 +112,7 @@ var pathToCustomMatchers = path.join(__dirname, 'assets', 'custom_matchers.js'); var pathToUnpolyfill = path.join(__dirname, 'assets', 'unpolyfill.js'); var pathToMathJax = path.join(constants.pathToDist, 'extras', 'mathjax'); -var reporters = (isFullSuite && !argv.tags) ? ['dots', 'spec'] : ['progress']; +var reporters = ((isFullSuite && !argv.tags) || argv.showSkipped) ? ['dots', 'spec'] : ['progress']; if(argv.failFast) reporters.push('fail-fast'); if(argv.verbose) reporters.push('verbose'); diff --git a/test/jasmine/tests/sankey_test.js b/test/jasmine/tests/sankey_test.js index 69ec5616e73..81eb93ae5c4 100644 --- a/test/jasmine/tests/sankey_test.js +++ b/test/jasmine/tests/sankey_test.js @@ -997,7 +997,7 @@ describe('sankey tests', function() { }); ['node', 'link'].forEach(function(obj) { - it('should not output hover/unhover event data when ' + obj + '.hoverinfo is skip', function(done) { + it('@flaky should not output hover/unhover event data when ' + obj + '.hoverinfo is skip', function(done) { var fig = Lib.extendDeep({}, mock); Plotly.plot(gd, fig)