diff --git a/circle.yml b/circle.yml index 584761eefcb..285ebd40ea8 100644 --- a/circle.yml +++ b/circle.yml @@ -1,7 +1,7 @@ general: artifacts: - - "build/test_images/" - - "build/test_images_diff/" + - build/test_images/ + - build/test_images_diff/ machine: node: @@ -12,21 +12,16 @@ machine: - docker dependencies: - pre: - - eval $(node tasks/docker.js pull) - post: - - eval $(node tasks/docker.js run) - - npm run cibuild + override: + - npm install && npm dedupe && npm prune && npm install + - npm ls || true + - npm run docker -- pull - npm run pretest - - eval $(node tasks/docker.js setup) - - npm prune && npm ls + - npm run docker -- run + - npm run cibuild + - npm run docker -- setup test: override: - - npm run test-image - - npm run test-image-gl2d - - npm run test-export - - npm run citest-jasmine - - npm run test-bundle - - npm run test-syntax - - eslint . + - ./tasks/ci_test.sh: + parallel: true diff --git a/package.json b/package.json index 788e9f0dfd8..d5399661250 100644 --- a/package.json +++ b/package.json @@ -28,12 +28,12 @@ "build": "npm run preprocess && npm run bundle && npm run header && npm run stats", "cibuild": "npm run preprocess && node tasks/cibundle.js", "watch": "node tasks/watch.js", - "lint": "eslint --version && eslint . || true", - "lint-fix": "eslint . --fix", + "lint": "eslint --version && eslint .", + "lint-fix": "eslint . --fix || true", "docker": "node tasks/docker.js", "pretest": "node tasks/pretest.js", "test-jasmine": "karma start test/jasmine/karma.conf.js", - "citest-jasmine": "karma start test/jasmine/karma.ciconf.js", + "citest-jasmine": "CIRCLECI=1 karma start test/jasmine/karma.conf.js", "test-image": "node tasks/test_image.js", "test-image-gl2d": "node tasks/test_image.js gl2d_* --queue", "test-export": "node tasks/test_export.js", @@ -119,6 +119,9 @@ "karma-coverage": "^1.0.0", "karma-firefox-launcher": "^1.0.0", "karma-jasmine": "^1.1.0", + "karma-jasmine-spec-tags": "^1.0.1", + "karma-spec-reporter": "0.0.30", + "karma-verbose-reporter": "0.0.6", "madge": "^1.6.0", "node-sass": "^4.5.0", "npm-link-check": "^1.2.0", @@ -128,7 +131,7 @@ "read-last-lines": "^1.1.0", "requirejs": "^2.3.1", "through2": "^2.0.3", - "uglify-js": "^2.7.5", + "uglify-js": "~2.7.5", "watchify": "^3.9.0", "xml2js": "^0.4.16" } diff --git a/tasks/ci_test.sh b/tasks/ci_test.sh new file mode 100755 index 00000000000..5e2b7d40a95 --- /dev/null +++ b/tasks/ci_test.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +EXIT_STATE=0 + +case $CIRCLE_NODE_INDEX in + + 0) + npm run test-image || EXIT_STATE=$? + npm run test-image-gl2d || EXIT_STATE=$? + npm run test-export || EXIT_STATE=$? + npm run test-syntax || EXIT_STATE=$? + npm run lint || EXIT_STATE=$? + exit $EXIT_STATE + ;; + + 1) + npm run test-jasmine || EXIT_STATE=$? + npm run test-bundle || EXIT_STATE=$? + exit $EXIT_STATE + ;; + +esac diff --git a/tasks/docker.js b/tasks/docker.js index d3ba12a0131..a767caa587c 100644 --- a/tasks/docker.js +++ b/tasks/docker.js @@ -46,12 +46,5 @@ switch(arg) { break; } -// Log command string on CircleCI, to then `eval` them, -// which appears to be more reliable then calling `child_process.exec()` -if(isCI) { - console.log(cmd); -} -else { - console.log(msg); - common.execCmd(cmd, cb, errorCb); -} +console.log(msg); +common.execCmd(cmd, cb, errorCb); diff --git a/tasks/noci_test.sh b/tasks/noci_test.sh new file mode 100755 index 00000000000..46d0c779e43 --- /dev/null +++ b/tasks/noci_test.sh @@ -0,0 +1,16 @@ +#! /bin/bash + +EXIT_STATE=0 + +# tests that aren't run on CI + +# jasmine specs with @noCI tag +npm run citest-jasmine -- tests/*_test.js --tags noCI || EXIT_STATE=$? + +# mapbox image tests take too much resources on CI +npm run test-image -- mapbox_* || EXIT_STATE=$? + +# run gl2d image test again (some mocks are skipped on CI) +npm run test-image-gl2d || EXIT_STATE=$? + +exit $EXIT_STATE diff --git a/tasks/test_export.js b/tasks/test_export.js index de08f97e7e7..d747d818b6e 100644 --- a/tasks/test_export.js +++ b/tasks/test_export.js @@ -14,4 +14,6 @@ var cmd = containerCommands.getRunCmd( ); console.log(msg); -common.execCmd(cmd); +common.execCmd(containerCommands.ping, function() { + common.execCmd(cmd); +}); diff --git a/tasks/test_image.js b/tasks/test_image.js index 8156cabaead..eed7ccd9148 100644 --- a/tasks/test_image.js +++ b/tasks/test_image.js @@ -14,4 +14,6 @@ var cmd = containerCommands.getRunCmd( ); console.log(msg); -common.execCmd(cmd); +common.execCmd(containerCommands.ping, function() { + common.execCmd(cmd); +}); diff --git a/tasks/util/container_commands.js b/tasks/util/container_commands.js index 11743613172..938cf694993 100644 --- a/tasks/util/container_commands.js +++ b/tasks/util/container_commands.js @@ -22,8 +22,6 @@ containerCommands.setup = [ containerCommands.injectEnv, containerCommands.restart, 'sleep 1', - containerCommands.ping, - 'echo ' ].join(' && '); containerCommands.dockerRun = [ diff --git a/test/jasmine/assets/has_webgl_support.js b/test/jasmine/assets/has_webgl_support.js deleted file mode 100644 index c5702b53ff9..00000000000 --- a/test/jasmine/assets/has_webgl_support.js +++ /dev/null @@ -1,18 +0,0 @@ -'use strict'; - -var getContext = require('webgl-context'); - -module.exports = function hasWebGLSupport(testName) { - var gl, canvas; - - canvas = document.createElement('canvas'); - gl = getContext({canvas: canvas}); - - var hasSupport = !!gl; - - if(!hasSupport) { - console.warn('Cannot get WebGL context. Skip test *' + testName + '*'); - } - - return hasSupport; -}; diff --git a/test/jasmine/karma.ciconf.js b/test/jasmine/karma.ciconf.js deleted file mode 100644 index 7b782e60cea..00000000000 --- a/test/jasmine/karma.ciconf.js +++ /dev/null @@ -1,42 +0,0 @@ -// Karma configuration - -function func(config) { - - // level of logging - // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG - func.defaultConfig.logLevel = config.LOG_INFO; - - // Note: config.LOG_DEBUG may not be verbose enough to pin down the source of failed tests. - // See the note in CONTRIBUTING.md about karma-verbose-reporter: - // func.defaultConfig.reporters = ['verbose']; - - - // Continuous Integration mode - - /* - * WebGL interaction test cases fail on the CircleCI - * most likely due to a WebGL/driver issue; - * exclude them from the CircleCI test bundle. - * - */ - func.defaultConfig.exclude = [ - 'tests/gl_plot_interact_test.js', - 'tests/gl_plot_interact_basic_test.js', - 'tests/gl2d_scatterplot_contour_test.js', - 'tests/gl2d_pointcloud_test.js' - ]; - - // if true, Karma captures browsers, runs the tests and exits - func.defaultConfig.singleRun = true; - - func.defaultConfig.browserNoActivityTimeout = 30000; // 30 seconds - - func.defaultConfig.autoWatch = false; - - func.defaultConfig.browsers = ['Firefox_WindowSized']; - - config.set(func.defaultConfig); -} - -func.defaultConfig = require('./karma.conf').defaultConfig; -module.exports = func; diff --git a/test/jasmine/karma.conf.js b/test/jasmine/karma.conf.js index 5210abba9a5..9412e30666c 100644 --- a/test/jasmine/karma.conf.js +++ b/test/jasmine/karma.conf.js @@ -17,6 +17,7 @@ var constants = require('../../tasks/util/constants'); var arg = process.argv[4]; +var isCI = !!process.env.CIRCLECI; var testFileGlob = arg ? arg : 'tests/*_test.js'; var isSingleSuiteRun = (arg && arg.indexOf('bundle_tests/') === -1); var isRequireJSTest = (arg && arg.indexOf('bundle_tests/requirejs') !== -1); @@ -53,13 +54,14 @@ func.defaultConfig = { // frameworks to use // available frameworks: https://npmjs.org/browse/keyword/karma-adapter - frameworks: ['jasmine', 'browserify'], + frameworks: ['jasmine', 'jasmine-spec-tags', 'browserify'], // list of files / patterns to load in the browser // // N.B. this field is filled below files: [], + // list of files / pattern to exclude exclude: [], // preprocess matching files before serving them to the browser @@ -75,7 +77,7 @@ func.defaultConfig = { // See note in CONTRIBUTING.md about more verbose reporting via karma-verbose-reporter: // https://www.npmjs.com/package/karma-verbose-reporter ('verbose') // - reporters: ['progress'], + reporters: isSingleSuiteRun ? ['progress'] : ['dots', 'spec'], // web server port port: 9876, @@ -84,14 +86,23 @@ func.defaultConfig = { colors: true, // enable / disable watching file and executing tests whenever any file changes - autoWatch: true, + autoWatch: !isCI, + + // if true, Karma captures browsers, runs the tests and exits + singleRun: isCI, + + // how long will Karma wait for a message from a browser before disconnecting (30 ms) + browserNoActivityTimeout: 30000, // start these browsers // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher browsers: ['Chrome_WindowSized'], // custom browser options + // // window-size values came from observing default size + // + // '--ignore-gpu-blacklist' allow to test WebGL on CI (!!!) customLaunchers: { Chrome_WindowSized: { base: 'Chrome', @@ -103,28 +114,40 @@ func.defaultConfig = { } }, - // Continuous Integration mode - // if true, Karma captures browsers, runs the tests and exits - singleRun: false, - browserify: { transform: ['../../tasks/util/shortcut_paths.js'], extensions: ['.js'], - watch: true, + watch: !isCI, debug: true + }, + + // unfortunately a few tests don't behave well on CI + // using `karma-jasmine-spec-tags` + // add @noCI to the spec description to skip a spec on CI + client: { + tagPrefix: '@', + skipTags: isCI ? 'noCI' : null + }, + + // use 'karma-spec-reporter' to log info about skipped specs + specReporter: { + suppressErrorSummary: true, + suppressFailed: true, + suppressPassed: true, + suppressSkipped: false, + showSpecTiming: false, + failFast: false } }; - // Add lib/index.js to single-suite runs, // to avoid import conflicts due to plotly.js // circular dependencies. if(isSingleSuiteRun) { - func.defaultConfig.files = [ + func.defaultConfig.files.push( pathToJQuery, - pathToMain, - testFileGlob - ]; + pathToMain + ); func.defaultConfig.preprocessors[pathToMain] = ['browserify']; func.defaultConfig.preprocessors[testFileGlob] = ['browserify']; @@ -132,8 +155,7 @@ if(isSingleSuiteRun) { else if(isRequireJSTest) { func.defaultConfig.files = [ constants.pathToRequireJS, - constants.pathToRequireJSFixture, - testFileGlob + constants.pathToRequireJSFixture ]; } else if(isIE9Test) { @@ -141,20 +163,15 @@ else if(isIE9Test) { // to catch reference errors that could occur // when plotly.js is first loaded. - func.defaultConfig.files = [ - './assets/ie9_mock.js', - testFileGlob - ]; - + func.defaultConfig.files.push('./assets/ie9_mock.js'); func.defaultConfig.preprocessors[testFileGlob] = ['browserify']; } else { - func.defaultConfig.files = [ - pathToJQuery, - testFileGlob - ]; - + func.defaultConfig.files.push(pathToJQuery); func.defaultConfig.preprocessors[testFileGlob] = ['browserify']; } +// lastly, load test file glob +func.defaultConfig.files.push(testFileGlob); + module.exports = func; diff --git a/test/jasmine/tests/annotations_test.js b/test/jasmine/tests/annotations_test.js index d6701bd1f71..967b31323eb 100644 --- a/test/jasmine/tests/annotations_test.js +++ b/test/jasmine/tests/annotations_test.js @@ -514,8 +514,8 @@ describe('annotations autorange', function() { function assertRanges(x, y, x2, y2, x3, y3) { var fullLayout = gd._fullLayout; - var PREC = 1; + var PREC = 1; // xaxis2 need a bit more tolerance to pass on CI // this most likely due to the different text bounding box values // on headfull vs headless browsers. @@ -523,6 +523,9 @@ describe('annotations autorange', function() { var PRECX2 = -10; // yaxis2 needs a bit more now too... var PRECY2 = 0.2; + // and xaxis3 too... + var PRECX3 = 0.2; + var dateAx = fullLayout.xaxis2; expect(fullLayout.xaxis.range).toBeCloseToArray(x, PREC, '- xaxis'); @@ -530,7 +533,7 @@ describe('annotations autorange', function() { expect(Lib.simpleMap(dateAx.range, dateAx.r2l)) .toBeCloseToArray(Lib.simpleMap(x2, dateAx.r2l), PRECX2, 'xaxis2 ' + dateAx.range); expect(fullLayout.yaxis2.range).toBeCloseToArray(y2, PRECY2, 'yaxis2'); - expect(fullLayout.xaxis3.range).toBeCloseToArray(x3, PREC, 'xaxis3'); + expect(fullLayout.xaxis3.range).toBeCloseToArray(x3, PRECX3, 'xaxis3'); expect(fullLayout.yaxis3.range).toBeCloseToArray(y3, PREC, 'yaxis3'); } diff --git a/test/jasmine/tests/gl2d_click_test.js b/test/jasmine/tests/gl2d_click_test.js index fe14d214b34..a7d679a09ed 100644 --- a/test/jasmine/tests/gl2d_click_test.js +++ b/test/jasmine/tests/gl2d_click_test.js @@ -4,7 +4,6 @@ var Lib = require('@src/lib'); var createGraphDiv = require('../assets/create_graph_div'); var destroyGraphDiv = require('../assets/destroy_graph_div'); var customMatchers = require('../assets/custom_matchers'); -var hasWebGLSupport = require('../assets/has_webgl_support'); // cartesian click events events use the hover data // from the mousemove events and then simulate @@ -17,9 +16,7 @@ Plotly.register([ require('@lib/contourgl') ]); -describe('Test hover and click interactions', function() { - - if(!hasWebGLSupport('gl2d_click_test')) return; +describe('@noCI Test hover and click interactions', function() { var mock = require('@mocks/gl2d_14.json'); var mock2 = require('@mocks/gl2d_pointcloud-basic.json'); diff --git a/test/jasmine/tests/gl2d_date_axis_render_test.js b/test/jasmine/tests/gl2d_date_axis_render_test.js index 53392de3e76..52d98209e48 100644 --- a/test/jasmine/tests/gl2d_date_axis_render_test.js +++ b/test/jasmine/tests/gl2d_date_axis_render_test.js @@ -1,14 +1,10 @@ var PlotlyInternal = require('@src/plotly'); -var hasWebGLSupport = require('../assets/has_webgl_support'); - var createGraphDiv = require('../assets/create_graph_div'); var destroyGraphDiv = require('../assets/destroy_graph_div'); describe('date axis', function() { - if(!hasWebGLSupport('axes_test date axis')) return; - var gd; beforeEach(function() { diff --git a/test/jasmine/tests/gl_plot_interact_test.js b/test/jasmine/tests/gl_plot_interact_test.js index db4a57922a2..746c9c5474c 100644 --- a/test/jasmine/tests/gl_plot_interact_test.js +++ b/test/jasmine/tests/gl_plot_interact_test.js @@ -11,12 +11,6 @@ var mouseEvent = require('../assets/mouse_event'); var selectButton = require('../assets/modebar_button'); var customMatchers = require('../assets/custom_matchers'); -/* - * WebGL interaction test cases fail on the CircleCI - * most likely due to a WebGL/driver issue - * - */ - var MODEBAR_DELAY = 500; diff --git a/test/jasmine/tests/hover_label_test.js b/test/jasmine/tests/hover_label_test.js index ebfd193ce8c..b8830036e8e 100644 --- a/test/jasmine/tests/hover_label_test.js +++ b/test/jasmine/tests/hover_label_test.js @@ -756,8 +756,8 @@ describe('hover on fill', function() { var transformParts = hoverText.attr('transform').split('('); expect(transformParts[0]).toEqual('translate'); var transformCoords = transformParts[1].split(')')[0].split(','); - expect(+transformCoords[0]).toBeCloseTo(labelPos[0], -1, labelText + ':x'); - expect(+transformCoords[1]).toBeCloseTo(labelPos[1], -1, labelText + ':y'); + expect(+transformCoords[0]).toBeCloseTo(labelPos[0], -1.2, labelText + ':x'); + expect(+transformCoords[1]).toBeCloseTo(labelPos[1], -1.2, labelText + ':y'); resolve(); }, constants.HOVERMINTIME); diff --git a/test/jasmine/tests/mapbox_test.js b/test/jasmine/tests/mapbox_test.js index 61a7e868a9b..d494573ecb2 100644 --- a/test/jasmine/tests/mapbox_test.js +++ b/test/jasmine/tests/mapbox_test.js @@ -7,7 +7,6 @@ var supplyLayoutDefaults = require('@src/plots/mapbox/layout_defaults'); var d3 = require('d3'); var createGraphDiv = require('../assets/create_graph_div'); var destroyGraphDiv = require('../assets/destroy_graph_div'); -var hasWebGLSupport = require('../assets/has_webgl_support'); var mouseEvent = require('../assets/mouse_event'); var customMatchers = require('../assets/custom_matchers'); var failTest = require('../assets/fail_test'); @@ -176,8 +175,6 @@ describe('mapbox defaults', function() { describe('mapbox credentials', function() { 'use strict'; - if(!hasWebGLSupport('mapbox credentials')) return; - var dummyToken = 'asfdsa124331wersdsa1321q3'; var gd; @@ -279,11 +276,9 @@ describe('mapbox credentials', function() { }, LONG_TIMEOUT_INTERVAL); }); -describe('mapbox plots', function() { +describe('@noCI, mapbox plots', function() { 'use strict'; - if(!hasWebGLSupport('mapbox plots')) return; - var mock = require('@mocks/mapbox_0.json'), gd; diff --git a/test/jasmine/tests/parcoords_test.js b/test/jasmine/tests/parcoords_test.js index 5528983a27d..6aa01532634 100644 --- a/test/jasmine/tests/parcoords_test.js +++ b/test/jasmine/tests/parcoords_test.js @@ -7,7 +7,6 @@ var attributes = require('@src/traces/parcoords/attributes'); var createGraphDiv = require('../assets/create_graph_div'); var destroyGraphDiv = require('../assets/destroy_graph_div'); -var hasWebGLSupport = require('../assets/has_webgl_support'); var mouseEvent = require('../assets/mouse_event'); // mock with two dimensions (one panel); special case, e.g. left and right panel is obv. the same @@ -220,9 +219,7 @@ describe('parcoords initialization tests', function() { }); }); -describe('parcoords', function() { - - if(!hasWebGLSupport('parcoords')) return; +describe('@noCI parcoords', function() { beforeAll(function() { mock.data[0].dimensions.forEach(function(d) { diff --git a/test/jasmine/tests/scattermapbox_test.js b/test/jasmine/tests/scattermapbox_test.js index 393bd4f5d27..716af5581a3 100644 --- a/test/jasmine/tests/scattermapbox_test.js +++ b/test/jasmine/tests/scattermapbox_test.js @@ -7,7 +7,6 @@ var convert = require('@src/traces/scattermapbox/convert'); var createGraphDiv = require('../assets/create_graph_div'); var destroyGraphDiv = require('../assets/destroy_graph_div'); -var hasWebGLSupport = require('../assets/has_webgl_support'); var customMatchers = require('../assets/custom_matchers'); Plotly.setPlotConfig({ @@ -452,11 +451,9 @@ describe('scattermapbox convert', function() { } }); -describe('scattermapbox hover', function() { +describe('@noCI scattermapbox hover', function() { 'use strict'; - if(!hasWebGLSupport('scattermapbox hover')) return; - var hoverPoints = ScatterMapbox.hoverPoints; var gd;