diff --git a/.circleci/test.sh b/.circleci/test.sh index 5bceb403cca..cb309a89850 100755 --- a/.circleci/test.sh +++ b/.circleci/test.sh @@ -75,11 +75,12 @@ case $1 in image) SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | circleci tests split) - npm run test-image -- $SUITE --filter || EXIT_STATE=$? + npm run test-image -- $SUITE --filter --skip-flaky || EXIT_STATE=$? exit $EXIT_STATE ;; image2) + retry npm run test-image -- --just-flaky npm run test-export || EXIT_STATE=$? exit $EXIT_STATE ;; diff --git a/test/image/compare_pixels_test.js b/test/image/compare_pixels_test.js index d7f7b337456..71973e3e86d 100644 --- a/test/image/compare_pixels_test.js +++ b/test/image/compare_pixels_test.js @@ -47,11 +47,13 @@ var QUEUE_WAIT = 10; * Run all gl3d image test in queue: * * npm run test-image -- gl3d_* --queue + * + * */ -var argv = minimist(process.argv.slice(2), {boolean: ['queue', 'filter' ]}); -var isInQueue = argv.queue; -var filter = argv.filter; +var argv = minimist(process.argv.slice(2), { + boolean: ['queue', 'filter', 'skip-flaky', 'just-flaky'] +}); var allMock = false; // If no pattern is provided, all mocks are compared @@ -80,40 +82,39 @@ allMockList = allMockList.filter(unique); // filter out untestable mocks if no pattern is specified (ie. we're testing all mocks) // or if flag '--filter' is provided -if(allMock || filter) { +console.log(''); +if(allMock || argv.filter) { console.log('Filtering out untestable mocks:'); - allMockList = allMockList.filter(untestableFilter); - console.log('\n'); -} - -sortGl2dMockList(allMockList); - -// main -if(isInQueue) { - runInQueue(allMockList); -} else { - runInBatch(allMockList); + // Test cases: + // - font-wishlist + // - all mapbox + // don't behave consistently from run-to-run and/or + // machine-to-machine; skip over them for now. + allMockList = allMockList.filter(function(mockName) { + var cond = !( + mockName === 'font-wishlist' || + mockName.indexOf('mapbox_') !== -1 + ); + if(!cond) console.log(' -', mockName); + return cond; + }); } -/* Test cases: - * - * - font-wishlist - * - all mapbox - * - * don't behave consistently from run-to-run and/or - * machine-to-machine; skip over them for now. - * - */ -function untestableFilter(mockName) { - var cond = - !( - mockName === 'font-wishlist' || - mockName.indexOf('mapbox_') !== -1 - ); - - if(!cond) console.log(' -', mockName); +var FLAKY_LIST = [ + 'treemap_textposition' +]; - return cond; +console.log(''); +if(argv['skip-flaky']) { + allMockList = allMockList.filter(function(mockName) { + var cond = FLAKY_LIST.indexOf(mockName) === -1; + if(!cond) console.log('Skipping flaky mock', mockName); + return cond; + }); +} else if(argv['just-flaky']) { + allMockList = allMockList.filter(function(mockName) { + return FLAKY_LIST.indexOf(mockName) !== -1; + }); } /* gl2d pointcloud and other non-regl gl2d mock(s) @@ -277,3 +278,13 @@ function comparePixels(mockName, cb) { .pipe(saveImageStream) .on('close', checkImage); } + +sortGl2dMockList(allMockList); +console.log(''); + +// main +if(argv.queue) { + runInQueue(allMockList); +} else { + runInBatch(allMockList); +} diff --git a/test/jasmine/tests/titles_test.js b/test/jasmine/tests/titles_test.js index 11811126c9b..9da52ddda25 100644 --- a/test/jasmine/tests/titles_test.js +++ b/test/jasmine/tests/titles_test.js @@ -1141,7 +1141,7 @@ describe('Editable titles', function() { .then(done); }); - it('has no hover effects for titles that used to be blank', function(done) { + it('@flaky has no hover effects for titles that used to be blank', function(done) { Plotly.plot(gd, data, { xaxis: {title: {text: ''}}, yaxis: {title: {text: ''}},