From da539696720c673c67a9f38c6e178f5755fc25aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= Date: Wed, 30 Oct 2019 11:27:18 -0400 Subject: [PATCH 1/4] add --skip-flaky and --just-flaky CLI opts for `text-image` --- test/image/compare_pixels_test.js | 76 +++++++++++++++++-------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/test/image/compare_pixels_test.js b/test/image/compare_pixels_test.js index d7f7b337456..caf35fa1caf 100644 --- a/test/image/compare_pixels_test.js +++ b/test/image/compare_pixels_test.js @@ -47,11 +47,13 @@ var QUEUE_WAIT = 10; * Run all gl3d image test in queue: * * npm run test-image -- gl3d_* --queue + * + * */ -var argv = minimist(process.argv.slice(2), {boolean: ['queue', 'filter' ]}); -var isInQueue = argv.queue; -var filter = argv.filter; +var argv = minimist(process.argv.slice(2), { + boolean: ['queue', 'filter', 'skip-flaky', 'just-flaky'] +}); var allMock = false; // If no pattern is provided, all mocks are compared @@ -80,40 +82,38 @@ allMockList = allMockList.filter(unique); // filter out untestable mocks if no pattern is specified (ie. we're testing all mocks) // or if flag '--filter' is provided -if(allMock || filter) { +console.log(''); +if(allMock || argv.filter) { console.log('Filtering out untestable mocks:'); - allMockList = allMockList.filter(untestableFilter); - console.log('\n'); -} - -sortGl2dMockList(allMockList); - -// main -if(isInQueue) { - runInQueue(allMockList); -} else { - runInBatch(allMockList); + // Test cases: + // - font-wishlist + // - all mapbox + // don't behave consistently from run-to-run and/or + // machine-to-machine; skip over them for now. + allMockList = allMockList.filter(function(mockName) { + var cond = !( + mockName === 'font-wishlist' || + mockName.indexOf('mapbox_') !== -1 + ); + if(!cond) console.log(' -', mockName); + return cond; + }); } -/* Test cases: - * - * - font-wishlist - * - all mapbox - * - * don't behave consistently from run-to-run and/or - * machine-to-machine; skip over them for now. - * - */ -function untestableFilter(mockName) { - var cond = - !( - mockName === 'font-wishlist' || - mockName.indexOf('mapbox_') !== -1 - ); - - if(!cond) console.log(' -', mockName); +var FLAKY_LIST = [ +]; - return cond; +console.log(''); +if(argv['skip-flaky']) { + allMockList = allMockList.filter(function(mockName) { + var cond = FLAKY_LIST.indexOf(mockName) === -1; + if(!cond) console.log('Skipping flaky mock', mockName); + return cond; + }); +} else if(argv['just-flaky']) { + allMockList = allMockList.filter(function(mockName) { + return FLAKY_LIST.indexOf(mockName) !== -1; + }); } /* gl2d pointcloud and other non-regl gl2d mock(s) @@ -277,3 +277,13 @@ function comparePixels(mockName, cb) { .pipe(saveImageStream) .on('close', checkImage); } + +sortGl2dMockList(allMockList); +console.log(''); + +// main +if(argv.queue) { + runInQueue(allMockList); +} else { + runInBatch(allMockList); +} From 0d11c5c64249de7406284d99efa3bdb704a1e382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= Date: Wed, 30 Oct 2019 12:05:08 -0400 Subject: [PATCH 2/4] add `treemap_textposition` to list of flaky mocks ... this list will grow eventually. --- test/image/compare_pixels_test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/image/compare_pixels_test.js b/test/image/compare_pixels_test.js index caf35fa1caf..71973e3e86d 100644 --- a/test/image/compare_pixels_test.js +++ b/test/image/compare_pixels_test.js @@ -101,6 +101,7 @@ if(allMock || argv.filter) { } var FLAKY_LIST = [ + 'treemap_textposition' ]; console.log(''); From 245b38f112ebe78ff4aad3948f7615352826d2fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= Date: Wed, 30 Oct 2019 12:05:37 -0400 Subject: [PATCH 3/4] add skip-flaky to image) run, run --just-flaky in image2) --- .circleci/test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/test.sh b/.circleci/test.sh index 5bceb403cca..cb309a89850 100755 --- a/.circleci/test.sh +++ b/.circleci/test.sh @@ -75,11 +75,12 @@ case $1 in image) SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | circleci tests split) - npm run test-image -- $SUITE --filter || EXIT_STATE=$? + npm run test-image -- $SUITE --filter --skip-flaky || EXIT_STATE=$? exit $EXIT_STATE ;; image2) + retry npm run test-image -- --just-flaky npm run test-export || EXIT_STATE=$? exit $EXIT_STATE ;; From 11bc6259b47eb41e9c869d25f5d36defce02bddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20T=C3=A9treault-Pinard?= Date: Wed, 30 Oct 2019 12:17:39 -0400 Subject: [PATCH 4/4] add `@flaky` to 'hover effects for titles that used to be blank' test --- test/jasmine/tests/titles_test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/jasmine/tests/titles_test.js b/test/jasmine/tests/titles_test.js index 11811126c9b..9da52ddda25 100644 --- a/test/jasmine/tests/titles_test.js +++ b/test/jasmine/tests/titles_test.js @@ -1141,7 +1141,7 @@ describe('Editable titles', function() { .then(done); }); - it('has no hover effects for titles that used to be blank', function(done) { + it('@flaky has no hover effects for titles that used to be blank', function(done) { Plotly.plot(gd, data, { xaxis: {title: {text: ''}}, yaxis: {title: {text: ''}},