test: separate test vs test setup execution

Author: jbedard

tests/legacy-cli/e2e_runner.ts

@@ -98,10 +98,9 @@ const tests = allTests.filter((name) => {
 });
 
 // Remove tests that are not part of this shard.
-const shardedTests = tests.filter((name, i) => shardId === null || i % nbShards == shardId);
-const testsToRun = allSetups.concat(shardedTests);
+const testsToRun = tests.filter((name, i) => shardId === null || i % nbShards == shardId);
 
-if (shardedTests.length === 0) {
+if (testsToRun.length === 0) {
   console.log(`No tests would be ran, aborting.`);
   process.exit(1);
 }
@@ -114,7 +113,7 @@ console.log(testsToRun.join('\n'));
 if (testsToRun.length == allTests.length) {
   console.log(`Running ${testsToRun.length} tests`);
 } else {
-  console.log(`Running ${testsToRun.length} tests (${allTests.length + allSetups.length} total)`);
+  console.log(`Running ${testsToRun.length} tests (${allTests.length} total)`);
 }
 
 setGlobalVariable('argv', argv);
@@ -131,8 +130,16 @@ Promise.all([findFreePort(), findFreePort()]).then(async ([httpPort, httpsPort])
   let lastTestRun: string | null = null;
 
   try {
+    for (const [setupIndex, setup] of allSetups.entries()) {
+      printSetupHeader(setup, setupIndex, allSetups.length);
+
+      await runSetup((lastTestRun = setup));
+    }
+
     for (const [testIndex, test] of testsToRun.entries()) {
-      await runTest((lastTestRun = test), testIndex);
+      printTestHeader(test, testIndex, testsToRun.length);
+
+      await runTest((lastTestRun = test));
     }
 
     console.log(colors.green('Done.'));
@@ -162,7 +169,7 @@ Promise.all([findFreePort(), findFreePort()]).then(async ([httpPort, httpsPort])
 
 function normalizeTestStep(relativeName: string) {
   // Make sure this is a windows compatible path.
-  let absoluteName = path.join(e2eRoot, relativeName);
+  let absoluteName = path.join(e2eRoot, relativeName).replace(/\.ts$/, '');
   if (/^win/.test(process.platform)) {
     absoluteName = absoluteName.replace(/\\/g, path.posix.sep);
   }
@@ -172,7 +179,18 @@ function normalizeTestStep(relativeName: string) {
   return { absoluteName, currentFileName };
 }
 
-async function runTest(relativeName: string, testIndex: number) {
+async function runSetup(relativeName: string) {
+  const { absoluteName, currentFileName } = normalizeTestStep(relativeName);
+  const start = +new Date();
+
+  const module = require(absoluteName);
+
+  await (typeof module === 'function' ? module : module.default)();
+
+  printFooter(currentFileName, start);
+}
+
+async function runTest(relativeName: string) {
   const { absoluteName, currentFileName } = normalizeTestStep(relativeName);
   const start = +new Date();
 
@@ -190,8 +208,6 @@ async function runTest(relativeName: string, testIndex: number) {
           throw new Error('Invalid test module.');
         };
 
-  printHeader(currentFileName, testIndex);
-
   let clean = true;
   let previousDir = process.cwd();
   try {
@@ -205,19 +221,18 @@ async function runTest(relativeName: string, testIndex: number) {
 
     console.log('----');
 
-    // If we're not in a setup, change the directory back to where it was before the test.
+    // Change the directory back to where it was before the test.
     // This allows tests to chdir without worrying about keeping the original directory.
-    if (!allSetups.includes(relativeName) && previousDir) {
+    if (previousDir) {
       process.chdir(previousDir);
 
       // Restore env variables before each test.
       console.log('  Restoring original environment variables...');
       process.env = originalEnvVariables;
     }
 
-    // Only clean after a real test, not a setup step. Also skip cleaning if the test
-    // requested an exception.
-    if (!allSetups.includes(relativeName) && clean) {
+    // Skip cleaning if the test requested an exception.
+    if (clean) {
       logStack.push(new logging.NullLogger());
       try {
         await gitClean();
@@ -234,19 +249,24 @@ async function runTest(relativeName: string, testIndex: number) {
   }
 }
 
-function printHeader(testName: string, testIndex: number) {
-  const text = `${testIndex + 1} of ${testsToRun.length}`;
-  const fullIndex =
-    (testIndex < allSetups.length
-      ? testIndex
-      : (testIndex - allSetups.length) * nbShards + shardId + allSetups.length) + 1;
-  const length = tests.length + allSetups.length;
+function printSetupHeader(setupName: string, setupIndex: number, count: number) {
+  const text = `${setupIndex + 1} of ${count}`;
+  console.log(
+    colors.green(`Running setup "${colors.bold.blue(setupName)}" (${colors.bold.white(text)})...`),
+  );
+}
+
+function printTestHeader(testName: string, testIndex: number, count: number) {
+  const text = `${testIndex + 1} of ${count}`;
+  const fullIndex = testIndex * nbShards + shardId + 1;
   const shard =
     shardId === null
       ? ''
-      : colors.yellow(` [${shardId}:${nbShards}]` + colors.bold(` (${fullIndex}/${length})`));
+      : colors.yellow(` [${shardId}:${nbShards}]` + colors.bold(` (${fullIndex}/${tests.length})`));
   console.log(
-    colors.green(`Running "${colors.bold.blue(testName)}" (${colors.bold.white(text)}${shard})...`),
+    colors.green(
+      `Running test "${colors.bold.blue(testName)}" (${colors.bold.white(text)}${shard})...`,
+    ),
   );
 }