Skip to content

Commit d885fbe

Browse files
jculveyjculvey
authored
C3: Improve e2e testing reliability (#3686)
* C3: Add e2e coverage for pnpm and yarn * Skipping some tests for debug purposes * Fixing env variable passing to e2e process * Change how we force package managers in e2e * Create a job to cleanup e2e pages projects before running * Add required env variables to cleanup job * Consolidate temp directory creation logic * debug cli tmp project creation * Skip react and next for debug * Re-enable all tests * Running tests serially for debug * Fix bug in cleanup * Debug: limit to npm * Update test account in c3 tests * Lower the concurrency to help with rate limiting * Fixing pnpm issues for react and next * Configure dummy git user and limit runs to npm * Add more logging * Parse deployment url from c3 run output. Fix hono expectations
1 parent e17d309 commit d885fbe

File tree

18 files changed

+309
-133
lines changed

18 files changed

+309
-133
lines changed

‎.github/actions/install-dependencies/action.yml

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: "Install Dependencies"
2+
description: "Install dependencies, fetching from cache when possible"
3+
runs:
4+
using: "composite"
5+
steps:
6+
- name: Use Node.js ${{ env.node-version }}
7+
uses: actions/setup-node@v3
8+
with:
9+
node-version: ${{ env.node-version }}
10+
cache: "npm" # cache ~/.npm in case 'npm ci' needs to run
11+
12+
- name: ESlint and Typescript caching
13+
uses: actions/cache@v3
14+
id: eslint-cache
15+
with:
16+
path: |
17+
.eslintcache
18+
tsconfig.tsbuildinfo
19+
key: ${{ matrix.os }}-eslint-tsbuildinfo-${{ hashFiles('**/*.ts','**/*.js', 'package.json', 'tsconfig.json') }}
20+
21+
# Attempt to cache all the node_modules directories based on the OS and package lock.
22+
- name: Cache node_modules
23+
id: npm-cache
24+
uses: actions/cache@v3
25+
env:
26+
cache-name: cache-node-modules
27+
with:
28+
key: ${{ runner.os }}-${{ env.node-version }}-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
29+
path: "**/node_modules"
30+
31+
# If the cache missed then install using `npm ci` to follow package lock exactly
32+
- if: ${{ steps.npm-cache.outputs.cache-hit != 'true'}}
33+
shell: bash
34+
name: Install NPM Dependencies
35+
run: npm ci

‎.github/workflows/test-c3.yml

+42-32
Original file line numberDiff line numberDiff line change
@@ -23,35 +23,8 @@ jobs:
2323
with:
2424
fetch-depth: 0
2525

26-
- name: Use Node.js ${{ env.node-version }}
27-
uses: actions/setup-node@v3
28-
with:
29-
node-version: ${{ env.node-version }}
30-
cache: "npm" # cache ~/.npm in case 'npm ci' needs to run
31-
32-
- name: ESlint and Typescript caching
33-
uses: actions/cache@v3
34-
id: eslint-cache
35-
with:
36-
path: |
37-
.eslintcache
38-
tsconfig.tsbuildinfo
39-
key: ${{ matrix.os }}-eslint-tsbuildinfo-${{ hashFiles('**/*.ts','**/*.js', 'package.json', 'tsconfig.json') }}
40-
41-
# Attempt to cache all the node_modules directories based on the OS and package lock.
42-
- name: Cache node_modules
43-
id: npm-cache
44-
uses: actions/cache@v3
45-
env:
46-
cache-name: cache-node-modules
47-
with:
48-
key: ${{ runner.os }}-${{ env.node-version }}-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
49-
path: "**/node_modules"
50-
51-
# If the cache missed then install using `npm ci` to follow package lock exactly
52-
- if: ${{ steps.npm-cache.outputs.cache-hit != 'true'}}
53-
name: Install NPM Dependencies
54-
run: npm ci
26+
- name: Install Dependencies
27+
uses: ./.github/actions/install-dependencies
5528

5629
- name: Check Types
5730
run: npm run check:type -w create-cloudflare
@@ -62,8 +35,45 @@ jobs:
6235
- name: Unit Tests
6336
run: npm run test:unit -w create-cloudflare
6437

38+
cleanup:
39+
runs-on: ubuntu-latest
40+
steps:
41+
- name: Checkout Repo
42+
uses: actions/checkout@v3
43+
with:
44+
fetch-depth: 0
45+
46+
- name: Install Dependencies
47+
uses: ./.github/actions/install-dependencies
48+
49+
- name: Cleanup E2E test projects
50+
run: npm run test:e2e:cleanup -w create-cloudflare
51+
env:
52+
CLOUDFLARE_API_TOKEN: ${{ secrets.C3_TEST_CLOUDFLARE_API_TOKEN }}
53+
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.C3_TEST_CLOUDFLARE_ACCOUNT_ID }}
54+
55+
e2e:
56+
needs: cleanup
57+
name: "E2E"
58+
strategy:
59+
matrix:
60+
# TODO: add back windows
61+
# os: [ubuntu-latest, windows-latest, macos-latest]
62+
os: [ubuntu-latest]
63+
# pm: [npm, yarn, pnpm]
64+
pm: [npm]
65+
runs-on: ${{ matrix.os }}
66+
steps:
67+
- name: Checkout Repo
68+
uses: actions/checkout@v3
69+
with:
70+
fetch-depth: 0
71+
72+
- name: Install Dependencies
73+
uses: ./.github/actions/install-dependencies
74+
6575
- name: E2E Tests
66-
run: npm run test:e2e -w create-cloudflare
76+
run: npm run test:e2e:${{matrix.pm}} -w create-cloudflare
6777
env:
68-
CLOUDFLARE_API_TOKEN: ${{ secrets.TEST_CLOUDFLARE_API_TOKEN }}
69-
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.TEST_CLOUDFLARE_ACCOUNT_ID }}
78+
CLOUDFLARE_API_TOKEN: ${{ secrets.C3_TEST_CLOUDFLARE_API_TOKEN }}
79+
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.C3_TEST_CLOUDFLARE_ACCOUNT_ID }}

‎package-lock.json

+28-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/create-cloudflare/e2e-tests/helpers.ts

+29-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
1+
import { existsSync, mkdtempSync, realpathSync, rmSync } from "fs";
2+
import crypto from "node:crypto";
3+
import { tmpdir } from "os";
4+
import { join } from "path";
15
import { spawn } from "cross-spawn";
26
import { spinnerFrames } from "helpers/interactive";
37

8+
export const C3_E2E_PREFIX = "c3-e2e-";
9+
410
export const keys = {
511
enter: "\x0d",
612
backspace: "\x7f",
@@ -39,9 +45,9 @@ export const runC3 = async ({
3945

4046
lines.forEach((line) => {
4147
// Uncomment to debug test output
42-
// if (filterLine(line)) {
43-
// console.log(line);
44-
// }
48+
if (filterLine(line)) {
49+
console.log(line);
50+
}
4551
stdout.push(line);
4652

4753
if (currentDialog && currentDialog.matcher.test(line)) {
@@ -89,6 +95,26 @@ export const runC3 = async ({
8995
};
9096
};
9197

98+
export const testProjectDir = (suite: string) => {
99+
const tmpDirPath = realpathSync(
100+
mkdtempSync(join(tmpdir(), `c3-tests-${suite}`))
101+
);
102+
103+
const randomSuffix = crypto.randomBytes(3).toString("hex");
104+
const baseProjectName = `${C3_E2E_PREFIX}${randomSuffix}`;
105+
106+
const getName = (suffix: string) => `${baseProjectName}-${suffix}`;
107+
const getPath = (suffix: string) => join(tmpDirPath, getName(suffix));
108+
const clean = (suffix: string) => {
109+
const path = getPath(suffix);
110+
if (existsSync(path)) {
111+
rmSync(path, { recursive: true, force: true });
112+
}
113+
};
114+
115+
return { getName, getPath, clean };
116+
};
117+
92118
// Removes lines from the output of c3 that aren't particularly useful for debugging tests
93119
export const condenseOutput = (lines: string[]) => {
94120
return lines.filter(filterLine);

‎packages/create-cloudflare/e2e-tests/pages.test.ts

+17-26
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,11 @@
1-
import { existsSync, mkdtempSync, realpathSync, rmSync } from "fs";
2-
import crypto from "node:crypto";
3-
import { tmpdir } from "os";
41
import { join } from "path";
52
import { FrameworkMap } from "frameworks/index";
63
import { readJSON } from "helpers/files";
74
import { fetch } from "undici";
85
import { describe, expect, test, afterEach, beforeEach } from "vitest";
9-
import { keys, runC3 } from "./helpers";
6+
import { keys, runC3, testProjectDir } from "./helpers";
107
import type { RunnerConfig } from "./helpers";
118

12-
export const TEST_PREFIX = "c3-e2e-";
13-
149
/*
1510
Areas for future improvement:
1611
- Add support for frameworks with global installs (like docusaurus, gatsby, etc)
@@ -21,34 +16,23 @@ type FrameworkTestConfig = RunnerConfig & {
2116
};
2217

2318
describe(`E2E: Web frameworks`, () => {
24-
const tmpDirPath = realpathSync(mkdtempSync(join(tmpdir(), "c3-tests")));
25-
const baseProjectName = `c3-e2e-${crypto.randomBytes(3).toString("hex")}`;
26-
27-
const getProjectName = (framework: string) =>
28-
`${baseProjectName}-${framework}`;
29-
const getProjectPath = (framework: string) =>
30-
join(tmpDirPath, getProjectName(framework));
19+
const { getPath, clean } = testProjectDir("pages");
3120

3221
beforeEach((ctx) => {
3322
const framework = ctx.meta.name;
34-
const projectPath = getProjectPath(framework);
35-
rmSync(projectPath, { recursive: true, force: true });
23+
clean(framework);
3624
});
3725

3826
afterEach((ctx) => {
3927
const framework = ctx.meta.name;
40-
const projectPath = getProjectPath(framework);
41-
42-
if (existsSync(projectPath)) {
43-
rmSync(projectPath, { recursive: true });
44-
}
28+
clean(framework);
4529
});
4630

4731
const runCli = async (
4832
framework: string,
4933
{ argv = [], promptHandlers = [], overrides }: RunnerConfig
5034
) => {
51-
const projectPath = getProjectPath(framework);
35+
const projectPath = getPath(framework);
5236

5337
const args = [
5438
projectPath,
@@ -96,19 +80,26 @@ describe(`E2E: Web frameworks`, () => {
9680
};
9781

9882
const runCliWithDeploy = async (framework: string) => {
99-
const projectName = `${baseProjectName}-${framework}`;
100-
10183
const { argv, overrides, promptHandlers, expectResponseToContain } =
10284
frameworkTests[framework];
10385

104-
await runCli(framework, {
86+
const { output } = await runCli(framework, {
10587
overrides,
10688
promptHandlers,
10789
argv: [...(argv ?? []), "--deploy", "--no-git"],
10890
});
10991

11092
// Verify deployment
111-
const projectUrl = `https://${projectName}.pages.dev/`;
93+
const deployedUrlRe =
94+
/deployment is ready at: (https:\/\/.+\.(pages|workers)\.dev)/;
95+
96+
const match = output.match(deployedUrlRe);
97+
if (!match || !match[1]) {
98+
expect(false, "Couldn't find deployment url in C3 output").toBe(true);
99+
return;
100+
}
101+
102+
const projectUrl = match[1];
112103

113104
const res = await fetch(projectUrl);
114105
expect(res.status).toBe(200);
@@ -126,7 +117,7 @@ describe(`E2E: Web frameworks`, () => {
126117
expectResponseToContain: "Hello, Astronaut!",
127118
},
128119
hono: {
129-
expectResponseToContain: "/api/hello",
120+
expectResponseToContain: "Hello Hono!",
130121
},
131122
qwik: {
132123
expectResponseToContain: "Welcome to Qwik",

‎packages/create-cloudflare/e2e-tests/workers.test.ts

+10-12
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,28 @@
1-
import { existsSync, rmSync, mkdtempSync, realpathSync } from "fs";
2-
import { tmpdir } from "os";
31
import { join } from "path";
42
import { describe, expect, test, afterEach, beforeEach } from "vitest";
5-
import { runC3 } from "./helpers";
3+
import { runC3, testProjectDir } from "./helpers";
64

75
/*
86
Areas for future improvement:
97
- Make these actually e2e by verifying that deployment works
10-
- Add support for frameworks with global installs (like docusaurus, gatsby, etc)
118
*/
129

1310
describe("E2E: Workers templates", () => {
14-
const tmpDirPath = realpathSync(mkdtempSync(join(tmpdir(), "workers-tests")));
15-
const projectPath = join(tmpDirPath, "pages-tests");
11+
const { getPath, clean } = testProjectDir("workers");
1612

17-
beforeEach(() => {
18-
rmSync(projectPath, { recursive: true, force: true });
13+
beforeEach((ctx) => {
14+
const template = ctx.meta.name;
15+
clean(template);
1916
});
2017

21-
afterEach(() => {
22-
if (existsSync(projectPath)) {
23-
rmSync(projectPath, { recursive: true });
24-
}
18+
afterEach((ctx) => {
19+
const template = ctx.meta.name;
20+
clean(template);
2521
});
2622

2723
const runCli = async (template: string) => {
24+
const projectPath = getPath(template);
25+
2826
const argv = [
2927
projectPath,
3028
"--type",

0 commit comments

Comments
 (0)