C3: Add e2e coverage for deployment (#3658)

* C3: Add e2e coverage for deployment

* PR feedback

* Adding cleanup script for test projects

* C3: Run framework e2e tests concurrently

* Fixing project cleanup and tweaking test concurrency

Author: jculvey

.github/workflows/test-c3.yml

@@ -64,3 +64,6 @@ jobs:
 
       - name: E2E Tests
         run: npm run test:e2e -w create-cloudflare
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.TEST_CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.TEST_CLOUDFLARE_ACCOUNT_ID }}

packages/create-cloudflare/e2e-tests/helpers.ts

@@ -1,4 +1,5 @@
 import { spawn } from "cross-spawn";
+import { spinnerFrames } from "helpers/interactive";
 
 export const keys = {
 	enter: "\x0d",
@@ -37,6 +38,10 @@ export const runC3 = async ({
 			const currentDialog = promptHandlers[0];
 
 			lines.forEach((line) => {
+				// Uncomment to debug test output
+				// if (filterLine(line)) {
+				// 	console.log(line);
+				// }
 				stdout.push(line);
 
 				if (currentDialog && currentDialog.matcher.test(line)) {
@@ -64,17 +69,39 @@ export const runC3 = async ({
 			if (code === 0) {
 				resolve(null);
 			} else {
+				console.log(stderr.join("\n").trim());
 				rejects(code);
 			}
 		});
 
-		proc.on("error", (err) => {
-			rejects(err);
+		proc.on("error", (exitCode) => {
+			rejects({
+				exitCode,
+				output: condenseOutput(stdout).join("\n").trim(),
+				errors: stderr.join("\n").trim(),
+			});
 		});
 	});
 
 	return {
-		output: stdout.join("\n").trim(),
+		output: condenseOutput(stdout).join("\n").trim(),
 		errors: stderr.join("\n").trim(),
 	};
 };
+
+// Removes lines from the output of c3 that aren't particularly useful for debugging tests
+export const condenseOutput = (lines: string[]) => {
+	return lines.filter(filterLine);
+};
+
+const filterLine = (line: string) => {
+	// Remove all lines with spinners
+	for (const frame of spinnerFrames) {
+		if (line.includes(frame)) return false;
+	}
+
+	// Remove empty lines
+	if (line.replace(/\s/g, "").length == 0) return false;
+
+	return true;
+};

packages/create-cloudflare/e2e-tests/pages.test.ts

@@ -1,63 +1,95 @@
 import { existsSync, mkdtempSync, realpathSync, rmSync } from "fs";
+import crypto from "node:crypto";
 import { tmpdir } from "os";
 import { join } from "path";
+import spawn from "cross-spawn";
 import { FrameworkMap } from "frameworks/index";
 import { readJSON } from "helpers/files";
+import { fetch } from "undici";
 import { describe, expect, test, afterEach, beforeEach } from "vitest";
 import { keys, runC3 } from "./helpers";
 import type { RunnerConfig } from "./helpers";
 
+export const TEST_PREFIX = "c3-e2e-";
+
 /*
 Areas for future improvement:
-- Make these actually e2e by verifying that deployment works
 - Add support for frameworks with global installs (like docusaurus, gatsby, etc)
 */
 
-describe("E2E: Web frameworks", () => {
+type FrameworkTestConfig = RunnerConfig & {
+	expectResponseToContain: string;
+};
+
+describe(`E2E: Web frameworks`, () => {
 	const tmpDirPath = realpathSync(mkdtempSync(join(tmpdir(), "c3-tests")));
-	const projectPath = join(tmpDirPath, "pages-tests");
+	const baseProjectName = `c3-e2e-${crypto.randomBytes(3).toString("hex")}`;
 
-	beforeEach(() => {
+	const getProjectName = (framework: string) =>
+		`${baseProjectName}-${framework}`;
+	const getProjectPath = (framework: string) =>
+		join(tmpDirPath, getProjectName(framework));
+
+	beforeEach((ctx) => {
+		const framework = ctx.meta.name;
+		const projectPath = getProjectPath(framework);
 		rmSync(projectPath, { recursive: true, force: true });
 	});
 
-	afterEach(() => {
+	afterEach((ctx) => {
+		const framework = ctx.meta.name;
+		const projectPath = getProjectPath(framework);
+		const projectName = getProjectName(framework);
+
 		if (existsSync(projectPath)) {
 			rmSync(projectPath, { recursive: true });
 		}
+
+		try {
+			const { output } = spawn.sync("npx", [
+				"wrangler",
+				"pages",
+				"project",
+				"delete",
+				"-y",
+				projectName,
+			]);
+
+			if (!output.toString().includes(`Successfully deleted ${projectName}`)) {
+				console.error(output.toString());
+			}
+		} catch (error) {
+			console.error(`Failed to cleanup project: ${projectName}`);
+			console.error(error);
+		}
 	});
 
 	const runCli = async (
 		framework: string,
-		{ argv = [], promptHandlers = [], overrides = {} }: RunnerConfig
+		{ argv = [], promptHandlers = [], overrides }: RunnerConfig
 	) => {
+		const projectPath = getProjectPath(framework);
+
 		const args = [
 			projectPath,
 			"--type",
 			"webFramework",
 			"--framework",
 			framework,
-			"--no-deploy",
+			"--deploy",
+			"--no-open",
 		];
 
-		if (argv.length > 0) {
-			args.push(...argv);
-		} else {
-			args.push("--no-git");
-		}
-
-		// For debugging purposes, uncomment the following to see the exact
-		// command the test uses. You can then run this via the command line.
-		// console.log("COMMAND: ", `node ${["./dist/cli.js", ...args].join(" ")}`);
+		args.push(...argv);
 
-		await runC3({ argv: args, promptHandlers });
+		const { output } = await runC3({ argv: args, promptHandlers });
 
 		// Relevant project files should have been created
 		expect(projectPath).toExist();
-
 		const pkgJsonPath = join(projectPath, "package.json");
 		expect(pkgJsonPath).toExist();
 
+		// Wrangler should be installed
 		const wranglerPath = join(projectPath, "node_modules/wrangler");
 		expect(wranglerPath).toExist();
 
@@ -68,7 +100,7 @@ describe("E2E: Web frameworks", () => {
 			...frameworkConfig.packageScripts,
 		} as Record<string, string>;
 
-		if (overrides.packageScripts) {
+		if (overrides && overrides.packageScripts) {
 			// override packageScripts with testing provided scripts
 			Object.entries(overrides.packageScripts).forEach(([target, cmd]) => {
 				frameworkTargetPackageScripts[target] = cmd;
@@ -79,46 +111,77 @@ describe("E2E: Web frameworks", () => {
 		Object.entries(frameworkTargetPackageScripts).forEach(([target, cmd]) => {
 			expect(pkgJson.scripts[target]).toEqual(cmd);
 		});
+
+		return { output };
 	};
 
-	test.each(["astro", "hono", "react", "remix", "vue"])("%s", async (name) => {
-		await runCli(name, {});
-	});
+	const runCliWithDeploy = async (framework: string) => {
+		const projectName = `${baseProjectName}-${framework}`;
 
-	test("Nuxt", async () => {
-		await runCli("nuxt", {
-			overrides: {
-				packageScripts: {
-					build: "NITRO_PRESET=cloudflare-pages nuxt build",
-				},
-			},
+		const { argv, overrides, promptHandlers, expectResponseToContain } =
+			frameworkTests[framework];
+
+		await runCli(framework, {
+			overrides,
+			promptHandlers,
+			argv: [...(argv ?? []), "--deploy", "--no-git"],
 		});
-	});
 
-	test("next", async () => {
-		await runCli("next", {
+		// Verify deployment
+		const projectUrl = `https://${projectName}.pages.dev/`;
+
+		const res = await fetch(projectUrl);
+		expect(res.status).toBe(200);
+
+		const body = await res.text();
+		expect(
+			body,
+			`(${framework}) Deployed page (${projectUrl}) didn't contain expected string: "${expectResponseToContain}"`
+		).toContain(expectResponseToContain);
+	};
+
+	// These are ordered based on speed and reliability for ease of debugging
+	const frameworkTests: Record<string, FrameworkTestConfig> = {
+		astro: {
+			expectResponseToContain: "Hello, Astronaut!",
+		},
+		hono: {
+			expectResponseToContain: "/api/hello",
+		},
+		qwik: {
+			expectResponseToContain: "Welcome to Qwik",
 			promptHandlers: [
 				{
-					matcher: /Do you want to use the next-on-pages eslint-plugin\?/,
-					input: ["y"],
+					matcher: /Yes looks good, finish update/,
+					input: [keys.enter],
 				},
 			],
-		});
-	});
-
-	test("qwik", async () => {
-		await runCli("qwik", {
+		},
+		remix: {
+			expectResponseToContain: "Welcome to Remix",
+		},
+		next: {
+			expectResponseToContain: "Create Next App",
 			promptHandlers: [
 				{
-					matcher: /Yes looks good, finish update/,
-					input: [keys.enter],
+					matcher: /Do you want to use the next-on-pages eslint-plugin\?/,
+					input: ["y"],
 				},
 			],
-		});
-	});
-
-	test("solid", async () => {
-		await runCli("solid", {
+		},
+		nuxt: {
+			expectResponseToContain: "Welcome to Nuxt!",
+			overrides: {
+				packageScripts: {
+					build: "NITRO_PRESET=cloudflare-pages nuxt build",
+				},
+			},
+		},
+		react: {
+			expectResponseToContain: "React App",
+		},
+		solid: {
+			expectResponseToContain: "Hello world",
 			promptHandlers: [
 				{
 					matcher: /Which template do you want to use/,
@@ -133,11 +196,9 @@ describe("E2E: Web frameworks", () => {
 					input: [keys.enter],
 				},
 			],
-		});
-	});
-
-	test("svelte", async () => {
-		await runCli("svelte", {
+		},
+		svelte: {
+			expectResponseToContain: "SvelteKit app",
 			promptHandlers: [
 				{
 					matcher: /Which Svelte app template/,
@@ -152,19 +213,21 @@ describe("E2E: Web frameworks", () => {
 					input: [keys.enter],
 				},
 			],
-		});
-	});
+		},
+		vue: {
+			expectResponseToContain: "Vite App",
+		},
+	};
+
+	test.concurrent.each(Object.keys(frameworkTests))(
+		"%s",
+		async (name) => {
+			await runCliWithDeploy(name);
+		},
+		{ retry: 3 }
+	);
 
-	// This test blows up in CI due to Github providing an unusual git user email address.
-	// E.g.
-	// ```
-	// fatal: empty ident name (for <[email protected].
-	// internal.cloudapp.net>) not allowed
-	// ```
 	test.skip("Hono (wrangler defaults)", async () => {
 		await runCli("hono", { argv: ["--wrangler-defaults"] });
-
-		// verify that wrangler-defaults defaults to `true` for using git
-		expect(join(projectPath, ".git")).toExist();
 	});
 });

packages/create-cloudflare/scripts/cleanupPagesProject.mjs

@@ -0,0 +1,47 @@
+import { execa } from "execa";
+
+if (!process.env.CLOUDFLARE_API_TOKEN) {
+	console.error("CLOUDFLARE_API_TOKEN must be set");
+	process.exit(1);
+}
+
+if (!process.env.CLOUDFLARE_ACCOUNT_ID) {
+	console.error("CLOUDFLARE_ACCOUNT_ID must be set");
+	process.exit(1);
+}
+
+const npx = async (args) => {
+	const argv = args.split(" ");
+	return execa("npx", argv);
+};
+
+const listProjectsToDelete = async () => {
+	const toDelete = [];
+
+	const { stdout } = await npx("wrangler pages project list");
+
+	for (const line of stdout.split("\n")) {
+		const c3ProjectRe = /(c3-e2e-[\w-]*)\s?│/;
+		const match = line.match(c3ProjectRe);
+
+		if (match) {
+			toDelete.push(match[1]);
+		}
+	}
+
+	return toDelete;
+};
+
+const deleteProjects = async (projects) => {
+	for (const project of projects) {
+		try {
+			console.log(`Deleting project: ${project}`);
+			await npx(`wrangler pages project delete -y ${project}`);
+		} catch (error) {
+			console.error(error);
+		}
+	}
+};
+
+const projects = await listProjectsToDelete();
+deleteProjects(projects);

packages/create-cloudflare/src/common.ts

@@ -93,7 +93,7 @@ export const runDeploy = async (ctx: PagesGeneratorContext) => {
 	const result = await runCommand(deployCmd, {
 		silent: true,
 		cwd: ctx.project.path,
-		env: { CLOUDFLARE_ACCOUNT_ID: ctx.account.id },
+		env: { CLOUDFLARE_ACCOUNT_ID: ctx.account.id, NODE_ENV: "production" },
 		startText: `Deploying your application`,
 		doneText: `${brandColor("deployed")} ${dim(`via \`${deployCmd}\``)}`,
 	});