fix: Retry deployment errors in wrangler pages publish (#3758)

Occasionally, creating a deployment can fail due to internal errors in
the POST /deployments API call. Rather than failing the deployment, this
will retry first.

Author: jahands

.changeset/hip-rules-cross.md

@@ -0,0 +1,7 @@
+---
+"wrangler": patch
+---
+
+fix: Retry deployment errors in wrangler pages publish
+
+This will improve reliability when deploying to Cloudflare Pages

packages/wrangler/src/__tests__/pages/deploy.test.ts

@@ -231,7 +231,7 @@ describe("deployment create", () => {
 							success: false,
 							errors: [
 								{
-									code: 800000,
+									code: 8000000,
 									message: "Something exploded, please retry",
 								},
 							],
@@ -298,6 +298,169 @@ describe("deployment create", () => {
 
 		await runWrangler("pages deploy . --project-name=foo");
 
+		// Should be 2 attempts to upload
+		expect(requests.length).toBe(2);
+
+		expect(normalizeProgressSteps(std.out)).toMatchInlineSnapshot(`
+		"✨ Success! Uploaded 1 files (TIMINGS)
+
+		✨ Deployment complete! Take a peek over at https://abcxyz.foo.pages.dev/"
+	`);
+	});
+
+	it("should retry POST /deployments", async () => {
+		writeFileSync("logo.txt", "foobar");
+
+		mockGetUploadTokenRequest(
+			"<<funfetti-auth-jwt>>",
+			"some-account-id",
+			"foo"
+		);
+
+		// Accumulate multiple requests then assert afterwards
+		const requests: RestRequest[] = [];
+		msw.use(
+			rest.post("*/pages/assets/check-missing", async (req, res, ctx) => {
+				const body = await req.json();
+
+				expect(req.headers.get("Authorization")).toBe(
+					"Bearer <<funfetti-auth-jwt>>"
+				);
+				expect(body).toMatchObject({
+					hashes: ["1a98fb08af91aca4a7df1764a2c4ddb0"],
+				});
+
+				return res.once(
+					ctx.status(200),
+					ctx.json({
+						success: true,
+						errors: [],
+						messages: [],
+						result: body.hashes,
+					})
+				);
+			}),
+			rest.post("*/pages/assets/upload", async (req, res, ctx) => {
+				expect(req.headers.get("Authorization")).toBe(
+					"Bearer <<funfetti-auth-jwt>>"
+				);
+				expect(await req.json()).toMatchObject([
+					{
+						key: "1a98fb08af91aca4a7df1764a2c4ddb0",
+						value: Buffer.from("foobar").toString("base64"),
+						metadata: {
+							contentType: "text/plain",
+						},
+						base64: true,
+					},
+				]);
+
+				return res(
+					ctx.status(200),
+					ctx.json({
+						success: true,
+						errors: [],
+						messages: [],
+						result: null,
+					})
+				);
+			}),
+			rest.post(
+				"*/accounts/:accountId/pages/projects/foo/deployments",
+				async (req, res, ctx) => {
+					requests.push(req);
+					expect(req.params.accountId).toEqual("some-account-id");
+					expect(await (req as RestRequestWithFormData).formData())
+						.toMatchInlineSnapshot(`
+				      FormData {
+				        Symbol(state): Array [
+				          Object {
+				            "name": "manifest",
+				            "value": "{\\"/logo.txt\\":\\"1a98fb08af91aca4a7df1764a2c4ddb0\\"}",
+				          },
+				        ],
+				      }
+			    `);
+
+					if (requests.length < 2) {
+						return res(
+							ctx.status(500),
+							ctx.json({
+								success: false,
+								errors: [
+									{
+										code: 8000000,
+										message: "Something exploded, please retry",
+									},
+								],
+								messages: [],
+								result: null,
+							})
+						);
+					} else {
+						return res.once(
+							ctx.status(200),
+							ctx.json({
+								success: true,
+								errors: [],
+								messages: [],
+								result: { url: "https://abcxyz.foo.pages.dev/" },
+							})
+						);
+					}
+				}
+			),
+			rest.post(
+				"*/accounts/:accountId/pages/projects/foo/deployments",
+				async (req, res, ctx) => {
+					requests.push(req);
+					expect(req.params.accountId).toEqual("some-account-id");
+					expect(await (req as RestRequestWithFormData).formData())
+						.toMatchInlineSnapshot(`
+				      FormData {
+				        Symbol(state): Array [
+				          Object {
+				            "name": "manifest",
+				            "value": "{\\"/logo.txt\\":\\"1a98fb08af91aca4a7df1764a2c4ddb0\\"}",
+				          },
+				        ],
+				      }
+			    `);
+
+					return res.once(
+						ctx.status(200),
+						ctx.json({
+							success: true,
+							errors: [],
+							messages: [],
+							result: { url: "https://abcxyz.foo.pages.dev/" },
+						})
+					);
+				}
+			),
+			rest.get(
+				"*/accounts/:accountId/pages/projects/foo",
+				async (req, res, ctx) => {
+					expect(req.params.accountId).toEqual("some-account-id");
+
+					return res.once(
+						ctx.status(200),
+						ctx.json({
+							success: true,
+							errors: [],
+							messages: [],
+							result: { deployment_configs: { production: {}, preview: {} } },
+						})
+					);
+				}
+			)
+		);
+
+		await runWrangler("pages deploy . --project-name=foo");
+
+		// Should be 2 attempts to POST /deployments
+		expect(requests.length).toBe(2);
+
 		expect(normalizeProgressSteps(std.out)).toMatchInlineSnapshot(`
 		"✨ Success! Uploaded 1 files (TIMINGS)
 

packages/wrangler/src/api/pages/deploy.tsx

@@ -22,6 +22,7 @@ import { validate } from "../../pages/validate";
 import { createUploadWorkerBundleContents } from "./create-worker-bundle-contents";
 import type { BundleResult } from "../../deployment-bundle/bundle";
 import type { Project, Deployment } from "@cloudflare/types";
+import { MAX_DEPLOYMENT_ATTEMPTS } from "../../pages/constants";
 
 interface PagesDeployOptions {
 	/**
@@ -363,12 +364,35 @@ export async function deploy({
 		}
 	}
 
-	const deploymentResponse = await fetchResult<Deployment>(
-		`/accounts/${accountId}/pages/projects/${projectName}/deployments`,
-		{
-			method: "POST",
-			body: formData,
+	let attempts = 0;
+	let lastErr: unknown;
+	while (attempts < MAX_DEPLOYMENT_ATTEMPTS) {
+		try {
+			const deploymentResponse = await fetchResult<Deployment>(
+				`/accounts/${accountId}/pages/projects/${projectName}/deployments`,
+				{
+					method: "POST",
+					body: formData,
+				}
+			);
+			return deploymentResponse;
+		} catch (e) {
+			lastErr = e;
+			if (
+				(e as { code: number }).code === 8000000 &&
+				attempts < MAX_DEPLOYMENT_ATTEMPTS
+			) {
+				logger.debug("failed:", e, "retrying...");
+				// Exponential backoff, 1 second first time, then 2 second, then 4 second etc.
+				await new Promise((resolvePromise) =>
+					setTimeout(resolvePromise, Math.pow(2, attempts++) * 1000)
+				);
+			} else {
+				logger.debug("failed:", e);
+				throw e;
+			}
 		}
-	);
-	return deploymentResponse;
+	}
+	// We should never make it here, but just in case
+	throw lastErr;
 }

packages/wrangler/src/pages/constants.ts

@@ -7,6 +7,7 @@ export const MAX_BUCKET_SIZE = 50 * 1024 * 1024;
 export const MAX_BUCKET_FILE_COUNT = 5000;
 export const BULK_UPLOAD_CONCURRENCY = 3;
 export const MAX_UPLOAD_ATTEMPTS = 5;
+export const MAX_DEPLOYMENT_ATTEMPTS = 3;
 export const MAX_CHECK_MISSING_ATTEMPTS = 5;
 export const SECONDS_TO_WAIT_FOR_PROXY = 5;
 export const isInPagesCI = !!process.env.CF_PAGES;