C3: Refactor dns polling to improve experience for WARP users (#3729)

* C3: Refactor dns resolution to improve reliability of DNS polling in summary step

Author: jculvey

.changeset/hip-lobsters-sniff.md

@@ -0,0 +1,5 @@
+---
+"create-cloudflare": patch
+---
+
+Improve experience for WARP users by improving the reliability of the polling logic that waits for newly created apps to become available.

package-lock.json

@@ -0,0 +1,11 @@
+import type {DnsAnswer as _DnsAnswer, DnsResponse as _DnsResponse} from 'dns2'
+
+declare module 'dns2' {
+  export interface DnsAnswer extends _DnsAnswer {
+    ns: string;
+  }
+
+  export interface DnsResponse extends _DnsResponse {
+    authorities: DnsAnswer[];
+  }
+}

packages/create-cloudflare/dns2.d.ts

@@ -2,7 +2,6 @@ import { existsSync, mkdtempSync, realpathSync, rmSync } from "fs";
 import crypto from "node:crypto";
 import { tmpdir } from "os";
 import { join } from "path";
-import spawn from "cross-spawn";
 import { FrameworkMap } from "frameworks/index";
 import { readJSON } from "helpers/files";
 import { fetch } from "undici";
@@ -39,29 +38,10 @@ describe(`E2E: Web frameworks`, () => {
 	afterEach((ctx) => {
 		const framework = ctx.meta.name;
 		const projectPath = getProjectPath(framework);
-		const projectName = getProjectName(framework);
 
 		if (existsSync(projectPath)) {
 			rmSync(projectPath, { recursive: true });
 		}
-
-		try {
-			const { output } = spawn.sync("npx", [
-				"wrangler",
-				"pages",
-				"project",
-				"delete",
-				"-y",
-				projectName,
-			]);
-
-			if (!output.toString().includes(`Successfully deleted ${projectName}`)) {
-				console.error(output.toString());
-			}
-		} catch (error) {
-			console.error(`Failed to cleanup project: ${projectName}`);
-			console.error(error);
-		}
 	});
 
 	const runCli = async (

packages/create-cloudflare/e2e-tests/pages.test.ts

@@ -44,6 +44,7 @@
 		"@cloudflare/workers-types": "^4.20230419.0",
 		"@types/command-exists": "^1.2.0",
 		"@types/cross-spawn": "^6.0.2",
+		"@types/dns2": "^2.0.3",
 		"@types/esprima": "^4.0.3",
 		"@types/node": "^18.15.3",
 		"@types/which-pm-runs": "^1.0.0",
@@ -53,6 +54,7 @@
 		"chalk": "^5.2.0",
 		"command-exists": "^1.2.9",
 		"cross-spawn": "^7.0.3",
+		"dns2": "^2.1.0",
 		"esbuild": "^0.17.12",
 		"execa": "^7.1.1",
 		"haikunator": "^2.1.2",

packages/create-cloudflare/package.json

@@ -1,26 +1,63 @@
-import { Resolver } from "node:dns/promises";
+import dns2 from "dns2";
 import { request } from "undici";
 import { blue, brandColor, dim } from "./colors";
 import { spinner } from "./interactive";
+import type { DnsAnswer, DnsResponse } from "dns2";
 
 const TIMEOUT = 1000 * 60 * 5;
 const POLL_INTERVAL = 1000;
 
+/*
+  A helper to wait until the newly deployed domain is available.
+
+  We do this by first polling DNS until the new domain is resolvable, and then polling
+  via HTTP until we get a successful response.
+
+  Note that when polling DNS we make queries against specific nameservers to avoid negative
+  caching. Similarly, we poll via HTTP using the 'no-cache' header for the same reason.
+*/
 export const poll = async (url: string): Promise<boolean> => {
 	const start = Date.now();
 	const domain = new URL(url).host;
 	const s = spinner();
 
 	s.start("Waiting for DNS to propagate");
+
+	// Start out by sleeping for 10 seconds since it's unlikely DNS changes will
+	// have propogated before then
+	await sleep(10 * 1000);
+
+	await pollDns(domain, start, s);
+	if (await pollHttp(url, start, s)) return true;
+
+	s.stop(
+		`${brandColor(
+			"timed out"
+		)} while waiting for ${url} - try accessing it in a few minutes.`
+	);
+	return false;
+};
+
+const pollDns = async (
+	domain: string,
+	start: number,
+	s: ReturnType<typeof spinner>
+) => {
 	while (Date.now() - start < TIMEOUT) {
 		s.update(`Waiting for DNS to propagate (${secondsSince(start)}s)`);
-		if (await dnsLookup(domain)) {
+		if (await isDomainResolvable(domain)) {
 			s.stop(`${brandColor("DNS propagation")} ${dim("complete")}.`);
-			break;
+			return;
 		}
 		await sleep(POLL_INTERVAL);
 	}
+};
 
+const pollHttp = async (
+	url: string,
+	start: number,
+	s: ReturnType<typeof spinner>
+) => {
 	s.start("Waiting for deployment to become available");
 	while (Date.now() - start < TIMEOUT) {
 		s.update(
@@ -45,29 +82,50 @@ export const poll = async (url: string): Promise<boolean> => {
 		}
 		await sleep(POLL_INTERVAL);
 	}
-
-	s.stop(
-		`${brandColor(
-			"timed out"
-		)} while waiting for ${url} - try accessing it in a few minutes.`
-	);
-	return false;
 };
 
-async function dnsLookup(domain: string): Promise<boolean> {
+// Determines if the domain is resolvable via DNS. Until this condition is true,
+// any HTTP requests will result in an NXDOMAIN error.
+export const isDomainResolvable = async (domain: string) => {
 	try {
-		const resolver = new Resolver({ timeout: TIMEOUT, tries: 1 });
-		resolver.setServers([
-			"1.1.1.1",
-			"1.0.0.1",
-			"2606:4700:4700::1111",
-			"2606:4700:4700::1001",
-		]);
-		return (await resolver.resolve4(domain)).length > 0;
-	} catch (e) {
+		const nameServers = await lookupSubdomainNameservers(domain);
+
+		// If the subdomain nameservers aren't resolvable yet, keep polling
+		if (nameServers.length === 0) return false;
+
+		// Once they are resolvable, query these nameservers for the domain's 'A' record
+		const dns = new dns2({ nameServers });
+		const res = await dns.resolve(domain, "A");
+		return res.answers.length > 0;
+	} catch (error) {
 		return false;
 	}
-}
+};
+
+// Looks up the nameservers that are responsible for this particular domain
+export const lookupSubdomainNameservers = async (domain: string) => {
+	const nameServers = await lookupDomainLevelNameservers(domain);
+	const dns = new dns2({ nameServers });
+	const res = (await dns.resolve(domain, "NS")) as DnsResponse;
+
+	return (
+		res.authorities
+			// Filter out non-authoritative authorities (ones that don't have an 'ns' property)
+			.filter((r) => Boolean(r.ns))
+			// Return only the hostnames of the authoritative servers
+			.map((r) => r.ns)
+	);
+};
+
+// Looks up the nameservers responsible for handling `pages.dev` or `workers.dev` domains
+export const lookupDomainLevelNameservers = async (domain: string) => {
+	// Get the last 2 parts of the domain (ie. `pages.dev` or `workers.dev`)
+	const baseDomain = domain.split(".").slice(-2).join(".");
+
+	const dns = new dns2({});
+	const nameservers = await dns.resolve(baseDomain, "NS");
+	return (nameservers.answers as DnsAnswer[]).map((n) => n.ns);
+};
 
 async function sleep(ms: number) {
 	return new Promise((res) => setTimeout(res, ms));