[installer] Add upgrade test werft flag

Author: nandajavarma

.werft/build.ts

@@ -1,58 +1,64 @@
-import * as fs from 'fs';
-import { SpanStatusCode } from '@opentelemetry/api';
-import { Werft } from './util/werft';
-import { reportBuildFailureInSlack } from './util/slack';
-import * as Tracing from './observability/tracing'
-import * as VM from './vm/vm'
-import { buildAndPublish } from './jobs/build/build-and-publish';
-import { validateChanges } from './jobs/build/validate-changes';
-import { prepare } from './jobs/build/prepare';
-import { deployToPreviewEnvironment } from './jobs/build/deploy-to-preview-environment';
-import { triggerIntegrationTests } from './jobs/build/trigger-integration-tests';
-import { jobConfig } from './jobs/build/job-config';
-import { typecheckWerftJobs } from './jobs/build/typecheck-werft-jobs';
+import * as fs from "fs";
+import { SpanStatusCode } from "@opentelemetry/api";
+import { Werft } from "./util/werft";
+import { reportBuildFailureInSlack } from "./util/slack";
+import * as Tracing from "./observability/tracing";
+import * as VM from "./vm/vm";
+import { buildAndPublish } from "./jobs/build/build-and-publish";
+import { validateChanges } from "./jobs/build/validate-changes";
+import { prepare } from "./jobs/build/prepare";
+import { deployToPreviewEnvironment } from "./jobs/build/deploy-to-preview-environment";
+import { triggerIntegrationTests } from "./jobs/build/trigger-integration-tests";
+import { triggerUpgradeTests } from "./jobs/build/self-hosted-upgrade-tests";
+import { jobConfig } from "./jobs/build/job-config";
+import { typecheckWerftJobs } from "./jobs/build/typecheck-werft-jobs";
 
 // Will be set once tracing has been initialized
-let werft: Werft
-const context: any = JSON.parse(fs.readFileSync('context.json').toString());
+let werft: Werft;
+const context: any = JSON.parse(fs.readFileSync("context.json").toString());
 
 Tracing.initialize()
     .then(() => {
-        werft = new Werft("build")
+        werft = new Werft("build");
     })
     .then(() => run(context))
     .catch((err) => {
         werft.rootSpan.setStatus({
             code: SpanStatusCode.ERROR,
-            message: err
-        })
+            message: err,
+        });
 
-        console.log('Error', err)
+        console.log("Error", err);
 
         if (context.Repository.ref === "refs/heads/main") {
             reportBuildFailureInSlack(context, err).catch((error: Error) => {
-                console.error("Failed to send message to Slack", error)
+                console.error("Failed to send message to Slack", error);
             });
         }
 
         // Explicitly not using process.exit as we need to flush tracing, see tracing.js
-        process.exitCode = 1
+        process.exitCode = 1;
     })
     .finally(() => {
-        werft.phase("Stop kubectl port forwards", "Stopping kubectl port forwards")
-        VM.stopKubectlPortForwards()
+        werft.phase("Stop kubectl port forwards", "Stopping kubectl port forwards");
+        VM.stopKubectlPortForwards();
 
-        werft.phase("Flushing telemetry", "Flushing telemetry before stopping job")
-        werft.endAllSpans()
-    })
+        werft.phase("Flushing telemetry", "Flushing telemetry before stopping job");
+        werft.endAllSpans();
+    });
 
 async function run(context: any) {
-    const config = jobConfig(werft, context)
+    const config = jobConfig(werft, context);
 
-    await validateChanges(werft, config)
-    await prepare(werft, config)
-    await typecheckWerftJobs(werft)
-    await buildAndPublish(werft, config)
+    await validateChanges(werft, config);
+    await prepare(werft, config);
+    if (config.withUpgradeTests) {
+        // this will trigger an upgrade test on a self-hosted gitpod instance on a new cluster.
+        await triggerUpgradeTests(werft, config, context.Owner);
+        return;
+    }
+    await typecheckWerftJobs(werft);
+    await buildAndPublish(werft, config);
 
     if (!config.withPreview || config.publishRelease) {
         werft.phase("deploy", "not deploying");
@@ -61,15 +67,15 @@ async function run(context: any) {
     }
 
     try {
-        await deployToPreviewEnvironment(werft, config)
+        await deployToPreviewEnvironment(werft, config);
     } catch (e) {
         // We currently don't support concurrent deployments to the same preview environment.
         // Until we do we don't want errors to mark the main build as failed.
         if (config.mainBuild) {
-            return
+            return;
         }
-        throw e
+        throw e;
     }
 
-    await triggerIntegrationTests(werft, config, context.Owner)
+    await triggerIntegrationTests(werft, config, context.Owner);
 }

.werft/installer-tests.ts

@@ -4,7 +4,10 @@ import { Werft } from "./util/werft";
 
 const testConfig: string = process.argv.length > 2 ? process.argv[2] : "STANDARD_K3S_TEST";
 // we can provide the version of the gitpod to install (eg: 2022.4.2)
-const version: string = process.argv.length > 3 ? process.argv[3] : "";
+// "-" is the default value which will install the latest version
+const version: string = process.argv.length > 3 ? process.argv[3] : "-";
+
+const channel: string = process.argv.length > 4 ? process.argv[4] : "unstable";
 
 const makefilePath: string = join("install/tests");
 
@@ -16,6 +19,78 @@ interface InfraConfig {
     description: string;
 }
 
+interface TestConfig {
+    DESCRIPTION: string;
+    PHASES: string[];
+    CLOUD: string;
+}
+
+// Each of the TEST_CONFIGURATIONS define an integration test end-to-end
+// It should be a combination of multiple INFRA_PHASES, order of PHASES slice is important
+const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
+    STANDARD_GKE_TEST: {
+        CLOUD: "gcp",
+        DESCRIPTION: "Deploy Gitpod on GKE, with managed DNS, and run integration tests",
+        PHASES: [
+            "STANDARD_GKE_CLUSTER",
+            "CERT_MANAGER",
+            "GCP_MANAGED_DNS",
+            "GENERATE_KOTS_CONFIG",
+            "INSTALL_GITPOD",
+            "CHECK_INSTALLATION",
+            "RUN_INTEGRATION_TESTS",
+            "RESULTS",
+            "DESTROY",
+        ],
+    },
+    STANDARD_GKE_UPGRADE_TEST: {
+        CLOUD: "gcp",
+        DESCRIPTION: `Deploy Gitpod on GKE, and test upgrade from ${version} to latest version`,
+        PHASES: [
+            "STANDARD_GKE_CLUSTER",
+            "CERT_MANAGER",
+            "GCP_MANAGED_DNS",
+            "GENERATE_KOTS_CONFIG",
+            "INSTALL_GITPOD",
+            "CHECK_INSTALLATION",
+            "KOTS_UPGRADE",
+            "CHECK_INSTALLATION",
+            "DESTROY",
+        ],
+    },
+    STANDARD_K3S_TEST: {
+        CLOUD: "gcp", // the cloud provider is still GCP
+        DESCRIPTION:
+            "Deploy Gitpod on a K3s cluster, created on a GCP instance," +
+            " with managed DNS and run integrations tests",
+        PHASES: [
+            "STANDARD_K3S_CLUSTER_ON_GCP",
+            "CERT_MANAGER",
+            "GENERATE_KOTS_CONFIG",
+            "INSTALL_GITPOD",
+            "CHECK_INSTALLATION",
+            "RUN_INTEGRATION_TESTS",
+            "RESULTS",
+            "DESTROY",
+        ],
+    },
+    STANDARD_K3S_PREVIEW: {
+        CLOUD: "gcp",
+        DESCRIPTION: "Create a SH Gitpod preview environment on a K3s cluster, created on a GCP instance",
+        PHASES: [
+            "STANDARD_K3S_CLUSTER_ON_GCP",
+            "CERT_MANAGER",
+            "GENERATE_KOTS_CONFIG",
+            "INSTALL_GITPOD",
+            "CHECK_INSTALLATION",
+            "RESULTS",
+        ],
+    },
+};
+
+const config: TestConfig = TEST_CONFIGURATIONS[testConfig];
+const cloud: string = config.CLOUD;
+
 // `INFRA_PHASES` describe the phases that can be mixed
 // and matched to form a test configuration
 // Each phase should contain a `makeTarget` which
@@ -41,14 +116,22 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
         makeTarget: "managed-dns",
         description: "Sets up external-dns & cloudDNS config",
     },
+    GENERATE_KOTS_CONFIG: {
+        phase: "generate-kots-config",
+        makeTarget: `generate-kots-config storage=${randomize("storage", cloud)} registry=${randomize(
+            "registry",
+            cloud,
+        )} db=${randomize("db", cloud)}`,
+        description: `Generate KOTS Config file`,
+    },
     INSTALL_GITPOD_IGNORE_PREFLIGHTS: {
         phase: "install-gitpod-without-preflights",
-        makeTarget: `kots-install channel=unstable version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this
+        makeTarget: `kots-install channel=${channel} version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this
         description: "Install gitpod using kots community edition without preflights",
     },
     INSTALL_GITPOD: {
         phase: "install-gitpod",
-        makeTarget: `kots-install channel=unstable version=${version} preflights=true`,
+        makeTarget: `kots-install channel=${channel} version=${version} preflights=true`,
         description: "Install gitpod using kots community edition",
     },
     CHECK_INSTALLATION: {
@@ -57,6 +140,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
         makeTarget: "check-gitpod-installation",
         description: "Check gitpod installation",
     },
+    KOTS_UPGRADE: {
+        phase: "kots-upgrade",
+        makeTarget: "kots-upgrade",
+        description: "Upgrade Gitpod installation to latest version using KOTS CLI",
+    },
     RUN_INTEGRATION_TESTS: {
         phase: "run-integration-tests",
         makeTarget: "run-tests",
@@ -74,56 +162,6 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
     },
 };
 
-interface TestConfig {
-    DESCRIPTION: string;
-    PHASES: string[];
-}
-
-// Each of the TEST_CONFIGURATIONS define an integration test end-to-end
-// It should be a combination of multiple INFRA_PHASES, order of PHASES slice is important
-const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
-    STANDARD_GKE_TEST: {
-        DESCRIPTION: "Deploy Gitpod on GKE, with managed DNS, and run integration tests",
-        PHASES: [
-            "STANDARD_GKE_CLUSTER",
-            "CERT_MANAGER",
-            "GCP_MANAGED_DNS",
-            "INSTALL_GITPOD",
-            "CHECK_INSTALLATION",
-            "RUN_INTEGRATION_TESTS",
-            "RESULTS",
-            "DESTROY",
-        ],
-    },
-    STANDARD_K3S_TEST: {
-        DESCRIPTION:
-            "Deploy Gitpod on a K3s cluster, created on a GCP instance," +
-            " with managed DNS and run integrations tests",
-        PHASES: [
-            "STANDARD_K3S_CLUSTER_ON_GCP",
-            "CERT_MANAGER",
-            "INSTALL_GITPOD_IGNORE_PREFLIGHTS",
-            "CHECK_INSTALLATION",
-            "RUN_INTEGRATION_TESTS",
-            "RESULTS",
-            "DESTROY",
-        ],
-    },
-    STANDARD_K3S_PREVIEW: {
-        DESCRIPTION: "Create a SH Gitpod preview environment on a K3s cluster, created on a GCP instance",
-        PHASES: [
-            "STANDARD_K3S_CLUSTER_ON_GCP",
-            "GCP_MANAGED_DNS",
-            "INSTALL_GITPOD_IGNORE_PREFLIGHTS",
-            "CHECK_INSTALLATION",
-            "RESULTS",
-        ],
-    },
-};
-
-// TODO better way to clean up
-const config: TestConfig = TEST_CONFIGURATIONS[testConfig];
-
 if (config === undefined) {
     console.log(`Unknown configuration specified: "${testConfig}", Exiting...`);
     process.exit(1);
@@ -135,12 +173,6 @@ installerTests(TEST_CONFIGURATIONS[testConfig]).catch((err) => {
     process.exit(1);
 });
 
-function getKubeconfig() {
-    const ret = exec(`make -C ${makefilePath} get-kubeconfig`);
-    const filename = ret.stdout.toString().split("\n").slice(1, -1);
-    exec(`echo ${filename}`);
-}
-
 export async function installerTests(config: TestConfig) {
     console.log(config.DESCRIPTION);
     for (let phase of config.PHASES) {
@@ -155,9 +187,13 @@ export async function installerTests(config: TestConfig) {
 }
 
 function callMakeTargets(phase: string, description: string, makeTarget: string) {
-    werft.phase(phase, description);
+    werft.phase(phase, `${description}`);
+    werft.log(phase, `calling ${makeTarget}`);
 
-    const response = exec(`make -C ${makefilePath} ${makeTarget}`, { slice: "call-make-target", dontCheckRc: true });
+    const response = exec(`make -C ${makefilePath} ${makeTarget}`, {
+        slice: "call-make-target",
+        dontCheckRc: true,
+    });
 
     if (response.code) {
         console.error(`Error: ${response.stderr}`);
@@ -170,6 +206,13 @@ function callMakeTargets(phase: string, description: string, makeTarget: string)
     return response.code;
 }
 
+function randomize(resource: string, platform: string): string {
+    // in the follow-up PR we will add `${platform}-${resource}` as an option here to
+    // test against resource dependencies(storage, db, registry) for each cloud platform
+    const options = ["incluster"];
+    return options[Math.floor(Math.random() * options.length)];
+}
+
 function cleanup() {
     const phase = "destroy-infrastructure";
     werft.phase(phase, "Destroying all the created resources");