[installer] Add upgrade test werft flag

Author: nandajavarma

.werft/build.ts

@@ -1,75 +1,81 @@
-import * as fs from 'fs';
-import { SpanStatusCode } from '@opentelemetry/api';
-import { Werft } from './util/werft';
-import { reportBuildFailureInSlack } from './util/slack';
-import * as Tracing from './observability/tracing'
-import * as VM from './vm/vm'
-import { buildAndPublish } from './jobs/build/build-and-publish';
-import { validateChanges } from './jobs/build/validate-changes';
-import { prepare } from './jobs/build/prepare';
-import { deployToPreviewEnvironment } from './jobs/build/deploy-to-preview-environment';
-import { triggerIntegrationTests } from './jobs/build/trigger-integration-tests';
-import { jobConfig } from './jobs/build/job-config';
-import { typecheckWerftJobs } from './jobs/build/typecheck-werft-jobs';
+import * as fs from "fs";
+import { SpanStatusCode } from "@opentelemetry/api";
+import { Werft } from "./util/werft";
+import { reportBuildFailureInSlack } from "./util/slack";
+import * as Tracing from "./observability/tracing";
+import * as VM from "./vm/vm";
+import { buildAndPublish } from "./jobs/build/build-and-publish";
+import { validateChanges } from "./jobs/build/validate-changes";
+import { prepare } from "./jobs/build/prepare";
+import { deployToPreviewEnvironment } from "./jobs/build/deploy-to-preview-environment";
+import { triggerIntegrationTests } from "./jobs/build/trigger-integration-tests";
+import { triggerUpgradeTests } from "./jobs/build/self-hosted-upgrade-tests";
+import { jobConfig } from "./jobs/build/job-config";
+import { typecheckWerftJobs } from "./jobs/build/typecheck-werft-jobs";
 
 // Will be set once tracing has been initialized
-let werft: Werft
-const context: any = JSON.parse(fs.readFileSync('context.json').toString());
+let werft: Werft;
+const context: any = JSON.parse(fs.readFileSync("context.json").toString());
 
 Tracing.initialize()
     .then(() => {
-        werft = new Werft("build")
+        werft = new Werft("build");
     })
     .then(() => run(context))
     .catch((err) => {
         werft.rootSpan.setStatus({
             code: SpanStatusCode.ERROR,
-            message: err
-        })
+            message: err,
+        });
 
-        console.log('Error', err)
+        console.log("Error", err);
 
         if (context.Repository.ref === "refs/heads/main") {
             reportBuildFailureInSlack(context, err).catch((error: Error) => {
-                console.error("Failed to send message to Slack", error)
+                console.error("Failed to send message to Slack", error);
             });
         }
 
         // Explicitly not using process.exit as we need to flush tracing, see tracing.js
-        process.exitCode = 1
+        process.exitCode = 1;
     })
     .finally(() => {
-        werft.phase("Stop kubectl port forwards", "Stopping kubectl port forwards")
-        VM.stopKubectlPortForwards()
+        werft.phase("Stop kubectl port forwards", "Stopping kubectl port forwards");
+        VM.stopKubectlPortForwards();
 
-        werft.phase("Flushing telemetry", "Flushing telemetry before stopping job")
-        werft.endAllSpans()
-    })
+        werft.phase("Flushing telemetry", "Flushing telemetry before stopping job");
+        werft.endAllSpans();
+    });
 
 async function run(context: any) {
-    const config = jobConfig(werft, context)
+    const config = jobConfig(werft, context);
 
-    await validateChanges(werft, config)
-    await prepare(werft, config)
-    await typecheckWerftJobs(werft)
-    await buildAndPublish(werft, config)
+    await validateChanges(werft, config);
+    await prepare(werft, config);
+    if (config.withUpgradeTests) {
+        // this will trigger an upgrade test on a self-hosted gitpod instance on a new cluster.
+        await triggerUpgradeTests(werft, config, context.Owner);
+        return;
+    }
+    await typecheckWerftJobs(werft);
+    await buildAndPublish(werft, config);
 
     if (config.noPreview) {
         werft.phase("deploy", "not deploying");
         console.log("no-preview or publish-release is set");
-        return
+        return;
     }
 
     try {
-        await deployToPreviewEnvironment(werft, config)
+        await deployToPreviewEnvironment(werft, config);
     } catch (e) {
         // We currently don't support concurrent deployments to the same preview environment.
         // Until we do we don't want errors to mark the main build as failed.
         if (config.mainBuild) {
-            return
+            return;
         }
-        throw e
+        throw e;
     }
 
-    await triggerIntegrationTests(werft, config, context.Owner)
+    await triggerIntegrationTests(werft, config, context.Owner);
 }

.werft/installer-tests.ts

@@ -4,7 +4,10 @@ import { Werft } from "./util/werft";
 
 const testConfig: string = process.argv.length > 2 ? process.argv[2] : "STANDARD_K3S_TEST";
 // we can provide the version of the gitpod to install (eg: 2022.4.2)
-const version: string = process.argv.length > 3 ? process.argv[3] : "";
+// "-" is the default value which will install the latest version
+const version: string = process.argv.length > 3 ? process.argv[3] : "-";
+
+const channel: string = process.argv.length > 4 ? process.argv[4] : "unstable";
 
 const makefilePath: string = join("install/tests");
 
@@ -41,14 +44,19 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
         makeTarget: "managed-dns",
         description: "Sets up external-dns & cloudDNS config",
     },
+    GENERATE_KOTS_CONFIG: {
+        phase: "generate-kots-config",
+        makeTarget: "generate-kots-config",
+        description: `Generate KOTS Config file`,
+    },
     INSTALL_GITPOD_IGNORE_PREFLIGHTS: {
         phase: "install-gitpod-without-preflights",
-        makeTarget: `kots-install channel=unstable version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this
+        makeTarget: `kots-install channel=${channel} version=${version} preflights=false`, // this is a bit of a hack, for now we pass params like this
         description: "Install gitpod using kots community edition without preflights",
     },
     INSTALL_GITPOD: {
         phase: "install-gitpod",
-        makeTarget: `kots-install channel=unstable version=${version} preflights=true`,
+        makeTarget: `kots-install channel=${channel} version=${version} preflights=true`,
         description: "Install gitpod using kots community edition",
     },
     CHECK_INSTALLATION: {
@@ -57,6 +65,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
         makeTarget: "check-gitpod-installation",
         description: "Check gitpod installation",
     },
+    KOTS_UPGRADE: {
+        phase: "kots-upgrade",
+        makeTarget: "kots-uprgade",
+        description: "Upgrade Gitpod installation to latest version using KOTS CLI",
+    },
     RUN_INTEGRATION_TESTS: {
         phase: "run-integration-tests",
         makeTarget: "run-tests",
@@ -88,21 +101,37 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
             "STANDARD_GKE_CLUSTER",
             "CERT_MANAGER",
             "GCP_MANAGED_DNS",
+            "GENERATE_KOTS_CONFIG",
             "INSTALL_GITPOD",
             "CHECK_INSTALLATION",
             "RUN_INTEGRATION_TESTS",
             "RESULTS",
             "DESTROY",
         ],
     },
+    STANDARD_GKE_UPGRADE_TEST: {
+        DESCRIPTION: `Deploy Gitpod on GKE, and test upgrade from ${version} to latest version`,
+        PHASES: [
+            "STANDARD_GKE_CLUSTER",
+            "CERT_MANAGER",
+            "GCP_MANAGED_DNS",
+            "GENERATE_KOTS_CONFIG",
+            "INSTALL_GITPOD",
+            "CHECK_INSTALLATION",
+            "KOTS_UPGRADE",
+            "CHECK_INSTALLATION",
+            "DESTROY",
+        ],
+    },
     STANDARD_K3S_TEST: {
         DESCRIPTION:
             "Deploy Gitpod on a K3s cluster, created on a GCP instance," +
             " with managed DNS and run integrations tests",
         PHASES: [
             "STANDARD_K3S_CLUSTER_ON_GCP",
             "CERT_MANAGER",
-            "INSTALL_GITPOD_IGNORE_PREFLIGHTS",
+            "GENERATE_KOTS_CONFIG",
+            "INSTALL_GITPOD",
             "CHECK_INSTALLATION",
             "RUN_INTEGRATION_TESTS",
             "RESULTS",
@@ -113,8 +142,9 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
         DESCRIPTION: "Create a SH Gitpod preview environment on a K3s cluster, created on a GCP instance",
         PHASES: [
             "STANDARD_K3S_CLUSTER_ON_GCP",
-            "GCP_MANAGED_DNS",
-            "INSTALL_GITPOD_IGNORE_PREFLIGHTS",
+            "CERT_MANAGER",
+            "GENERATE_KOTS_CONFIG",
+            "INSTALL_GITPOD",
             "CHECK_INSTALLATION",
             "RESULTS",
         ],
@@ -144,8 +174,9 @@ function getKubeconfig() {
 export async function installerTests(config: TestConfig) {
     console.log(config.DESCRIPTION);
     for (let phase of config.PHASES) {
-        const phaseSteps = INFRA_PHASES[phase];
-        const ret = callMakeTargets(phaseSteps.phase, phaseSteps.description, phaseSteps.makeTarget);
+        const args = phase.split(" ");
+        const phaseSteps = INFRA_PHASES[args[0]];
+        const ret = callMakeTargets(phaseSteps.phase, phaseSteps.description, phaseSteps.makeTarget, args.slice(1));
         if (ret) {
             // there is not point in continuing if one stage fails
             // TODO: maybe add failable, phases
@@ -154,8 +185,8 @@ export async function installerTests(config: TestConfig) {
     }
 }
 
-function callMakeTargets(phase: string, description: string, makeTarget: string) {
-    werft.phase(phase, description);
+function callMakeTargets(phase: string, description: string, makeTarget: string, args: string[]) {
+    werft.phase(phase, `${description} ${args}`);
 
     const response = exec(`make -C ${makefilePath} ${makeTarget}`, { slice: "call-make-target", dontCheckRc: true });
 
@@ -170,6 +201,10 @@ function callMakeTargets(phase: string, description: string, makeTarget: string)
     return response.code;
 }
 
+function sample(stages: string[]): string {
+    return stages[Math.floor(Math.random() * stages.length)];
+}
+
 function cleanup() {
     const phase = "destroy-infrastructure";
     werft.phase(phase, "Destroying all the created resources");