fixing issue with k3s

Author: nandajavarma

.werft/eks-installer-tests.yaml

@@ -78,6 +78,9 @@ pod:
 
         (cd .werft && yarn install && mv node_modules ..) | werft log slice prep
         printf '{{ toJson . }}' > context.json
+        curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+        unzip awscliv2.zip
+        sudo ./aws/install
 
         npx ts-node .werft/installer-tests.ts "STANDARD_EKS_TEST"
 # The bit below makes this a cron job

.werft/installer-tests.ts

@@ -35,6 +35,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
             "STANDARD_GKE_CLUSTER",
             "CERT_MANAGER",
             "GCP_MANAGED_DNS",
+            "CLUSTER_ISSUER",
             "GENERATE_KOTS_CONFIG",
             "INSTALL_GITPOD",
             "CHECK_INSTALLATION",
@@ -49,6 +50,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
         PHASES: [
             "STANDARD_GKE_CLUSTER",
             "CERT_MANAGER",
+            "CLUSTER_ISSUER",
             "GCP_MANAGED_DNS",
             "GENERATE_KOTS_CONFIG",
             "INSTALL_GITPOD",
@@ -66,11 +68,12 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
         PHASES: [
             "STANDARD_K3S_CLUSTER_ON_GCP",
             "CERT_MANAGER",
+            "CLUSTER_ISSUER",
             "GENERATE_KOTS_CONFIG",
             "INSTALL_GITPOD",
+            "RESULTS",
             "CHECK_INSTALLATION",
             "RUN_INTEGRATION_TESTS",
-            "RESULTS",
             "DESTROY",
         ],
     },
@@ -80,6 +83,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
         PHASES: [
             "STANDARD_K3S_CLUSTER_ON_GCP",
             "CERT_MANAGER",
+            "CLUSTER_ISSUER",
             "GENERATE_KOTS_CONFIG",
             "INSTALL_GITPOD",
             "CHECK_INSTALLATION",
@@ -116,7 +120,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
             "RESULTS",
             "INSTALL_GITPOD",
             "CHECK_INSTALLATION",
-            // "RUN_INTEGRATION_TESTS",
+            "RUN_INTEGRATION_TESTS",
             "DESTROY",
         ],
     },
@@ -169,18 +173,18 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
         description: `Generate KOTS Config file`,
     },
     CLUSTER_ISSUER: {
-        phase: `setup-azure-cluster-issuer cloud=${cloud}`,
-        makeTarget: "azure-issuer",
-        description: "Deploys ClusterIssuer for azure",
+        phase: "setup-cluster-issuer",
+        makeTarget: `cluster-issuer cloud=${cloud}`,
+        description: `Deploys ClusterIssuer for ${cloud}`,
     },
     EXTERNALDNS: {
         phase: "external-dns",
         makeTarget: `external-dns cloud=${cloud}`,
         description: `Deploys external-dns with ${cloud} provider`,
     },
     ADD_NS_RECORD: {
-        phase: `add-ns-record cloud=${cloud}`,
-        makeTarget: "add-ns-record",
+        phase: "add-ns-record",
+        makeTarget: `add-ns-record cloud=${cloud}`,
         description: "Adds NS record for subdomain under gitpod-self-hosted.com",
     },
     INSTALL_GITPOD_IGNORE_PREFLIGHTS: {
@@ -246,23 +250,24 @@ export async function installerTests(config: TestConfig) {
 }
 
 function callMakeTargets(phase: string, description: string, makeTarget: string) {
-    werft.phase(phase, `${description}`);
-    werft.log(phase, `calling ${makeTarget}`);
+    werft.phase(phase, description);
 
     const response = exec(`make -C ${makefilePath} ${makeTarget}`, {
-        slice: "call-make-target",
+        slice: phase,
         dontCheckRc: true,
     });
 
     if (response.code) {
         console.error(`Error: ${response.stderr}`);
         werft.fail(phase, "Operation failed");
-    } else {
-        werft.log(phase, response.stdout.toString());
-        werft.done(phase);
+        return response.code;
     }
 
+    werft.log(phase, response.stdout.toString());
+    werft.done(phase);
+
     return response.code;
+
 }
 
 function randomize(resource: string, platform: string): string {

install/tests/Makefile

@@ -34,13 +34,15 @@ eks-standard-cluster:
 	terraform init --upgrade && \
 	terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
 	terraform apply -target=module.eks -var kubeconfig=${KUBECONFIG} --auto-approve
+	@echo "Done creating EKS cluster"
 
 .PHONY:
 ## aks-standard-cluster: Creates an AKS cluster
 aks-standard-cluster:
 	terraform init --upgrade && \
 	terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
 	terraform apply -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve
+	@echo "Done creating AKS cluster"
 
 cloud ?= azure
 .PHONY:
@@ -49,6 +51,7 @@ add-ns-record:
 	terraform init --upgrade && \
 	terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
 	terraform apply -target=module.$(cloud)-add-dns-record  -var kubeconfig=${KUBECONFIG} --auto-approve
+	@echo "Done adding NS record"
 
 cloud ?= azure
 .PHONY:
@@ -57,35 +60,37 @@ cluster-issuer:
 	terraform init --upgrade && \
 	terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
 	terraform apply -target=module.$(cloud)-issuer  -var kubeconfig=${KUBECONFIG} --auto-approve
+	@echo "Done creating cluster issuer"
 
 .PHONY:
 ## k3s-standard-cluster: Creates a K3S cluster on GCP with one master and 1 worker node
 k3s-standard-cluster:
 	terraform init --upgrade && \
 	terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
 	terraform apply -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve
-
-CLUSTER_ISSUER_CLOUD_DNS := "./manifests/gcp-issuer.yaml"
+	@echo "Done creating k3s cluster"
 
 .PHONY:
 ## cert-manager: Installs cert-manager, optionally create secret for cloud-dns access
 cert-manager:
 	terraform workspace select $(TF_VAR_TEST_ID) && \
 	terraform apply -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve
+	@echo "Done installing cert-manager"
 
 .PHONY:
 ## managed-dns: Installs external-dns, and setup up CloudDNS access
 managed-dns: check-env-sub-domain
 	terraform workspace select $(TF_VAR_TEST_ID) && \
-	terraform apply -target=module.clouddns-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve && \
-    kubectl --kubeconfig=${KUBECONFIG} apply -f ${CLUSTER_ISSUER_CLOUD_DNS}
+	terraform apply -target=module.clouddns-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve
+	@echo "Done created GCP managed DNS"
 
 cloud ?= "azure"
 .PHONY:
 ## external-dns: Installs external-dns
 external-dns: check-env-sub-domain
 	terraform workspace select $(TF_VAR_TEST_ID) && \
 	terraform apply -target=module.$(cloud)-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve
+	@echo "Done creating externaldns for ${cloud}"
 
 .PHONY:
 ## get-kubeconfig: Returns KUBECONFIG of a just created cluster

install/tests/main.tf

@@ -39,6 +39,20 @@ module "k3s" {
   domain_name      = "${var.TEST_ID}.gitpod-self-hosted.com"
 }
 
+module "gcp-issuer" {
+  source              = "../infra/terraform/tools/issuer"
+  kubeconfig          = var.kubeconfig
+  issuer_name         = "cloudDNS"
+  cert_manager_issuer = {
+    project  = "dns-for-playgrounds"
+    serviceAccountSecretRef = {
+      name = "clouddns-dns01-solver"
+      key = "keys.json"
+    }
+  }
+}
+
+
 module "aks" {
   # source = "github.com/gitpod-io/gitpod//install/infra/terraform/aks?ref=main" # we can later use tags here
   source                   = "../infra/terraform/aks"