Add tests for external dependencies in AKS

Author: nandajavarma

.werft/build.ts

@@ -63,7 +63,7 @@ async function run(context: any) {
     if (!config.withPreview || config.publishRelease) {
         werft.phase("deploy", "not deploying");
         console.log("running without preview environment or publish-release is set");
-        return
+        return;
     }
 
     try {

install/infra/terraform/aks/kubernetes.tf

@@ -40,7 +40,7 @@ resource "azurerm_kubernetes_cluster" "k8s" {
     zones                        = []
 
     enable_auto_scaling  = true
-    min_count            = 2
+    min_count            = 1
     max_count            = 10
     orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
     node_labels          = local.nodes.0.labels
@@ -72,7 +72,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "pools" {
   vm_size               = local.machine
 
   enable_auto_scaling  = true
-  min_count            = 2
+  min_count            = 1
   max_count            = 10
   orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
   node_labels          = local.nodes[count.index + 1].labels

install/infra/terraform/aks/local.tf

@@ -8,13 +8,13 @@ locals {
   })
   dns_enabled = var.domain_name != null
   name_format = join("-", [
-    "gitpod",
+    "gitpod-test",
     "%s", # region
     "%s", # name
     local.workspace_name
   ])
   name_format_global = join("-", [
-    "gitpod",
+    "gitpod-test",
     "%s", # name
     local.workspace_name
   ])

install/infra/terraform/tools/azure-external-dns/main.tf

@@ -1,6 +1,7 @@
 variable settings {}
 variable domain_name { default = "test"}
 variable kubeconfig { default = "conf"}
+variable txt_owner_id { default = "nightly-test"}
 
 provider "helm" {
   kubernetes {
@@ -50,12 +51,8 @@ resource "helm_release" "external_dns" {
     value = var.settings["azure.resourceGroup"]
   }
 
-  # TODO Add tags using dynamic block
-  # https://github.com/hashicorp/terraform/issues/22340
-  #  dynamic "set" {
-  #    for_each = var.tags
-  #    iterator = "tag"
-  #    name     = "podLabels[${index(var.tags, tag.key)}]"
-  #    value    = tag.value
-  #  }
+  set {
+    name  = "txt-owner-id"
+    value = var.txt_owner_id
+  }
 }

install/infra/terraform/tools/issuer/azure/main.tf

@@ -15,6 +15,7 @@ resource "kubernetes_manifest" "clusterissuer_gitpod" {
           "name" = "issuer-account-key"
         }
         "server" = "https://acme-v02.api.letsencrypt.org/directory"
+        # "server" = "https://acme-staging-v02.api.letsencrypt.org/directory"
         "solvers" = [
           {
             "dns01" = {

install/tests/Makefile

@@ -33,10 +33,11 @@ gke-standard-cluster:
 aks-standard-cluster:
 	terraform init --upgrade && \
 	terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
-	terraform apply -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve
+	terraform apply -target=module.aks -var k8s_flavor="aks" -var kubeconfig=${KUBECONFIG} --auto-approve
 
 .PHONY:
 ## azure-external-dns: Sets up external-dns with azure provider
+azure-external-dns:
 azure-external-dns:
 	terraform init --upgrade && \
 	terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \
@@ -111,10 +112,24 @@ get-config-gcp-db:
 	yq m -i tmp_config.yml tmp_5_config.yml
 
 get-config-azure-storage:
+	export PASSWORD=$$(terraform output -json storage | yq r - 'password') && \
+	export USERNAME=$$(terraform output -json storage | yq r - 'username') && \
+	envsubst < ./manifests/kots-config-azure-storage.yaml > tmp_2_config.yml
+	yq m -i tmp_config.yml tmp_2_config.yml
 
 get-config-azure-db:
+	export DBHOST=$$(terraform output -json database | yq r - 'host') && \
+	export DBPASS=$$(terraform output -json database | yq r - 'password') && \
+	export DBUSER=$$(terraform output -json database | yq r - 'username') && \
+	envsubst < ./manifests/kots-config-azure-db.yaml > tmp_2_config.yml
+	yq m -i tmp_config.yml tmp_2_config.yml
 
 get-config-azure-registry:
+	export SERVER=$$(terraform output -json registry | yq r - 'server') && \
+	export PASSWORD=$$(terraform output -json registry | yq r - 'password') && \
+	export USERNAME=$$(terraform output -json registry | yq r - 'username') && \
+	envsubst < ./manifests/kots-config-azure-registry.yaml > tmp_2_config.yml
+	yq m -i tmp_config.yml tmp_2_config.yml
 
 storage ?= incluster
 registry ?= incluster
@@ -158,7 +173,7 @@ check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain
 run-tests:
 	./tests.sh ${KUBECONFIG}
 
-kots-upgrade:
+kots-uprgade:
 	@echo "Upgrade gitpod KOTS app to latest"
 	kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy
 
@@ -174,13 +189,13 @@ destroy-certmanager: select-workspace
 	ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
 
 destroy-gcpns: select-workspace
-	ls ${KUBECONFIG} && terraform destroy -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
+	terraform destroy -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve
 
 destroy-aks-edns: select-workspace
-	ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve
+	ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve  || echo "No kubeconfig file"
 
 destroy-aks-issuer: select-workspace
-	ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve
+	ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file"
 
 destroy-gke: select-workspace
 	terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve
@@ -193,7 +208,7 @@ destroy-aks: select-workspace
 
 get-results:
 	@echo "If you have gotten this far, it means your setup succeeded"
-	@echo "The IP address of you setup is "https://$(TF_VAR_TEST_ID).gitpod-self-hosted.com""
+	@echo "The IP address of you setup is "$(TF_VAR_TEST_ID).gitpod-self-hosted.com""
 	@echo "Following is the KUBECONFIG you can use to connect to the cluster:"
 	@cat ${KUBECONFIG}
 

install/tests/main.tf

@@ -1,6 +1,8 @@
 variable "kubeconfig" { }
 variable "TEST_ID" { default = "nightly" }
 
+variable "k8s_flavor" { default = "gke" }
+
 # We store the state always in a GCS bucket
 terraform {
   backend "gcs" {
@@ -45,9 +47,9 @@ module "aks" {
 
   domain_name              = "${var.TEST_ID}.gitpod-self-hosted.com"
   enable_airgapped         = false
-  enable_external_database = false
-  enable_external_registry = false
-  enable_external_storage  = false
+  enable_external_database = true
+  enable_external_registry = true
+  enable_external_storage  = true
   dns_enabled              = true
   workspace_name           = var.TEST_ID
 }
@@ -73,6 +75,7 @@ module "azure-externaldns" {
   kubeconfig     = var.kubeconfig
   settings = module.aks.external_dns_settings
   domain_name = "${var.TEST_ID}.gitpod-self-hosted.com"
+  txt_owner_id   = var.TEST_ID
 }
 
 module "azure-issuer" {

install/tests/manifests/kots-config-azure-db.yaml

@@ -0,0 +1,16 @@
+apiVersion: kots.io/v1beta1
+kind: ConfigValues
+spec:
+  values:
+      db_incluster:
+        value: "0"
+        data: "db_incluster"
+      db_host:
+        value: ${DBHOST}
+        data: "db_host"
+      db_username:
+        value: ${DBUSER}
+        data: "db_username"
+      db_password:
+        value: ${DBPASS}
+        data: "db_password"

install/tests/manifests/kots-config-azure-registry.yaml

@@ -0,0 +1,16 @@
+apiVersion: kots.io/v1beta1
+kind: ConfigValues
+spec:
+  values:
+      reg_incluster:
+        value: "0"
+        data: "reg_incluster"
+      reg_url:
+        value: ${SERVER}
+        data: "reg_url"
+      reg_username:
+        value: ${USERNAME}
+        data: "reg_username"
+      reg_password:
+        value: ${PASSWORD}
+        data: "reg_password"

install/tests/manifests/kots-config-azure-storage.yaml

@@ -0,0 +1,16 @@
+apiVersion: kots.io/v1beta1
+kind: ConfigValues
+spec:
+  values:
+      store_provider:
+        value: azure
+        data: "store_provider"
+      store_region:
+        value: "northeurope"
+        data: "store_region"
+      store_azure_account_name:
+        value: ${USERNAME}
+        data: "store_azure_account_name"
+      store_azure_access_key:
+        value: ${PASSWORD}
+        data: "store_azure_access_key"

install/tests/output.tf

@@ -0,0 +1,18 @@
+locals {
+    cloud = var.k8s_flavor == "aks" ? module.aks : null
+}
+
+output "storage" {
+    sensitive = true
+    value = try(lookup(local.cloud, "storage"), {})
+}
+
+output "registry" {
+    sensitive = true
+    value = try(lookup(local.cloud, "registry"), {})
+}
+
+output "database" {
+    sensitive = true
+    value = try(lookup(local.cloud, "database"), {})
+}