From 7bb26d2d5a5a204154c5c108f0794f6d585a33f1 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Mon, 20 Jun 2022 08:49:14 +0000 Subject: [PATCH 1/3] Add tests for external dependencies in AKS --- install/infra/terraform/tools/issuer/azure/main.tf | 1 + install/tests/Makefile | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/install/infra/terraform/tools/issuer/azure/main.tf b/install/infra/terraform/tools/issuer/azure/main.tf index d2ad4fd71ce4fe..e76a6d9761de05 100644 --- a/install/infra/terraform/tools/issuer/azure/main.tf +++ b/install/infra/terraform/tools/issuer/azure/main.tf @@ -15,6 +15,7 @@ resource "kubernetes_manifest" "clusterissuer_gitpod" { "name" = "issuer-account-key" } "server" = "https://acme-v02.api.letsencrypt.org/directory" + # "server" = "https://acme-staging-v02.api.letsencrypt.org/directory" "solvers" = [ { "dns01" = { diff --git a/install/tests/Makefile b/install/tests/Makefile index 2de4ee38c075a6..05b0b2eb81cb53 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -37,6 +37,7 @@ aks-standard-cluster: .PHONY: ## azure-external-dns: Sets up external-dns with azure provider +azure-external-dns: azure-external-dns: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ @@ -172,7 +173,7 @@ check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain run-tests: ./tests.sh ${KUBECONFIG} -kots-upgrade: +kots-uprgade: @echo "Upgrade gitpod KOTS app to latest" kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy From cdb458037aab63f9a6a00e091eea5ff4e7220e2c Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Thu, 16 Jun 2022 19:50:54 +0000 Subject: [PATCH 2/3] [installer] Add EKS installer test --- .werft/eks-installer-tests.yaml | 88 +++++++++++ .werft/installer-tests.ts | 70 ++++++--- install/infra/terraform/aks/output.tf | 34 ++++- install/infra/terraform/eks/database.tf | 40 +++++ install/infra/terraform/eks/dns.tf | 114 ++++++++++++++ install/infra/terraform/eks/kubernetes.tf | 142 ++++++++++++++++++ install/infra/terraform/eks/output.tf | 72 +++++++++ install/infra/terraform/eks/providers.tf | 16 ++ install/infra/terraform/eks/registry.tf | 12 ++ install/infra/terraform/eks/storage.tf | 37 +++++ install/infra/terraform/eks/variables.tf | 70 +++++++++ .../README.md | 0 .../tools/cloud-dns-external-dns/main.tf | 74 +++++++++ .../variables.tf | 0 .../terraform/tools/external-dns/main.tf | 76 +++------- .../tools/issuer/{azure => }/main.tf | 14 +- .../tools/issuer/{azure => }/variables.tf | 8 + install/tests/Makefile | 125 +++++++++------ install/tests/main.tf | 87 ++++++++--- .../tests/manifests/kots-config-aws-db.yaml | 16 ++ .../manifests/kots-config-aws-registry.yaml | 16 ++ .../manifests/kots-config-aws-storage.yaml | 22 +++ install/tests/output.tf | 27 +++- 23 files changed, 1000 insertions(+), 160 deletions(-) create mode 100644 .werft/eks-installer-tests.yaml create mode 100644 install/infra/terraform/eks/database.tf create mode 100644 install/infra/terraform/eks/dns.tf create mode 100644 install/infra/terraform/eks/kubernetes.tf create mode 100644 install/infra/terraform/eks/output.tf create mode 100644 install/infra/terraform/eks/providers.tf create mode 100644 install/infra/terraform/eks/registry.tf create mode 100644 install/infra/terraform/eks/storage.tf create mode 100644 install/infra/terraform/eks/variables.tf rename install/infra/terraform/tools/{external-dns => cloud-dns-external-dns}/README.md (100%) create mode 100644 install/infra/terraform/tools/cloud-dns-external-dns/main.tf rename install/infra/terraform/tools/{external-dns => cloud-dns-external-dns}/variables.tf (100%) rename install/infra/terraform/tools/issuer/{azure => }/main.tf (67%) rename install/infra/terraform/tools/issuer/{azure => }/variables.tf (62%) create mode 100644 install/tests/manifests/kots-config-aws-db.yaml create mode 100644 install/tests/manifests/kots-config-aws-registry.yaml create mode 100644 install/tests/manifests/kots-config-aws-storage.yaml diff --git a/.werft/eks-installer-tests.yaml b/.werft/eks-installer-tests.yaml new file mode 100644 index 00000000000000..eaf803b836f308 --- /dev/null +++ b/.werft/eks-installer-tests.yaml @@ -0,0 +1,88 @@ +# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/eks-installer-tests.yaml -a debug=true` +pod: + serviceAccount: werft + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: dev/workload + operator: In + values: + - "builds" + securityContext: + runAsUser: 0 + volumes: + - name: sh-playground-sa-perm + secret: + secretName: sh-playground-sa-perm + - name: sh-playground-dns-perm + secret: + secretName: sh-playground-dns-perm + - name: sh-aks-perm + secret: + secretName: aks-credentials + containers: + - name: nightly-test + image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0 + workingDir: /workspace + imagePullPolicy: Always + volumeMounts: + - name: sh-playground-sa-perm + mountPath: /mnt/secrets/sh-playground-sa-perm + - name: sh-playground-dns-perm # this sa is used for the DNS management + mountPath: /mnt/secrets/sh-playground-dns-perm + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-credentials + key: aws-access-key + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-credentials + key: aws-secret-key + - name: AWS_REGION + valueFrom: + secretKeyRef: + name: aws-credentials + key: aws-region + - name: WERFT_HOST + value: "werft.werft.svc.cluster.local:7777" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: WERFT_K8S_NAMESPACE + value: "werft" + - name: WERFT_K8S_LABEL + value: "component=werft" + - name: TF_VAR_sa_creds + value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json" + - name: TF_VAR_dns_sa_creds + value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json" + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - bash + - -c + - | + sleep 1 + set -Eeuo pipefail + + sudo chown -R gitpod:gitpod /workspace + sudo apt update && apt install gettext-base + + export TF_VAR_TEST_ID="$(echo $RANDOM | md5sum | head -c 5; echo)" + + (cd .werft && yarn install && mv node_modules ..) | werft log slice prep + printf '{{ toJson . }}' > context.json + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install + + npx ts-node .werft/installer-tests.ts "STANDARD_EKS_TEST" +# The bit below makes this a cron job +# plugins: +# cron: "15 3 * * *" diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts index 35290735f948f2..8e9bf01d0f4be3 100644 --- a/.werft/installer-tests.ts +++ b/.werft/installer-tests.ts @@ -35,6 +35,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { "STANDARD_GKE_CLUSTER", "CERT_MANAGER", "GCP_MANAGED_DNS", + "CLUSTER_ISSUER", "GENERATE_KOTS_CONFIG", "INSTALL_GITPOD", "CHECK_INSTALLATION", @@ -49,6 +50,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { PHASES: [ "STANDARD_GKE_CLUSTER", "CERT_MANAGER", + "CLUSTER_ISSUER", "GCP_MANAGED_DNS", "GENERATE_KOTS_CONFIG", "INSTALL_GITPOD", @@ -66,11 +68,12 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { PHASES: [ "STANDARD_K3S_CLUSTER_ON_GCP", "CERT_MANAGER", + "CLUSTER_ISSUER", "GENERATE_KOTS_CONFIG", "INSTALL_GITPOD", + "RESULTS", "CHECK_INSTALLATION", "RUN_INTEGRATION_TESTS", - "RESULTS", "DESTROY", ], }, @@ -80,6 +83,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { PHASES: [ "STANDARD_K3S_CLUSTER_ON_GCP", "CERT_MANAGER", + "CLUSTER_ISSUER", "GENERATE_KOTS_CONFIG", "INSTALL_GITPOD", "CHECK_INSTALLATION", @@ -92,8 +96,8 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { PHASES: [ "STANDARD_AKS_CLUSTER", "CERT_MANAGER", - "AZURE_ISSUER", - "AZURE_EXTERNALDNS", + "CLUSTER_ISSUER", + "EXTERNALDNS", "ADD_NS_RECORD", "GENERATE_KOTS_CONFIG", "INSTALL_GITPOD", @@ -103,6 +107,23 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { "DESTROY", ], }, + STANDARD_EKS_TEST: { + CLOUD: "aws", + DESCRIPTION: "Create an EKS cluster", + PHASES: [ + "STANDARD_EKS_CLUSTER", + "CERT_MANAGER", + "EXTERNALDNS", + "CLUSTER_ISSUER", + "ADD_NS_RECORD", + "GENERATE_KOTS_CONFIG", + "RESULTS", + "INSTALL_GITPOD", + "CHECK_INSTALLATION", + "RUN_INTEGRATION_TESTS", + "DESTROY", + ], + }, }; const config: TestConfig = TEST_CONFIGURATIONS[testConfig]; @@ -128,6 +149,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { makeTarget: "aks-standard-cluster", description: "Creating an aks cluster(azure)", }, + STANDARD_EKS_CLUSTER: { + phase: "create-std-eks-cluster", + makeTarget: "eks-standard-cluster", + description: "Creating a EKS cluster with 1 nodepool each for workspace and server", + }, CERT_MANAGER: { phase: "setup-cert-manager", makeTarget: "cert-manager", @@ -146,19 +172,19 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { )} db=${randomize("db", cloud)}`, description: `Generate KOTS Config file`, }, - AZURE_ISSUER: { - phase: "setup-azure-cluster-issuer", - makeTarget: "azure-issuer", - description: "Deploys ClusterIssuer for azure", + CLUSTER_ISSUER: { + phase: "setup-cluster-issuer", + makeTarget: `cluster-issuer cloud=${cloud}`, + description: `Deploys ClusterIssuer for ${cloud}`, }, - AZURE_EXTERNALDNS: { - phase: "azure-external-dns", - makeTarget: "azure-external-dns", - description: "Deploys external-dns with azure provider", + EXTERNALDNS: { + phase: "external-dns", + makeTarget: `external-dns cloud=${cloud}`, + description: `Deploys external-dns with ${cloud} provider`, }, ADD_NS_RECORD: { phase: "add-ns-record", - makeTarget: "add-ns-record", + makeTarget: `add-ns-record cloud=${cloud}`, description: "Adds NS record for subdomain under gitpod-self-hosted.com", }, INSTALL_GITPOD_IGNORE_PREFLIGHTS: { @@ -189,7 +215,7 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { }, DESTROY: { phase: "destroy", - makeTarget: "cleanup", + makeTarget: `cleanup cloud=${cloud}`, description: "Destroy the created infrastucture", }, RESULTS: { @@ -224,23 +250,24 @@ export async function installerTests(config: TestConfig) { } function callMakeTargets(phase: string, description: string, makeTarget: string) { - werft.phase(phase, `${description}`); - werft.log(phase, `calling ${makeTarget}`); + werft.phase(phase, description); const response = exec(`make -C ${makefilePath} ${makeTarget}`, { - slice: "call-make-target", + slice: phase, dontCheckRc: true, }); if (response.code) { console.error(`Error: ${response.stderr}`); werft.fail(phase, "Operation failed"); - } else { - werft.log(phase, response.stdout.toString()); - werft.done(phase); + return response.code; } + werft.log(phase, response.stdout.toString()); + werft.done(phase); + return response.code; + } function randomize(resource: string, platform: string): string { @@ -254,7 +281,10 @@ function cleanup() { const phase = "destroy-infrastructure"; werft.phase(phase, "Destroying all the created resources"); - const response = exec(`make -C ${makefilePath} cleanup`, { slice: "run-terrafrom-destroy", dontCheckRc: true }); + const response = exec(`make -C ${makefilePath} cleanup cloud=${cloud}`, { + slice: "run-terrafrom-destroy", + dontCheckRc: true, + }); // if the destroy command fail, we check if any resources are pending to be removed // if nothing is yet to be cleaned, we return with success diff --git a/install/infra/terraform/aks/output.tf b/install/infra/terraform/aks/output.tf index 0ea0d4bf2f8b0e..83c60ee2205c81 100644 --- a/install/infra/terraform/aks/output.tf +++ b/install/infra/terraform/aks/output.tf @@ -36,14 +36,32 @@ output "external_dns_secrets" { } output "external_dns_settings" { - value = { - provider = "azure" - "azure.resourceGroup" = azurerm_resource_group.gitpod.name - "azure.subscriptionId" = data.azurerm_client_config.current.subscription_id - "azure.tenantId" = data.azurerm_client_config.current.tenant_id - "azure.useManagedIdentityExtension" = true - "azure.userAssignedIdentityID" = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id - } + value = [ + { + "name": "provider", + "value": "azure" + }, + { + "name": "azure.resourceGroup", + "value": azurerm_resource_group.gitpod.name, + }, + { + "name": "azure.subscriptionId", + "value": data.azurerm_client_config.current.subscription_id, + }, + { + "name": "azure.tenantId", + "value": data.azurerm_client_config.current.tenant_id, + }, + { + "name": "azure.useManagedIdentityExtension", + "value": true + }, + { + "name": "azure.userAssignedIdentityID", + "value": azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id + }, + ] } output "k8s_connection" { diff --git a/install/infra/terraform/eks/database.tf b/install/infra/terraform/eks/database.tf new file mode 100644 index 00000000000000..4d759db54906c9 --- /dev/null +++ b/install/infra/terraform/eks/database.tf @@ -0,0 +1,40 @@ +resource "aws_db_subnet_group" "gitpod_subnets" { + name = "db-sg-${var.cluster_name}" + subnet_ids = [module.vpc.public_subnets[2], module.vpc.public_subnets[3]] +} + +resource "aws_security_group" "rdssg" { + name = "dh-sg-${var.cluster_name}" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 0 + to_port = 3306 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_db_instance" "gitpod" { + allocated_storage = 10 + max_allocated_storage = 100 + engine = "mysql" + engine_version = "5.7" + instance_class = "db.t3.micro" + vpc_security_group_ids = [ aws_security_group.rdssg.id ] + identifier = "db-${var.cluster_name}" + name = "gitpod" + username = "gitpod" + password = "gitpod-qwat" + parameter_group_name = "default.mysql5.7" + db_subnet_group_name = aws_db_subnet_group.gitpod_subnets.name + skip_final_snapshot = true + publicly_accessible = true +} diff --git a/install/infra/terraform/eks/dns.tf b/install/infra/terraform/eks/dns.tf new file mode 100644 index 00000000000000..1c1c62e8a39626 --- /dev/null +++ b/install/infra/terraform/eks/dns.tf @@ -0,0 +1,114 @@ +resource "aws_route53_zone" "gitpod" { + force_destroy = true + name = var.domain_name + + tags = { + Environment = "test" + } +} + +# This creates an SSL certificate +resource "aws_acm_certificate" "gitpod" { + domain_name = var.domain_name + subject_alternative_names = ["*.ws.${var.domain_name}", "ws.${var.domain_name}", "*.${var.domain_name}"] + validation_method = "DNS" +} + +# This is a DNS record for the ACM certificate validation to prove we own the domain +# +# This example, we make an assumption that the certificate is for a single domain name so can just use the first value of the +# domain_validation_options. It allows the terraform to apply without having to be targeted. +# This is somewhat less complex than the example at https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/acm_certificate_validation +# - that above example, won't apply without targeting + +resource "aws_route53_record" "cert_validation" { + allow_overwrite = true + name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[0].resource_record_name + type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[0].resource_record_type + zone_id = "${resource.aws_route53_zone.gitpod.zone_id}" + records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[0].resource_record_value] + ttl = 60 +} + +resource "aws_route53_record" "cert_validation_1" { + allow_overwrite = true + name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[1].resource_record_name + type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[1].resource_record_type + zone_id = "${resource.aws_route53_zone.gitpod.zone_id}" + records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[1].resource_record_value] + ttl = 60 +} + +resource "aws_route53_record" "cert_validation_2" { + allow_overwrite = true + name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[2].resource_record_name + type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[2].resource_record_type + zone_id = "${resource.aws_route53_zone.gitpod.zone_id}" + records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[2].resource_record_value] + ttl = 60 +} + +resource "aws_route53_record" "cert_validation_3" { + allow_overwrite = true + name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[3].resource_record_name + type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[3].resource_record_type + zone_id = "${resource.aws_route53_zone.gitpod.zone_id}" + records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[3].resource_record_value] + ttl = 6 +} + +resource "aws_iam_policy" "gitpod" { + name = "role-${var.cluster_name}" + + # Terraform's "jsonencode" function converts a + # Terraform expression result to valid JSON syntax. + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Action = [ + "route53:ChangeResourceRecordSets" + ], + Resource = [ + "arn:aws:route53:::hostedzone/*" + ] + }, + { + Effect = "Allow", + Action = [ + "route53:GetChange", + ], + Resource = [ + "arn:aws:route53:::change/*" + ] + }, + { + Effect = "Allow", + Action = [ + "route53:ListHostedZones", + "route53:ListResourceRecordSets" + ], + Resource = [ "*" ] + } + ], + }) +} + +resource "aws_iam_user_policy_attachment" "test-attach" { + user = aws_iam_user.edns.name + policy_arn = aws_iam_policy.gitpod.arn +} + +resource "aws_iam_user" "edns" { + name = "${var.cluster_name}-external-dns" + force_destroy = true + + tags = { + env = "test" + } +} + +resource "aws_iam_access_key" "edns" { + user = aws_iam_user.edns.name +} diff --git a/install/infra/terraform/eks/kubernetes.tf b/install/infra/terraform/eks/kubernetes.tf new file mode 100644 index 00000000000000..090d498fc1e71f --- /dev/null +++ b/install/infra/terraform/eks/kubernetes.tf @@ -0,0 +1,142 @@ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "3.12.0" + + name = "vpc-${var.cluster_name}" + cidr = var.vpc_cidr + azs = var.vpc_availability_zones + private_subnets = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr] + public_subnets = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2] + enable_nat_gateway = true + enable_dns_hostnames = true +} + +resource "aws_security_group" "nodes" { + name = "nodes-sg-${var.cluster_name}" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "18.8.1" + + cluster_name = var.cluster_name + cluster_version = "1.22" + + cluster_endpoint_public_access = true + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.public_subnets + + cluster_addons = { + coredns = { + resolve_conflicts = "OVERWRITE" + } + kube-proxy = {} + vpc-cni = { + resolve_conflicts = "OVERWRITE" + service_account_role_arn = module.vpc_cni_irsa.iam_role_arn + } + } + + eks_managed_node_group_defaults = { + disk_size = 150 + ami_type = "CUSTOM" + iam_role_attach_cni_policy = true + ami_id = var.image_id + enable_bootstrap_user_data = true + vpc_security_group_ids = [aws_security_group.nodes.id] + } + + eks_managed_node_groups = { + Services = { + enable_bootstrap_user_data = true + instance_types = [var.service_machine_type] + name = "service-${var.cluster_name}" + subnet_ids = module.vpc.public_subnets + min_size = 1 + max_size = 10 + desired_size = 1 + labels = { + "gitpod.io/workload_meta" = true + "gitpod.io/workload_ide" = true + } + pre_bootstrap_user_data = <<-EOT + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + EOT + } + + Workspaces = { + instance_types = [var.workspace_machine_type] + name = "ws-${var.cluster_name}" + subnet_ids = module.vpc.public_subnets + min_size = 1 + max_size = 10 + desired_size = 1 + enable_bootstrap_user_data = true + labels = { + "gitpod.io/workload_workspace_services" = true + "gitpod.io/workload_workspace_regular" = true + "gitpod.io/workload_workspace_headless" = true + } + pre_bootstrap_user_data = <<-EOT + #!/bin/bash + set -ex + cat <<-EOF > /etc/profile.d/bootstrap.sh + export CONTAINER_RUNTIME="containerd" + export USE_MAX_PODS=false + EOF + # Source extra environment variables in bootstrap script + sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh + EOT + } + } +} + +module "vpc_cni_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 4.12" + + role_name_prefix = "VPC-CNI-IRSA" + attach_vpc_cni_policy = true + vpc_cni_enable_ipv4 = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-node"] + } + } +} + +resource "null_resource" "kubeconfig" { + depends_on = [ module.eks ] + provisioner "local-exec" { + command = "aws eks update-kubeconfig --region ${var.region} --name ${var.cluster_name} --kubeconfig ${var.kubeconfig}" + } + + lifecycle { + create_before_destroy = true + } +} diff --git a/install/infra/terraform/eks/output.tf b/install/infra/terraform/eks/output.tf new file mode 100644 index 00000000000000..151443ca504021 --- /dev/null +++ b/install/infra/terraform/eks/output.tf @@ -0,0 +1,72 @@ +output "external_dns_settings" { + value = [ + { + "name" = "provider", + "value" = "aws" + }, + { + "name" = "aws.region", + "value" = "eu-west-1" + }, + { + "name" = "aws.credentials.secretKey", + "value" = aws_iam_access_key.edns.secret + }, + { + "name" = "aws.credentials.accessKey", + "value" = aws_iam_access_key.edns.id + } + ] +} + +output "secretAccessKey" { + sensitive = true + value = try("${aws_iam_access_key.edns.secret}", "") +} + +output "cert_manager_issuer" { + value = try({ + region = "eu-west-1" + secretAccessKeySecretRef = { + name = "route53-credentials" + key = "secret-access-key" + } + + hostedZoneID = aws_route53_zone.gitpod.zone_id + accessKeyID = aws_iam_access_key.edns.id + }, {}) +} + +output "domain_nameservers" { + value = formatlist("%s.", resource.aws_route53_zone.gitpod.name_servers) +} + +output "database" { + sensitive = true + value = try({ + host = "${aws_db_instance.gitpod.address}" + password = "gitpod-qwat" # FIXME hardcoded at this point + port = 3306 + username = "${aws_db_instance.gitpod.username}" + }, {}) +} + +output "registry" { + sensitive = true + value = try({ + server = aws_ecr_repository.gitpod.repository_url + username = data.aws_ecr_authorization_token.gitpod.user_name + password = data.aws_ecr_authorization_token.gitpod.password + }, {}) +} + +output "storage" { + sensitive = true + value = try({ + access_key_id = aws_iam_access_key.bucket_storage_user.id + secret_access_key = aws_iam_access_key.bucket_storage_user.secret + region = aws_s3_bucket.gitpod-storage.region + bucket_name = aws_s3_bucket.gitpod-storage.id + endpoint = "s3.amazonaws.com" + }, {}) +} diff --git a/install/infra/terraform/eks/providers.tf b/install/infra/terraform/eks/providers.tf new file mode 100644 index 00000000000000..7e1676c2f73eca --- /dev/null +++ b/install/infra/terraform/eks/providers.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + version = " ~> 3.0" + source = "registry.terraform.io/hashicorp/aws" + } + helm = { + source = "hashicorp/helm" + version = "2.5.1" + } + } +} + +provider "aws" { + region = var.region +} diff --git a/install/infra/terraform/eks/registry.tf b/install/infra/terraform/eks/registry.tf new file mode 100644 index 00000000000000..9756b6ed4b8858 --- /dev/null +++ b/install/infra/terraform/eks/registry.tf @@ -0,0 +1,12 @@ +resource "aws_ecr_repository" "gitpod" { + name = "registry-${var.cluster_name}" + image_tag_mutability = "MUTABLE" + + image_scanning_configuration { + scan_on_push = false + } +} + +data "aws_ecr_authorization_token" "gitpod" { + registry_id = aws_ecr_repository.gitpod.registry_id +} diff --git a/install/infra/terraform/eks/storage.tf b/install/infra/terraform/eks/storage.tf new file mode 100644 index 00000000000000..772187015ab190 --- /dev/null +++ b/install/infra/terraform/eks/storage.tf @@ -0,0 +1,37 @@ +resource "aws_s3_bucket" "gitpod-storage" { + force_destroy = true + bucket = "bucket-${var.cluster_name}" + acl = "private" + + versioning { + enabled = true + } +} + +data "aws_iam_policy_document" "s3_policy" { + statement { + actions = ["s3:*"] + resources = ["*"] + effect = "Allow" + } +} + +resource "aws_iam_policy" "policy" { + name = "spolicy-${var.cluster_name}" + description = "s3 storage bucket policy" + policy = data.aws_iam_policy_document.s3_policy.json +} + +resource "aws_iam_user" "bucket_storage" { + name = "suser-${var.cluster_name}" + +} + +resource "aws_iam_user_policy_attachment" "attachment" { + user = aws_iam_user.bucket_storage.name + policy_arn = aws_iam_policy.policy.arn +} + +resource "aws_iam_access_key" "bucket_storage_user" { + user = aws_iam_user.bucket_storage.name +} diff --git a/install/infra/terraform/eks/variables.tf b/install/infra/terraform/eks/variables.tf new file mode 100644 index 00000000000000..ccbc32f5207217 --- /dev/null +++ b/install/infra/terraform/eks/variables.tf @@ -0,0 +1,70 @@ +variable "cluster_name" { + type = string + description = "EKS cluster name." + default = "nan-eks-3" +} +variable "kubeconfig" { + type = string + description = "Path to the kubeconfig file" + default = "kubeconfig" +} + +variable "image_id" { + type = string + description = "AMI Image ID specific to the region" + // latest ubuntu image for 1.22 k8s for eu-west-1 region, refer https://cloud-images.ubuntu.com/docs/aws/eks/ + default = "ami-0793b4124359a6ad7" +} + +variable "service_machine_type" { + type = string + description = "Machine type for service workload node pool" + default = "m6i.xlarge" +} + +variable "workspace_machine_type" { + type = string + description = "Machine type for workspace workload node pool" + default = "m6i.2xlarge" +} + +variable "region" { + type = string + default = "eu-west-1" +} + +variable "vpc_availability_zones" { + type = list(string) + default = ["eu-west-1c", "eu-west-1b"] +} + +variable "domain_name" { +} + +variable "vpc_cidr" { + default = "10.100.0.0/16" +} + +variable "private_primary_subnet_cidr" { + default = "10.100.160.0/19" +} + +variable "private_secondary_subnet_cidr" { + default = "10.100.128.0/19" +} + +variable "public_primary_subnet_cidr" { + default = "10.100.64.0/18" +} + +variable "public_secondary_subnet_cidr" { + default = "10.100.0.0/18" +} + +variable "public_db_subnet_cidr_1" { + default = "10.100.192.0/19" +} + +variable "public_db_subnet_cidr_2" { + default = "10.100.224.0/19" +} diff --git a/install/infra/terraform/tools/external-dns/README.md b/install/infra/terraform/tools/cloud-dns-external-dns/README.md similarity index 100% rename from install/infra/terraform/tools/external-dns/README.md rename to install/infra/terraform/tools/cloud-dns-external-dns/README.md diff --git a/install/infra/terraform/tools/cloud-dns-external-dns/main.tf b/install/infra/terraform/tools/cloud-dns-external-dns/main.tf new file mode 100644 index 00000000000000..bb719f259abc5e --- /dev/null +++ b/install/infra/terraform/tools/cloud-dns-external-dns/main.tf @@ -0,0 +1,74 @@ +provider "kubernetes" { + config_path = var.kubeconfig +} + +data local_file "gcp_credentials" { + filename = var.credentials +} + +provider "google" { + credentials = var.credentials + project = var.gcp_project + region = var.gcp_region + zone = var.gcp_zone +} + +provider "helm" { + kubernetes { + config_path = var.kubeconfig + } +} + +#create namespace for external-dns +resource "kubernetes_namespace" "external_dns" { + metadata { + name = "external-dns" + } +} + +resource "kubernetes_secret" "external_dns" { + depends_on = [ + kubernetes_namespace.external_dns + ] + metadata { + name = "external-dns" + namespace = "external-dns" + } + data = { + "credentials.json" = data.local_file.gcp_credentials.content + } +} + +resource "helm_release" "external-dns" { + depends_on = [ + kubernetes_secret.external_dns, + kubernetes_namespace.external_dns + ] + name = "external-dns" + namespace = "external-dns" + create_namespace = true + chart = "external-dns" + repository = "https://charts.bitnami.com/bitnami" + cleanup_on_fail = true + replace = true + set { + name = "provider" + value = "google" + } + set { + name = "google.project" + value = var.gcp_project + } + set { + name = "logFormat" + value = "json" + } + set { + name = "google.serviceAccountSecret" + value = "external-dns" + } + set { + name = "txt-owner-id" + value = var.txt_owner_id + } +} diff --git a/install/infra/terraform/tools/external-dns/variables.tf b/install/infra/terraform/tools/cloud-dns-external-dns/variables.tf similarity index 100% rename from install/infra/terraform/tools/external-dns/variables.tf rename to install/infra/terraform/tools/cloud-dns-external-dns/variables.tf diff --git a/install/infra/terraform/tools/external-dns/main.tf b/install/infra/terraform/tools/external-dns/main.tf index bb719f259abc5e..688f1ae6d1f00d 100644 --- a/install/infra/terraform/tools/external-dns/main.tf +++ b/install/infra/terraform/tools/external-dns/main.tf @@ -1,17 +1,7 @@ -provider "kubernetes" { - config_path = var.kubeconfig -} - -data local_file "gcp_credentials" { - filename = var.credentials -} - -provider "google" { - credentials = var.credentials - project = var.gcp_project - region = var.gcp_region - zone = var.gcp_zone -} +variable settings {} +variable domain_name { default = "test"} +variable kubeconfig { default = "conf"} +variable txt_owner_id { default = "nightly-test" } provider "helm" { kubernetes { @@ -19,56 +9,28 @@ provider "helm" { } } -#create namespace for external-dns -resource "kubernetes_namespace" "external_dns" { - metadata { - name = "external-dns" - } -} - -resource "kubernetes_secret" "external_dns" { - depends_on = [ - kubernetes_namespace.external_dns - ] - metadata { - name = "external-dns" - namespace = "external-dns" - } - data = { - "credentials.json" = data.local_file.gcp_credentials.content - } -} - -resource "helm_release" "external-dns" { - depends_on = [ - kubernetes_secret.external_dns, - kubernetes_namespace.external_dns - ] +# External DNS Deployment using Helm +resource "helm_release" "external_dns" { name = "external-dns" + repository = "https://charts.bitnami.com" + chart = "external-dns" namespace = "external-dns" create_namespace = true - chart = "external-dns" - repository = "https://charts.bitnami.com/bitnami" - cleanup_on_fail = true - replace = true - set { - name = "provider" - value = "google" - } - set { - name = "google.project" - value = var.gcp_project - } - set { - name = "logFormat" - value = "json" - } + set { - name = "google.serviceAccountSecret" - value = "external-dns" + name = "domainFilters[0]" + value = var.domain_name } set { name = "txt-owner-id" value = var.txt_owner_id } + + dynamic "set" { + for_each = var.settings + content { + name = set.value["name"] + value = set.value["value"] + } + } } diff --git a/install/infra/terraform/tools/issuer/azure/main.tf b/install/infra/terraform/tools/issuer/main.tf similarity index 67% rename from install/infra/terraform/tools/issuer/azure/main.tf rename to install/infra/terraform/tools/issuer/main.tf index e76a6d9761de05..079b6cb12ecd35 100644 --- a/install/infra/terraform/tools/issuer/azure/main.tf +++ b/install/infra/terraform/tools/issuer/main.tf @@ -2,6 +2,18 @@ provider "kubernetes" { config_path = var.kubeconfig } +resource "kubernetes_secret" "dns_solver" { + count = var.secretAccessKey == null ? 0 : 1 + metadata { + name = "route53-credentials" + namespace = "cert-manager" + } + data = { + "secret-access-key" = var.secretAccessKey + } +} + + resource "kubernetes_manifest" "clusterissuer_gitpod" { manifest = { "apiVersion" = "cert-manager.io/v1" @@ -19,7 +31,7 @@ resource "kubernetes_manifest" "clusterissuer_gitpod" { "solvers" = [ { "dns01" = { - "azureDNS" = var.cert_manager_issuer + "${var.issuer_name}" = "${var.cert_manager_issuer}" } } ] diff --git a/install/infra/terraform/tools/issuer/azure/variables.tf b/install/infra/terraform/tools/issuer/variables.tf similarity index 62% rename from install/infra/terraform/tools/issuer/azure/variables.tf rename to install/infra/terraform/tools/issuer/variables.tf index 2960f621b2a55e..c9622796efa386 100644 --- a/install/infra/terraform/tools/issuer/azure/variables.tf +++ b/install/infra/terraform/tools/issuer/variables.tf @@ -6,3 +6,11 @@ variable "kubeconfig" { variable "cert_manager_issuer" { default = null } + +variable "secretAccessKey" { + default = null +} + +variable "issuer_name" { + default = "route53" +} diff --git a/install/tests/Makefile b/install/tests/Makefile index 05b0b2eb81cb53..4151b92732bbb6 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -29,33 +29,38 @@ gke-standard-cluster: terraform apply -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve .PHONY: -## aks-standard-cluster: Creates an AKS cluster -aks-standard-cluster: +## eks-standard-cluster: Creates an EKS cluster +eks-standard-cluster: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.aks -var k8s_flavor="aks" -var kubeconfig=${KUBECONFIG} --auto-approve + terraform apply -target=module.eks -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done creating EKS cluster" .PHONY: -## azure-external-dns: Sets up external-dns with azure provider -azure-external-dns: -azure-external-dns: +## aks-standard-cluster: Creates an AKS cluster +aks-standard-cluster: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve + terraform apply -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done creating AKS cluster" +cloud ?= azure .PHONY: ## add-ns-record: Adds NS record for subdomain under gitpod-selfhosted.com add-ns-record: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve + terraform apply -target=module.$(cloud)-add-dns-record -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done adding NS record" +cloud ?= azure .PHONY: -## azure-issuer: Creates a cluster issuer with AD access -azure-issuer: +## cluster-issuer: Creates a cluster issuer for the correspondign provider +cluster-issuer: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve + terraform apply -target=module.$(cloud)-issuer -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done creating cluster issuer" .PHONY: ## k3s-standard-cluster: Creates a K3S cluster on GCP with one master and 1 worker node @@ -63,21 +68,29 @@ k3s-standard-cluster: terraform init --upgrade && \ terraform workspace new $(TF_VAR_TEST_ID) || terraform workspace select $(TF_VAR_TEST_ID) && \ terraform apply -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve - -CLUSTER_ISSUER_CLOUD_DNS := "./manifests/gcp-issuer.yaml" + @echo "Done creating k3s cluster" .PHONY: ## cert-manager: Installs cert-manager, optionally create secret for cloud-dns access cert-manager: terraform workspace select $(TF_VAR_TEST_ID) && \ terraform apply -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done installing cert-manager" .PHONY: ## managed-dns: Installs external-dns, and setup up CloudDNS access managed-dns: check-env-sub-domain terraform workspace select $(TF_VAR_TEST_ID) && \ - terraform apply -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve && \ - kubectl --kubeconfig=${KUBECONFIG} apply -f ${CLUSTER_ISSUER_CLOUD_DNS} + terraform apply -target=module.clouddns-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done created GCP managed DNS" + +cloud ?= "azure" +.PHONY: +## external-dns: Installs external-dns +external-dns: check-env-sub-domain + terraform workspace select $(TF_VAR_TEST_ID) && \ + terraform apply -target=module.$(cloud)-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done creating externaldns for ${cloud}" .PHONY: ## get-kubeconfig: Returns KUBECONFIG of a just created cluster @@ -111,24 +124,47 @@ get-config-gcp-db: envsubst '$${TF_VAR_TEST_ID}' < tmp_4_config.yml > tmp_5_config.yml yq m -i tmp_config.yml tmp_5_config.yml +get-config-azure-registry: + export SERVER=$$(terraform output -json azure_registry | yq r - 'server') && \ + export PASSWORD=$$(terraform output -json azure_registry | yq r - 'password') && \ + export USERNAME=$$(terraform output -json azure_registry | yq r - 'username') && \ + envsubst < ./manifests/kots-config-azure-registry.yaml > tmp_2_config.yml + yq m -i tmp_config.yml tmp_2_config.yml + get-config-azure-storage: - export PASSWORD=$$(terraform output -json storage | yq r - 'password') && \ - export USERNAME=$$(terraform output -json storage | yq r - 'username') && \ + export PASSWORD=$$(terraform output -json azure_storage | yq r - 'password') && \ + export USERNAME=$$(terraform output -json azure_storage | yq r - 'username') && \ envsubst < ./manifests/kots-config-azure-storage.yaml > tmp_2_config.yml yq m -i tmp_config.yml tmp_2_config.yml get-config-azure-db: - export DBHOST=$$(terraform output -json database | yq r - 'host') && \ - export DBPASS=$$(terraform output -json database | yq r - 'password') && \ - export DBUSER=$$(terraform output -json database | yq r - 'username') && \ + export DBHOST=$$(terraform output -json azure_database | yq r - 'host') && \ + export DBPASS=$$(terraform output -json azure_database | yq r - 'password') && \ + export DBUSER=$$(terraform output -json azure_database | yq r - 'username') && \ envsubst < ./manifests/kots-config-azure-db.yaml > tmp_2_config.yml yq m -i tmp_config.yml tmp_2_config.yml -get-config-azure-registry: - export SERVER=$$(terraform output -json registry | yq r - 'server') && \ - export PASSWORD=$$(terraform output -json registry | yq r - 'password') && \ - export USERNAME=$$(terraform output -json registry | yq r - 'username') && \ - envsubst < ./manifests/kots-config-azure-registry.yaml > tmp_2_config.yml +get-config-aws-db: + export DBHOST=$$(terraform output -json aws_database | yq r - 'host') && \ + export DBPASS=$$(terraform output -json aws_database | yq r - 'password') && \ + export DBUSER=$$(terraform output -json aws_database | yq r - 'username') && \ + envsubst < ./manifests/kots-config-aws-db.yaml > tmp_2_config.yml + yq m -i tmp_config.yml tmp_2_config.yml + +get-config-aws-storage: + export REGION=$$(terraform output -json aws_storage | yq r - 'region') && \ + export ENDPOINT=$$(terraform output -json aws_storage | yq r - 'endpoint') && \ + export BUCKET=$$(terraform output -json aws_storage | yq r - 'bucket_name') && \ + export S3_ACCESS_KEY_ID=$$(terraform output -json aws_storage | yq r - 'access_key_id') && \ + export S3_SECRET_ACCESS_KEY=$$(terraform output -json aws_storage | yq r - 'secret_access_key') && \ + envsubst < ./manifests/kots-config-aws-storage.yaml > tmp_2_config.yml + yq m -i tmp_config.yml tmp_2_config.yml + +get-config-aws-registry: + export SERVER=$$(terraform output -json aws_registry | yq r - 'server' | cut -d / -f 1) && \ + export PASSWORD=$$(terraform output -json aws_registry | yq r - 'password') && \ + export USERNAME=$$(terraform output -json aws_registry | yq r - 'username') && \ + envsubst < ./manifests/kots-config-aws-registry.yaml > tmp_2_config.yml yq m -i tmp_config.yml tmp_2_config.yml storage ?= incluster @@ -173,38 +209,35 @@ check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain run-tests: ./tests.sh ${KUBECONFIG} -kots-uprgade: +kots-upgrade: @echo "Upgrade gitpod KOTS app to latest" kubectl kots upstream upgrade --kubeconfig=${KUBECONFIG} gitpod -n gitpod --deploy -cleanup: destroy-gcp-externaldns destroy-gcpns destroy-aks-edns destroy-aks-issuer destroy-certmanager destroy-k3s destroy-gke destroy-aks +cloud ?= gcp +cleanup: destroy-$(cloud) select-workspace: terraform workspace select $(TF_VAR_TEST_ID) -destroy-gcp-externaldns: select-workspace - ls ${KUBECONFIG} && terraform destroy -target=module.externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" - -destroy-certmanager: select-workspace +destroy-gcp: select-workspace + ls ${KUBECONFIG} && terraform destroy -target=module.clouddns-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" - -destroy-gcpns: select-workspace - terraform destroy -target=module.add_gcp_nameservers -var kubeconfig=${KUBECONFIG} --auto-approve - -destroy-aks-edns: select-workspace - ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" - -destroy-aks-issuer: select-workspace - ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" - -destroy-gke: select-workspace + terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve terraform destroy -target=module.gke -var kubeconfig=${KUBECONFIG} --auto-approve -destroy-k3s: select-workspace - terraform destroy -target=module.k3s -var kubeconfig=${KUBECONFIG} --auto-approve +destroy-aws: select-workspace + terraform destroy -target=module.aws-add-dns-record -var kubeconfig=${KUBECONFIG} --auto-approve + ls ${KUBECONFIG} && terraform destroy -target=module.aws-issuer -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + ls ${KUBECONFIG} && terraform destroy -target=module.aws-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + terraform destroy -target=module.eks -var kubeconfig=${KUBECONFIG} --auto-approve -destroy-aks: select-workspace - terraform destroy -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve +destroy-azure: select-workspace + ls ${KUBECONFIG} && terraform destroy -target=module.azure-issuer -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + terraform destroy -target=module.azure-add-dns-record -var kubeconfig=${KUBECONFIG} --auto-approve + ls ${KUBECONFIG} && terraform destroy -target=module.azure-externaldns -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + ls ${KUBECONFIG} && terraform destroy -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve || echo "No kubeconfig file" + [ -z "$$ARM_SUBSCRIPTION_ID" ] || terraform destroy -target=module.aks -var kubeconfig=${KUBECONFIG} --auto-approve get-results: @echo "If you have gotten this far, it means your setup succeeded" diff --git a/install/tests/main.tf b/install/tests/main.tf index 2766913b761872..38e84b448a9b8d 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -1,8 +1,6 @@ variable "kubeconfig" { } variable "TEST_ID" { default = "nightly" } -variable "k8s_flavor" { default = "gke" } - # We store the state always in a GCS bucket terraform { backend "gcs" { @@ -41,9 +39,23 @@ module "k3s" { domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" } +module "gcp-issuer" { + source = "../infra/terraform/tools/issuer" + kubeconfig = var.kubeconfig + issuer_name = "cloudDNS" + cert_manager_issuer = { + project = "dns-for-playgrounds" + serviceAccountSecretRef = { + name = "clouddns-dns01-solver" + key = "keys.json" + } + } +} + + module "aks" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/aks?ref=main" # we can later use tags here - source = "../infra/terraform/aks" + source = "../infra/terraform/aks" domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" enable_airgapped = false @@ -52,40 +64,66 @@ module "aks" { enable_external_storage = true dns_enabled = true workspace_name = var.TEST_ID + kubeconfig = var.kubeconfig +} + +module "eks" { + source = "../infra/terraform/eks" + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + cluster_name = var.TEST_ID + region = "eu-west-1" + vpc_availability_zones = ["eu-west-1c", "eu-west-1b"] + image_id = "ami-0793b4124359a6ad7" // this AMI is regional + kubeconfig = var.kubeconfig } module "certmanager" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cert-manager?ref=main" - source = "../infra/terraform/tools/cert-manager" + source = "../infra/terraform/tools/cert-manager" - kubeconfig = var.kubeconfig - credentials = var.dns_sa_creds + kubeconfig = var.kubeconfig + credentials = var.dns_sa_creds } -module "externaldns" { +module "clouddns-externaldns" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/external-dns?ref=main" - source = "../infra/terraform/tools/external-dns" - kubeconfig = var.kubeconfig - credentials = var.dns_sa_creds - txt_owner_id = var.TEST_ID + source = "../infra/terraform/tools/cloud-dns-external-dns" + kubeconfig = var.kubeconfig + credentials = var.dns_sa_creds } module "azure-externaldns" { - source = "../infra/terraform/tools/azure-external-dns" - kubeconfig = var.kubeconfig - settings = module.aks.external_dns_settings - domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" - txt_owner_id = var.TEST_ID + source = "../infra/terraform/tools/external-dns" + kubeconfig = var.kubeconfig + settings = module.aks.external_dns_settings + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + txt_owner_id = var.TEST_ID +} + +module "aws-externaldns" { + source = "../infra/terraform/tools/external-dns" + kubeconfig = var.kubeconfig + settings = module.eks.external_dns_settings + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + txt_owner_id = var.TEST_ID } module "azure-issuer" { - source = "../infra/terraform/tools/issuer/azure" - kubeconfig = var.kubeconfig + source = "../infra/terraform/tools/issuer" + kubeconfig = var.kubeconfig cert_manager_issuer = module.aks.cert_manager_issuer + issuer_name = "azureDNS" +} + +module "aws-issuer" { + source = "../infra/terraform/tools/issuer" + kubeconfig = var.kubeconfig + cert_manager_issuer = module.eks.cert_manager_issuer + secretAccessKey = module.eks.secretAccessKey + issuer_name = "route53" } -module "add_gcp_nameservers" { - # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cloud-dns-ns?ref=main" +module "azure-add-dns-record" { source = "../infra/terraform/tools/cloud-dns-ns" credentials = var.dns_sa_creds nameservers = module.aks.domain_nameservers @@ -93,3 +131,12 @@ module "add_gcp_nameservers" { managed_dns_zone = "gitpod-self-hosted-com" domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" } + +module "aws-add-dns-record" { + source = "../infra/terraform/tools/cloud-dns-ns" + credentials = var.dns_sa_creds + nameservers = module.eks.domain_nameservers + dns_project = "dns-for-playgrounds" + managed_dns_zone = "gitpod-self-hosted-com" + domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" +} diff --git a/install/tests/manifests/kots-config-aws-db.yaml b/install/tests/manifests/kots-config-aws-db.yaml new file mode 100644 index 00000000000000..1e113404ae2907 --- /dev/null +++ b/install/tests/manifests/kots-config-aws-db.yaml @@ -0,0 +1,16 @@ +apiVersion: kots.io/v1beta1 +kind: ConfigValues +spec: + values: + db_incluster: + value: "0" + data: "db_incluster" + db_host: + value: ${DBHOST} + data: "db_host" + db_username: + value: ${DBUSER} + data: "db_username" + db_password: + value: ${DBPASS} + data: "db_password" diff --git a/install/tests/manifests/kots-config-aws-registry.yaml b/install/tests/manifests/kots-config-aws-registry.yaml new file mode 100644 index 00000000000000..1442501e3f07b5 --- /dev/null +++ b/install/tests/manifests/kots-config-aws-registry.yaml @@ -0,0 +1,16 @@ +apiVersion: kots.io/v1beta1 +kind: ConfigValues +spec: + values: + reg_incluster: + value: "0" + data: "reg_incluster" + reg_url: + value: ${SERVER} + data: "reg_url" + reg_username: + value: ${USERNAME} + data: "reg_username" + reg_password: + value: ${PASSWORD} + data: "reg_password" diff --git a/install/tests/manifests/kots-config-aws-storage.yaml b/install/tests/manifests/kots-config-aws-storage.yaml new file mode 100644 index 00000000000000..af7896eff483d5 --- /dev/null +++ b/install/tests/manifests/kots-config-aws-storage.yaml @@ -0,0 +1,22 @@ +apiVersion: kots.io/v1beta1 +kind: ConfigValues +spec: + values: + store_provider: + value: "s3" + data: "store_provider" + store_region: + value: "${REGION}" + data: "store_region" + store_s3_endpoint: + value: "${ENDPOINT}" + data: "store_s3_endpoint" + store_s3_bucket: + value: "${BUCKET}" + data: "store_s3_bucket" + store_s3_access_key_id: + value: "${S3_ACCESS_KEY_ID}" + data: "store_s3_access_key_id" + store_s3_secret_access_key: + value: "${S3_SECRET_ACCESS_KEY}" + data: "store_s3_secret_access_key" diff --git a/install/tests/output.tf b/install/tests/output.tf index 3982885ebec276..7af88c204d704a 100644 --- a/install/tests/output.tf +++ b/install/tests/output.tf @@ -1,18 +1,29 @@ -locals { - cloud = var.k8s_flavor == "aks" ? module.aks : null +output "aws_storage" { + sensitive = true + value = try(module.eks.storage, null) +} + +output "aws_registry" { + sensitive = true + value = try(module.eks.registry, null) +} + +output "aws_database" { + sensitive = true + value = try(module.eks.database, null) } -output "storage" { +output "azure_database" { sensitive = true - value = try(lookup(local.cloud, "storage"), {}) + value = try(module.aks.database, null) } -output "registry" { +output "azure_registry" { sensitive = true - value = try(lookup(local.cloud, "registry"), {}) + value = try(module.aks.registry, null) } -output "database" { +output "azure_storage" { sensitive = true - value = try(lookup(local.cloud, "database"), {}) + value = try(module.aks.storage, null) } From e0b8659f9afb29c499229ad02a44430e6c3750d6 Mon Sep 17 00:00:00 2001 From: Nandaja Varma Date: Fri, 1 Jul 2022 07:21:57 +0000 Subject: [PATCH 3/3] fix review suggestions --- .werft/eks-installer-tests.yaml | 3 --- .werft/installer-tests.ts | 4 ++-- install/infra/terraform/eks/database.tf | 8 +++++++- install/infra/terraform/eks/dns.tf | 2 +- install/infra/terraform/eks/output.tf | 6 +++--- install/infra/terraform/eks/variables.tf | 1 - install/tests/Makefile | 6 +++--- install/tests/main.tf | 16 ++++++++-------- install/tests/manifests/kots-config-aws-db.yaml | 2 +- install/tests/manifests/kots-config.yaml | 2 +- 10 files changed, 26 insertions(+), 24 deletions(-) diff --git a/.werft/eks-installer-tests.yaml b/.werft/eks-installer-tests.yaml index eaf803b836f308..2fefc488d699b4 100644 --- a/.werft/eks-installer-tests.yaml +++ b/.werft/eks-installer-tests.yaml @@ -19,9 +19,6 @@ pod: - name: sh-playground-dns-perm secret: secretName: sh-playground-dns-perm - - name: sh-aks-perm - secret: - secretName: aks-credentials containers: - name: nightly-test image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0 diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts index 8e9bf01d0f4be3..18cffef121ecb8 100644 --- a/.werft/installer-tests.ts +++ b/.werft/installer-tests.ts @@ -185,7 +185,7 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { ADD_NS_RECORD: { phase: "add-ns-record", makeTarget: `add-ns-record cloud=${cloud}`, - description: "Adds NS record for subdomain under gitpod-self-hosted.com", + description: "Adds NS record for subdomain under tests.gitpod-self-hosted.com", }, INSTALL_GITPOD_IGNORE_PREFLIGHTS: { phase: "install-gitpod-without-preflights", @@ -282,7 +282,7 @@ function cleanup() { werft.phase(phase, "Destroying all the created resources"); const response = exec(`make -C ${makefilePath} cleanup cloud=${cloud}`, { - slice: "run-terrafrom-destroy", + slice: "run-terraform-destroy", dontCheckRc: true, }); diff --git a/install/infra/terraform/eks/database.tf b/install/infra/terraform/eks/database.tf index 4d759db54906c9..97265f2f5ea00e 100644 --- a/install/infra/terraform/eks/database.tf +++ b/install/infra/terraform/eks/database.tf @@ -1,3 +1,9 @@ +resource "random_password" "password" { + length = 16 + special = true + override_special = "!#$%&*()-_=+[]{}<>:?" +} + resource "aws_db_subnet_group" "gitpod_subnets" { name = "db-sg-${var.cluster_name}" subnet_ids = [module.vpc.public_subnets[2], module.vpc.public_subnets[3]] @@ -32,7 +38,7 @@ resource "aws_db_instance" "gitpod" { identifier = "db-${var.cluster_name}" name = "gitpod" username = "gitpod" - password = "gitpod-qwat" + password = random_password.password.result parameter_group_name = "default.mysql5.7" db_subnet_group_name = aws_db_subnet_group.gitpod_subnets.name skip_final_snapshot = true diff --git a/install/infra/terraform/eks/dns.tf b/install/infra/terraform/eks/dns.tf index 1c1c62e8a39626..bd699ac9b01b6d 100644 --- a/install/infra/terraform/eks/dns.tf +++ b/install/infra/terraform/eks/dns.tf @@ -54,7 +54,7 @@ resource "aws_route53_record" "cert_validation_3" { type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[3].resource_record_type zone_id = "${resource.aws_route53_zone.gitpod.zone_id}" records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[3].resource_record_value] - ttl = 6 + ttl = 60 } resource "aws_iam_policy" "gitpod" { diff --git a/install/infra/terraform/eks/output.tf b/install/infra/terraform/eks/output.tf index 151443ca504021..4e34598e907e12 100644 --- a/install/infra/terraform/eks/output.tf +++ b/install/infra/terraform/eks/output.tf @@ -6,7 +6,7 @@ output "external_dns_settings" { }, { "name" = "aws.region", - "value" = "eu-west-1" + "value" = var.region }, { "name" = "aws.credentials.secretKey", @@ -26,7 +26,7 @@ output "secretAccessKey" { output "cert_manager_issuer" { value = try({ - region = "eu-west-1" + region = var.region secretAccessKeySecretRef = { name = "route53-credentials" key = "secret-access-key" @@ -45,7 +45,7 @@ output "database" { sensitive = true value = try({ host = "${aws_db_instance.gitpod.address}" - password = "gitpod-qwat" # FIXME hardcoded at this point + password = random_password.password.result port = 3306 username = "${aws_db_instance.gitpod.username}" }, {}) diff --git a/install/infra/terraform/eks/variables.tf b/install/infra/terraform/eks/variables.tf index ccbc32f5207217..8b7528ef17c567 100644 --- a/install/infra/terraform/eks/variables.tf +++ b/install/infra/terraform/eks/variables.tf @@ -1,7 +1,6 @@ variable "cluster_name" { type = string description = "EKS cluster name." - default = "nan-eks-3" } variable "kubeconfig" { type = string diff --git a/install/tests/Makefile b/install/tests/Makefile index 4151b92732bbb6..639f7201b966f2 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -203,8 +203,8 @@ check-kots-app: kubectl kots get --kubeconfig=${KUBECONFIG} app gitpod -n gitpod | grep gitpod | awk '{print $$2}' | grep "ready" || { echo "Gitpod is not ready"; exit 1; } check-gitpod-installation: delete-cm-setup check-kots-app check-env-sub-domain - @echo "Curling http://${TF_VAR_TEST_ID}.gitpod-self-hosted.com/api/version" - curl -i -X GET http://${TF_VAR_TEST_ID}.gitpod-self-hosted.com/api/version || { echo "Curling Gitpod endpoint failed"; exit 1; } + @echo "Curling http://${TF_VAR_TEST_ID}.tests.gitpod-self-hosted.com/api/version" + curl -i -X GET http://${TF_VAR_TEST_ID}.tests.gitpod-self-hosted.com/api/version || { echo "Curling Gitpod endpoint failed"; exit 1; } run-tests: ./tests.sh ${KUBECONFIG} @@ -241,7 +241,7 @@ destroy-azure: select-workspace get-results: @echo "If you have gotten this far, it means your setup succeeded" - @echo "The IP address of you setup is "$(TF_VAR_TEST_ID).gitpod-self-hosted.com"" + @echo "The IP address of you setup is "$(TF_VAR_TEST_ID).tests.gitpod-self-hosted.com"" @echo "Following is the KUBECONFIG you can use to connect to the cluster:" @cat ${KUBECONFIG} diff --git a/install/tests/main.tf b/install/tests/main.tf index 38e84b448a9b8d..0214db751dcb15 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -36,7 +36,7 @@ module "k3s" { dns_sa_creds = var.dns_sa_creds dns_project = "dns-for-playgrounds" managed_dns_zone = "gitpod-self-hosted-com" - domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" } module "gcp-issuer" { @@ -57,7 +57,7 @@ module "aks" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/aks?ref=main" # we can later use tags here source = "../infra/terraform/aks" - domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" enable_airgapped = false enable_external_database = true enable_external_registry = true @@ -69,7 +69,7 @@ module "aks" { module "eks" { source = "../infra/terraform/eks" - domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" cluster_name = var.TEST_ID region = "eu-west-1" vpc_availability_zones = ["eu-west-1c", "eu-west-1b"] @@ -96,7 +96,7 @@ module "azure-externaldns" { source = "../infra/terraform/tools/external-dns" kubeconfig = var.kubeconfig settings = module.aks.external_dns_settings - domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" txt_owner_id = var.TEST_ID } @@ -104,7 +104,7 @@ module "aws-externaldns" { source = "../infra/terraform/tools/external-dns" kubeconfig = var.kubeconfig settings = module.eks.external_dns_settings - domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" txt_owner_id = var.TEST_ID } @@ -129,7 +129,7 @@ module "azure-add-dns-record" { nameservers = module.aks.domain_nameservers dns_project = "dns-for-playgrounds" managed_dns_zone = "gitpod-self-hosted-com" - domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" } module "aws-add-dns-record" { @@ -137,6 +137,6 @@ module "aws-add-dns-record" { credentials = var.dns_sa_creds nameservers = module.eks.domain_nameservers dns_project = "dns-for-playgrounds" - managed_dns_zone = "gitpod-self-hosted-com" - domain_name = "${var.TEST_ID}.gitpod-self-hosted.com" + managed_dns_zone = "tests-gitpod-self-hosted-com" + domain_name = "${var.TEST_ID}.tests.gitpod-self-hosted.com" } diff --git a/install/tests/manifests/kots-config-aws-db.yaml b/install/tests/manifests/kots-config-aws-db.yaml index 1e113404ae2907..059934129e33f9 100644 --- a/install/tests/manifests/kots-config-aws-db.yaml +++ b/install/tests/manifests/kots-config-aws-db.yaml @@ -12,5 +12,5 @@ spec: value: ${DBUSER} data: "db_username" db_password: - value: ${DBPASS} + value: "${DBPASS}" data: "db_password" diff --git a/install/tests/manifests/kots-config.yaml b/install/tests/manifests/kots-config.yaml index f6f7fd0383a899..b24aac58b1c11e 100644 --- a/install/tests/manifests/kots-config.yaml +++ b/install/tests/manifests/kots-config.yaml @@ -3,7 +3,7 @@ kind: ConfigValues spec: values: domain: - value: "${TF_VAR_TEST_ID}.gitpod-self-hosted.com" + value: "${TF_VAR_TEST_ID}.tests.gitpod-self-hosted.com" data: "domain" ssh_gateway: value: "1"