Skip to content

Commit 2f31b99

Browse files
nandajavarmanandajavarma
committed
[installer] Add EKS installer test
1 parent 6d59df5 commit 2f31b99

File tree

24 files changed

+948
-147
lines changed

24 files changed

+948
-147
lines changed

‎.werft/eks-installer-tests.yaml

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/eks-installer-tests.yaml -a debug=true`
2+
pod:
3+
serviceAccount: werft
4+
affinity:
5+
nodeAffinity:
6+
requiredDuringSchedulingIgnoredDuringExecution:
7+
nodeSelectorTerms:
8+
- matchExpressions:
9+
- key: dev/workload
10+
operator: In
11+
values:
12+
- "builds"
13+
securityContext:
14+
runAsUser: 0
15+
volumes:
16+
- name: sh-playground-sa-perm
17+
secret:
18+
secretName: sh-playground-sa-perm
19+
- name: sh-playground-dns-perm
20+
secret:
21+
secretName: sh-playground-dns-perm
22+
- name: sh-aks-perm
23+
secret:
24+
secretName: aks-credentials
25+
containers:
26+
- name: nightly-test
27+
image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:cw-werft-cred.0
28+
workingDir: /workspace
29+
imagePullPolicy: Always
30+
volumeMounts:
31+
- name: sh-playground-sa-perm
32+
mountPath: /mnt/secrets/sh-playground-sa-perm
33+
- name: sh-playground-dns-perm # this sa is used for the DNS management
34+
mountPath: /mnt/secrets/sh-playground-dns-perm
35+
env:
36+
- name: AWS_ACCESS_KEY_ID
37+
valueFrom:
38+
secretKeyRef:
39+
name: aws-credentials
40+
key: aws-access-key
41+
- name: AWS_SECRET_ACCESS_KEY
42+
valueFrom:
43+
secretKeyRef:
44+
name: aws-credentials
45+
key: aws-secret-key
46+
- name: AWS_REGION
47+
valueFrom:
48+
secretKeyRef:
49+
name: aws-credentials
50+
key: aws-region
51+
- name: WERFT_HOST
52+
value: "werft.werft.svc.cluster.local:7777"
53+
- name: GOOGLE_APPLICATION_CREDENTIALS
54+
value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
55+
- name: WERFT_K8S_NAMESPACE
56+
value: "werft"
57+
- name: WERFT_K8S_LABEL
58+
value: "component=werft"
59+
- name: TF_VAR_sa_creds
60+
value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
61+
- name: TF_VAR_dns_sa_creds
62+
value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json"
63+
- name: NODENAME
64+
valueFrom:
65+
fieldRef:
66+
fieldPath: spec.nodeName
67+
command:
68+
- bash
69+
- -c
70+
- |
71+
sleep 1
72+
set -Eeuo pipefail
73+
74+
sudo chown -R gitpod:gitpod /workspace
75+
sudo apt update && apt install gettext-base
76+
77+
export TF_VAR_TEST_ID="$(echo $RANDOM | md5sum | head -c 5; echo)"
78+
79+
(cd .werft && yarn install && mv node_modules ..) | werft log slice prep
80+
printf '{{ toJson . }}' > context.json
81+
82+
npx ts-node .werft/installer-tests.ts "STANDARD_EKS_TEST"
83+
# The bit below makes this a cron job
84+
# plugins:
85+
# cron: "15 3 * * *"

‎.werft/installer-tests.ts

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,8 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
9292
PHASES: [
9393
"STANDARD_AKS_CLUSTER",
9494
"CERT_MANAGER",
95-
"AZURE_ISSUER",
96-
"AZURE_EXTERNALDNS",
95+
"CLUSTER_ISSUER",
96+
"EXTERNALDNS",
9797
"ADD_NS_RECORD",
9898
"GENERATE_KOTS_CONFIG",
9999
"INSTALL_GITPOD",
@@ -103,6 +103,23 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
103103
"DESTROY",
104104
],
105105
},
106+
STANDARD_EKS_TEST: {
107+
CLOUD: "aws",
108+
DESCRIPTION: "Create an EKS cluster",
109+
PHASES: [
110+
"STANDARD_EKS_CLUSTER",
111+
"CERT_MANAGER",
112+
"EXTERNALDNS",
113+
"CLUSTER_ISSUER",
114+
"ADD_NS_RECORD",
115+
"GENERATE_KOTS_CONFIG",
116+
"RESULTS",
117+
"INSTALL_GITPOD",
118+
"CHECK_INSTALLATION",
119+
// "RUN_INTEGRATION_TESTS",
120+
"DESTROY",
121+
],
122+
},
106123
};
107124

108125
const config: TestConfig = TEST_CONFIGURATIONS[testConfig];
@@ -128,6 +145,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
128145
makeTarget: "aks-standard-cluster",
129146
description: "Creating an aks cluster(azure)",
130147
},
148+
STANDARD_EKS_CLUSTER: {
149+
phase: "create-std-eks-cluster",
150+
makeTarget: "eks-standard-cluster",
151+
description: "Creating a EKS cluster with 1 nodepool each for workspace and server",
152+
},
131153
CERT_MANAGER: {
132154
phase: "setup-cert-manager",
133155
makeTarget: "cert-manager",
@@ -146,18 +168,18 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
146168
)} db=${randomize("db", cloud)}`,
147169
description: `Generate KOTS Config file`,
148170
},
149-
AZURE_ISSUER: {
150-
phase: "setup-azure-cluster-issuer",
171+
CLUSTER_ISSUER: {
172+
phase: `setup-azure-cluster-issuer cloud=${cloud}`,
151173
makeTarget: "azure-issuer",
152174
description: "Deploys ClusterIssuer for azure",
153175
},
154-
AZURE_EXTERNALDNS: {
155-
phase: "azure-external-dns",
156-
makeTarget: "azure-external-dns",
157-
description: "Deploys external-dns with azure provider",
176+
EXTERNALDNS: {
177+
phase: "external-dns",
178+
makeTarget: `external-dns cloud=${cloud}`,
179+
description: `Deploys external-dns with ${cloud} provider`,
158180
},
159181
ADD_NS_RECORD: {
160-
phase: "add-ns-record",
182+
phase: `add-ns-record cloud=${cloud}`,
161183
makeTarget: "add-ns-record",
162184
description: "Adds NS record for subdomain under gitpod-self-hosted.com",
163185
},
@@ -189,7 +211,7 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
189211
},
190212
DESTROY: {
191213
phase: "destroy",
192-
makeTarget: "cleanup",
214+
makeTarget: `cleanup cloud=${cloud}`,
193215
description: "Destroy the created infrastucture",
194216
},
195217
RESULTS: {
@@ -254,7 +276,10 @@ function cleanup() {
254276
const phase = "destroy-infrastructure";
255277
werft.phase(phase, "Destroying all the created resources");
256278

257-
const response = exec(`make -C ${makefilePath} cleanup`, { slice: "run-terrafrom-destroy", dontCheckRc: true });
279+
const response = exec(`make -C ${makefilePath} cleanup cloud=${cloud}`, {
280+
slice: "run-terrafrom-destroy",
281+
dontCheckRc: true,
282+
});
258283

259284
// if the destroy command fail, we check if any resources are pending to be removed
260285
// if nothing is yet to be cleaned, we return with success

‎install/infra/terraform/aks/output.tf

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,32 @@ output "external_dns_secrets" {
3636
}
3737

3838
output "external_dns_settings" {
39-
value = {
40-
provider = "azure"
41-
"azure.resourceGroup" = azurerm_resource_group.gitpod.name
42-
"azure.subscriptionId" = data.azurerm_client_config.current.subscription_id
43-
"azure.tenantId" = data.azurerm_client_config.current.tenant_id
44-
"azure.useManagedIdentityExtension" = true
45-
"azure.userAssignedIdentityID" = azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id
46-
}
39+
value = [
40+
{
41+
"name": "provider",
42+
"value": "azure"
43+
},
44+
{
45+
"name": "azure.resourceGroup",
46+
"value": azurerm_resource_group.gitpod.name,
47+
},
48+
{
49+
"name": "azure.subscriptionId",
50+
"value": data.azurerm_client_config.current.subscription_id,
51+
},
52+
{
53+
"name": "azure.tenantId",
54+
"value": data.azurerm_client_config.current.tenant_id,
55+
},
56+
{
57+
"name": "azure.useManagedIdentityExtension",
58+
"value": true
59+
},
60+
{
61+
"name": "azure.userAssignedIdentityID",
62+
"value": azurerm_kubernetes_cluster.k8s.kubelet_identity.0.client_id
63+
},
64+
]
4765
}
4866

4967
output "k8s_connection" {

‎install/infra/terraform/eks/database.tf

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
resource "aws_db_subnet_group" "gitpod_subnets" {
2+
name = "db-sg-${var.cluster_name}"
3+
subnet_ids = [module.vpc.public_subnets[2], module.vpc.public_subnets[3]]
4+
}
5+
6+
resource "aws_security_group" "rdssg" {
7+
name = "dh-sg-${var.cluster_name}"
8+
vpc_id = module.vpc.vpc_id
9+
10+
ingress {
11+
from_port = 0
12+
to_port = 3306
13+
protocol = "tcp"
14+
cidr_blocks = ["0.0.0.0/0"]
15+
}
16+
17+
egress {
18+
from_port = 0
19+
to_port = 0
20+
protocol = "-1"
21+
cidr_blocks = ["0.0.0.0/0"]
22+
}
23+
}
24+
25+
resource "aws_db_instance" "gitpod" {
26+
allocated_storage = 10
27+
max_allocated_storage = 100
28+
engine = "mysql"
29+
engine_version = "5.7"
30+
instance_class = "db.t3.micro"
31+
vpc_security_group_ids = [ aws_security_group.rdssg.id ]
32+
identifier = "db-${var.cluster_name}"
33+
name = "gitpod"
34+
username = "gitpod"
35+
password = "gitpod-qwat"
36+
parameter_group_name = "default.mysql5.7"
37+
db_subnet_group_name = aws_db_subnet_group.gitpod_subnets.name
38+
skip_final_snapshot = true
39+
publicly_accessible = true
40+
}

‎install/infra/terraform/eks/dns.tf

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
resource "aws_route53_zone" "gitpod" {
2+
force_destroy = true
3+
name = var.domain_name
4+
5+
tags = {
6+
Environment = "test"
7+
}
8+
}
9+
10+
# This creates an SSL certificate
11+
resource "aws_acm_certificate" "gitpod" {
12+
domain_name = var.domain_name
13+
subject_alternative_names = ["*.ws.${var.domain_name}", "ws.${var.domain_name}", "*.${var.domain_name}"]
14+
validation_method = "DNS"
15+
}
16+
17+
# This is a DNS record for the ACM certificate validation to prove we own the domain
18+
#
19+
# This example, we make an assumption that the certificate is for a single domain name so can just use the first value of the
20+
# domain_validation_options. It allows the terraform to apply without having to be targeted.
21+
# This is somewhat less complex than the example at https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/acm_certificate_validation
22+
# - that above example, won't apply without targeting
23+
24+
resource "aws_route53_record" "cert_validation" {
25+
allow_overwrite = true
26+
name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[0].resource_record_name
27+
type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[0].resource_record_type
28+
zone_id = "${resource.aws_route53_zone.gitpod.zone_id}"
29+
records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[0].resource_record_value]
30+
ttl = 60
31+
}
32+
33+
resource "aws_route53_record" "cert_validation_1" {
34+
allow_overwrite = true
35+
name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[1].resource_record_name
36+
type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[1].resource_record_type
37+
zone_id = "${resource.aws_route53_zone.gitpod.zone_id}"
38+
records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[1].resource_record_value]
39+
ttl = 60
40+
}
41+
42+
resource "aws_route53_record" "cert_validation_2" {
43+
allow_overwrite = true
44+
name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[2].resource_record_name
45+
type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[2].resource_record_type
46+
zone_id = "${resource.aws_route53_zone.gitpod.zone_id}"
47+
records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[2].resource_record_value]
48+
ttl = 60
49+
}
50+
51+
resource "aws_route53_record" "cert_validation_3" {
52+
allow_overwrite = true
53+
name = tolist(aws_acm_certificate.gitpod.domain_validation_options)[3].resource_record_name
54+
type = tolist(aws_acm_certificate.gitpod.domain_validation_options)[3].resource_record_type
55+
zone_id = "${resource.aws_route53_zone.gitpod.zone_id}"
56+
records = [tolist(aws_acm_certificate.gitpod.domain_validation_options)[3].resource_record_value]
57+
ttl = 6
58+
}
59+
60+
resource "aws_iam_policy" "gitpod" {
61+
name = "role-${var.cluster_name}"
62+
63+
# Terraform's "jsonencode" function converts a
64+
# Terraform expression result to valid JSON syntax.
65+
policy = jsonencode({
66+
Version = "2012-10-17",
67+
Statement = [
68+
{
69+
Effect = "Allow",
70+
Action = [
71+
"route53:ChangeResourceRecordSets"
72+
],
73+
Resource = [
74+
"arn:aws:route53:::hostedzone/*"
75+
]
76+
},
77+
{
78+
Effect = "Allow",
79+
Action = [
80+
"route53:GetChange",
81+
],
82+
Resource = [
83+
"arn:aws:route53:::change/*"
84+
]
85+
},
86+
{
87+
Effect = "Allow",
88+
Action = [
89+
"route53:ListHostedZones",
90+
"route53:ListResourceRecordSets"
91+
],
92+
Resource = [ "*" ]
93+
}
94+
],
95+
})
96+
}
97+
98+
resource "aws_iam_user_policy_attachment" "test-attach" {
99+
user = aws_iam_user.edns.name
100+
policy_arn = aws_iam_policy.gitpod.arn
101+
}
102+
103+
resource "aws_iam_user" "edns" {
104+
name = "${var.cluster_name}-external-dns"
105+
force_destroy = true
106+
107+
tags = {
108+
env = "test"
109+
}
110+
}
111+
112+
resource "aws_iam_access_key" "edns" {
113+
user = aws_iam_user.edns.name
114+
}

0 commit comments

Comments
 (0)