integration tests in Jenkins CI (#54)

## Description

created integration test files for Jenkins CI



Co-authored-by: Razvan-Daniel Mihai <[email protected]>

Author: backstreetkiwi

.ci/integration-tests/hcloud-centos-8/cluster.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: t2.stackable.tech/v1
+kind: Infra
+template: hcloud-centos-8
+metadata:
+  name: spark-operator-integration-tests
+  description: "Cluster for Spark K8s Operator Integration Tests (Hetzner Cloud / CentOS 8)"
+domain: stackable.test
+publicKeys: []
+spec:
+  location: "hel1"
+  k8sVersion: "$K8S_VERSION"
+  wireguard: false
+  versions:
+    _-operator: NIGHTLY
+    spark-k8s-operator: "$SPARK_K8S_OPERATOR_VERSION"
+  nodes:
+    main:
+      numberOfNodes: 3

.ci/integration-tests/hcloud-centos-8/test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+git clone -b "$GIT_BRANCH" https://github.com/stackabletech/spark-k8s-operator.git
+(cd spark-k8s-operator/ && ./scripts/run_tests.sh)
+exit_code=$?
+./operator-logs.sh spark-k8s > /target/spark-k8s-operator.log
+./operator-logs.sh spark > /target/spark-operator.log
+exit $exit_code

tests/templates/kuttl/spark-ny-public-s3/00-assert.yaml

@@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
   name: minio
-timeout: 300
+timeout: 900
 ---
 apiVersion: v1
 kind: Service

tests/templates/kuttl/spark-ny-public-s3/00-s3-upload-container.yaml

@@ -1,4 +1,15 @@
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-mc
+  labels:
+    app: minio-mc
+spec:
+  clusterIP: None
+  selector:
+    app: minio-mc
+---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -7,6 +18,7 @@ metadata:
     app: minio-mc
 spec:
   replicas: 1
+  serviceName: "minio-mc"
   selector:
     matchLabels:
       app: minio-mc
@@ -18,6 +30,5 @@ spec:
       containers:
         - name: minio-mc
           image: bitnami/minio:2022-debian-10
-          command: ["/bin/sh"]
           stdin: true
           tty: true

tests/templates/kuttl/spark-ny-public-s3/00-setup-minio.yaml

@@ -9,4 +9,5 @@ commands:
       --set replicas=1
       --set persistence.enabled=false
       --set buckets[0].name=my-bucket,buckets[0].policy=public
+      --set resources.requests.memory=1Gi
       --repo https://charts.min.io/ minio

tests/templates/kuttl/spark-ny-public-s3/02-assert.yaml

@@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
   name: spark-ny-deps-job
-timeout: 300
+timeout: 900
 ---
 apiVersion: batch/v1
 kind: Job

tests/templates/kuttl/spark-ny-public-s3/10-assert.yaml

@@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
   name: spark-ny-cm
-timeout: 300
+timeout: 900
 ---
 # The Job starting the whole process
 apiVersion: batch/v1

tests/templates/kuttl/spark-pi-private-s3/00-assert.yaml

@@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
   name: minio
-timeout: 300
+timeout: 900
 ---
 apiVersion: v1
 kind: Service

tests/templates/kuttl/spark-pi-private-s3/00-s3-upload-container.yaml

@@ -1,4 +1,15 @@
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-mc
+  labels:
+    app: minio-mc
+spec:
+  clusterIP: None
+  selector:
+    app: minio-mc
+---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -7,6 +18,7 @@ metadata:
     app: minio-mc
 spec:
   replicas: 1
+  serviceName: "minio-mc"
   selector:
     matchLabels:
       app: minio-mc
@@ -18,6 +30,5 @@ spec:
       containers:
         - name: minio-mc
           image: bitnami/minio:2022-debian-10
-          command: ["/bin/sh"]
           stdin: true
           tty: true

tests/templates/kuttl/spark-pi-private-s3/00-setup-minio.yaml

@@ -10,4 +10,5 @@ commands:
       --set persistence.enabled=false
       --set buckets[0].name=my-bucket,buckets[0].policy=none
       --set users[0].accessKey=minioAccessKey,users[0].secretKey=minioSecretKey,users[0].policy=readwrite
+      --set resources.requests.memory=1Gi
       --repo https://charts.min.io/ minio

tests/templates/kuttl/spark-pi-private-s3/02-assert.yaml

@@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
   name: spark-pi-deps-job
-timeout: 300
+timeout: 900
 ---
 apiVersion: batch/v1
 kind: Job

tests/templates/kuttl/spark-pi-private-s3/10-assert.yaml

@@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
   name: spark-pi
-timeout: 300
+timeout: 900
 ---
 # The Job starting the whole process
 apiVersion: batch/v1

tests/templates/kuttl/spark-pi-public-s3/00-assert.yaml

@@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
   name: minio
-timeout: 300
+timeout: 900
 ---
 apiVersion: v1
 kind: Service

tests/templates/kuttl/spark-pi-public-s3/00-s3-upload-container.yaml

@@ -1,4 +1,15 @@
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-mc
+  labels:
+    app: minio-mc
+spec:
+  clusterIP: None
+  selector:
+    app: minio-mc
+---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -7,6 +18,7 @@ metadata:
     app: minio-mc
 spec:
   replicas: 1
+  serviceName: "minio-mc"
   selector:
     matchLabels:
       app: minio-mc
@@ -18,6 +30,5 @@ spec:
       containers:
         - name: minio-mc
           image: bitnami/minio:2022-debian-10
-          command: ["/bin/sh"]
           stdin: true
           tty: true

tests/templates/kuttl/spark-pi-public-s3/00-setup-minio.yaml

@@ -9,4 +9,5 @@ commands:
       --set replicas=1
       --set persistence.enabled=false
       --set buckets[0].name=my-bucket,buckets[0].policy=public
+      --set resources.requests.memory=1Gi
       --repo https://charts.min.io/ minio

tests/templates/kuttl/spark-pi-public-s3/02-assert.yaml

@@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
   name: spark-pi-deps-job
-timeout: 300
+timeout: 900
 ---
 apiVersion: batch/v1
 kind: Job

tests/templates/kuttl/spark-pi-public-s3/10-assert.yaml

@@ -3,7 +3,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
   name: spark-pi
-timeout: 300
+timeout: 900
 ---
 # The Job starting the whole process
 apiVersion: batch/v1