From d56234340fb2a06396da61197c1543dbc7da67c4 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 30 Sep 2022 11:42:14 +0200 Subject: [PATCH 1/6] replace yq usage with standard tools --- tests/templates/kuttl/resources/13-assert.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/resources/13-assert.yaml b/tests/templates/kuttl/resources/13-assert.yaml index 70e61f8a..f66a7b6e 100644 --- a/tests/templates/kuttl/resources/13-assert.yaml +++ b/tests/templates/kuttl/resources/13-assert.yaml @@ -4,7 +4,6 @@ kind: TestAssert timeout: 240 skipLogOutput: true commands: - - script: kubectl get cm -n $NAMESPACE spark-resources-pod-template -o yaml | yq -r '.data."driver.yml"' | yq -r '.spec.containers[0].resources.limits.cpu' | grep 1500m - - script: kubectl get cm -n $NAMESPACE spark-resources-pod-template -o yaml | yq -r '.data."driver.yml"' | yq -r '.spec.containers[0].resources.requests.cpu' | grep 1 - - script: kubectl get cm -n $NAMESPACE spark-resources-pod-template -o yaml | yq -r '.data."executor.yml"' | yq -r '.spec.containers[0].resources.limits.cpu' | grep 2 - - script: kubectl get cm -n $NAMESPACE spark-resources-pod-template -o yaml | yq -r '.data."executor.yml"' | yq -r '.spec.containers[0].resources.requests.cpu' | grep 1 + # read the config map, stripping indents and new lines and doing some replacement, then grepping for the relevant section + - script: kubectl get cm -n $NAMESPACE spark-resources-pod-template -o yaml | sed -e 's/^\s*//' -e '/^$/d' | tr '\n' ' ' | tr ':' '|' | grep -o 'spark-driver resources| limits| cpu| 1500m memory| 1Gi requests| cpu| "1" memory| 1Gi' + - script: kubectl get cm -n $NAMESPACE spark-resources-pod-template -o yaml | sed -e 's/^\s*//' -e '/^$/d' | tr '\n' ' ' | tr ':' '|' | grep -o 'spark-executor resources| limits| cpu| "2" memory| 1Gi requests| cpu| "1" memory| 1Gi' From 48a09f05d3324d75ab8485f1a3daca4da589bd57 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 30 Sep 2022 12:30:00 +0200 Subject: [PATCH 2/6] updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a009d9e7..5f1ba5bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,11 @@ All notable changes to this project will be documented in this file. - Bumped image to `3.3.0-stackable0.2.0` in tests and docs ([#145]) - BREAKING: use resource limit struct instead of passing spark configuration arguments ([#147]) +- Fixed resources test ([#150]) [#145]: https://github.com/stackabletech/spark-k8s-operator/pull/145 [#147]: https://github.com/stackabletech/spark-k8s-operator/pull/147 +[#150]: https://github.com/stackabletech/spark-k8s-operator/pull/150 ## [0.5.0] - 2022-09-06 From 3616dcf4fbe657c49907dbb9e084f205a06f574b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 30 Sep 2022 15:11:02 +0200 Subject: [PATCH 3/6] merged --- .../templates/kuttl/resources/10-assert.yaml | 74 ++++++++++++++++++- .../templates/kuttl/resources/13-assert.yaml | 9 --- 2 files changed, 73 insertions(+), 10 deletions(-) delete mode 100644 tests/templates/kuttl/resources/13-assert.yaml diff --git a/tests/templates/kuttl/resources/10-assert.yaml b/tests/templates/kuttl/resources/10-assert.yaml index 3af4f245..661aed83 100644 --- a/tests/templates/kuttl/resources/10-assert.yaml +++ b/tests/templates/kuttl/resources/10-assert.yaml @@ -3,10 +3,82 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 900 --- -# The Job starting the whole process apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication metadata: name: spark-resources status: phase: Succeeded +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + spark-role: driver + spark-app-name: spark-resources + app.kubernetes.io/managed-by: spark-k8s-operator +spec: + containers: + - name: spark-driver + resources: + limits: + cpu: 1500m + # N.B. spark adds memoryOverhead of 10% by default + memory: 1408Mi + requests: + cpu: "1" + memory: 1408Mi +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + job-name: spark-resources + app.kubernetes.io/managed-by: spark-k8s-operator +spec: + containers: + - name: spark-submit + resources: + limits: + cpu: 200m + memory: 1Gi + requests: + cpu: 100m + memory: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + spark-app-name: sparkals + spark-exec-id: "1" + spark-role: executor +spec: + containers: + - name: spark-executor + resources: + limits: + cpu: "2" + # N.B. spark adds memoryOverhead of 10% by default + memory: 1408Mi + requests: + cpu: "1" + memory: 1408Mi +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + spark-app-name: sparkals + spark-exec-id: "2" + spark-role: executor +spec: + containers: + - name: spark-executor + resources: + limits: + cpu: "2" + memory: 1408Mi + requests: + cpu: "1" + memory: 1408Mi diff --git a/tests/templates/kuttl/resources/13-assert.yaml b/tests/templates/kuttl/resources/13-assert.yaml deleted file mode 100644 index f66a7b6e..00000000 --- a/tests/templates/kuttl/resources/13-assert.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 240 -skipLogOutput: true -commands: - # read the config map, stripping indents and new lines and doing some replacement, then grepping for the relevant section - - script: kubectl get cm -n $NAMESPACE spark-resources-pod-template -o yaml | sed -e 's/^\s*//' -e '/^$/d' | tr '\n' ' ' | tr ':' '|' | grep -o 'spark-driver resources| limits| cpu| 1500m memory| 1Gi requests| cpu| "1" memory| 1Gi' - - script: kubectl get cm -n $NAMESPACE spark-resources-pod-template -o yaml | sed -e 's/^\s*//' -e '/^$/d' | tr '\n' ' ' | tr ':' '|' | grep -o 'spark-executor resources| limits| cpu| "2" memory| 1Gi requests| cpu| "1" memory| 1Gi' From 0cc027922e4f1dfd02a8cd08b2c1acb90a14fa20 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 30 Sep 2022 15:14:24 +0200 Subject: [PATCH 4/6] changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f1ba5bd..614b3c4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,11 @@ All notable changes to this project will be documented in this file. - Bumped image to `3.3.0-stackable0.2.0` in tests and docs ([#145]) - BREAKING: use resource limit struct instead of passing spark configuration arguments ([#147]) -- Fixed resources test ([#150]) +- Fixed resources test ([#151]) [#145]: https://github.com/stackabletech/spark-k8s-operator/pull/145 [#147]: https://github.com/stackabletech/spark-k8s-operator/pull/147 -[#150]: https://github.com/stackabletech/spark-k8s-operator/pull/150 +[#151]: https://github.com/stackabletech/spark-k8s-operator/pull/151 ## [0.5.0] - 2022-09-06 From 7a8ac32e1a66768db14e6ec4fabd0ce902a7f901 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 30 Sep 2022 19:09:43 +0200 Subject: [PATCH 5/6] removed label --- tests/templates/kuttl/resources/10-assert.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/templates/kuttl/resources/10-assert.yaml b/tests/templates/kuttl/resources/10-assert.yaml index 661aed83..2c24de1a 100644 --- a/tests/templates/kuttl/resources/10-assert.yaml +++ b/tests/templates/kuttl/resources/10-assert.yaml @@ -15,7 +15,6 @@ kind: Pod metadata: labels: spark-role: driver - spark-app-name: spark-resources app.kubernetes.io/managed-by: spark-k8s-operator spec: containers: @@ -50,7 +49,6 @@ apiVersion: v1 kind: Pod metadata: labels: - spark-app-name: sparkals spark-exec-id: "1" spark-role: executor spec: @@ -69,7 +67,6 @@ apiVersion: v1 kind: Pod metadata: labels: - spark-app-name: sparkals spark-exec-id: "2" spark-role: executor spec: From ba24fad0c1641df0dbb19026c8a347bcaa41fdae Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Sat, 1 Oct 2022 11:39:28 +0200 Subject: [PATCH 6/6] removed executor assert --- .../templates/kuttl/resources/10-assert.yaml | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/tests/templates/kuttl/resources/10-assert.yaml b/tests/templates/kuttl/resources/10-assert.yaml index 2c24de1a..e40a762e 100644 --- a/tests/templates/kuttl/resources/10-assert.yaml +++ b/tests/templates/kuttl/resources/10-assert.yaml @@ -44,38 +44,3 @@ spec: requests: cpu: 100m memory: 1Gi ---- -apiVersion: v1 -kind: Pod -metadata: - labels: - spark-exec-id: "1" - spark-role: executor -spec: - containers: - - name: spark-executor - resources: - limits: - cpu: "2" - # N.B. spark adds memoryOverhead of 10% by default - memory: 1408Mi - requests: - cpu: "1" - memory: 1408Mi ---- -apiVersion: v1 -kind: Pod -metadata: - labels: - spark-exec-id: "2" - spark-role: executor -spec: - containers: - - name: spark-executor - resources: - limits: - cpu: "2" - memory: 1408Mi - requests: - cpu: "1" - memory: 1408Mi