Skip to content

Commit f4441de

Browse files
sbernauersbernauer
committed
Add support for Spark 3.4.0 (#243)
# Description For #242
1 parent e23dae7 commit f4441de

40 files changed

+48
-69
lines changed

‎CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
77
### Added
88

99
- Generate OLM bundle for Release 23.4.0 ([#238]).
10+
- Add support for Spark 3.4.0 ([#243]).
1011

1112
### Changed
1213

@@ -15,10 +16,15 @@ All notable changes to this project will be documented in this file.
1516
- Use testing-tools 0.2.0 ([#236])
1617
- Run as root group ([#241]).
1718

19+
### Fixed
20+
21+
- Fix quoting issues when spark config values contain spaces ([#243]).
22+
1823
[#235]: https://github.com/stackabletech/spark-k8s-operator/pull/235
1924
[#236]: https://github.com/stackabletech/spark-k8s-operator/pull/236
2025
[#238]: https://github.com/stackabletech/spark-k8s-operator/pull/238
2126
[#241]: https://github.com/stackabletech/spark-k8s-operator/pull/241
27+
[#243]: https://github.com/stackabletech/spark-k8s-operator/pull/243
2228

2329
## [23.4.0] - 2023-04-17
2430

‎deploy/helm/spark-k8s-operator/templates/roles.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ rules:
2929
verbs:
3030
- create
3131
- delete
32+
- deletecollection
3233
- get
3334
- list
3435
- patch

‎docs/modules/spark-k8s/examples/example-encapsulated.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ metadata:
55
name: spark-pi
66
spec:
77
version: "1.0"
8-
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.3.0 # <1>
8+
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.0.0-dev # <1>
99
mode: cluster
1010
mainClass: org.apache.spark.examples.SparkPi
11-
mainApplicationFile: /stackable/spark/examples/jars/spark-examples_2.12-3.3.0.jar # <2>
11+
mainApplicationFile: /stackable/spark/examples/jars/spark-examples.jar # <2>
1212
executor:
1313
instances: 3

‎docs/modules/spark-k8s/examples/example-history-app.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ metadata:
55
name: spark-pi-s3-1
66
spec:
77
version: "1.0"
8-
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.3.0
8+
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.0.0-dev
99
sparkImagePullPolicy: IfNotPresent
1010
mode: cluster
1111
mainClass: org.apache.spark.examples.SparkPi
12-
mainApplicationFile: s3a://my-bucket/spark-examples_2.12-3.3.0.jar
12+
mainApplicationFile: s3a://my-bucket/spark-examples.jar
1313
s3connection: # <1>
1414
inline:
1515
host: test-minio

‎docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ metadata:
66
namespace: default
77
spec:
88
version: "1.0"
9-
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.3.0
9+
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.0.0-dev
1010
mode: cluster
1111
mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3>
1212
mainClass: tech.stackable.demo.spark.NYTLCReport

‎docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ metadata:
66
namespace: default
77
spec:
88
version: "1.0"
9-
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.3.0
9+
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.0.0-dev
1010
mode: cluster
1111
mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny_tlc_report.py # <1>
1212
args:

‎docs/modules/spark-k8s/examples/example-sparkapp-image.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ metadata:
77
spec:
88
version: "1.0"
99
image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0 # <1>
10-
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.3.0
10+
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.0.0-dev
1111
mode: cluster
1212
mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2>
1313
args:

‎docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ metadata:
66
namespace: default
77
spec:
88
version: "1.0"
9-
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.3.0
9+
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.0.0-dev
1010
mode: cluster
1111
mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.0-SNAPSHOT.jar # <1>
1212
mainClass: org.example.App # <2>

‎docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ metadata:
55
name: example-sparkapp-s3-private
66
spec:
77
version: "1.0"
8-
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.3.0
8+
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.0.0-dev
99
mode: cluster
10-
mainApplicationFile: s3a://my-bucket/spark-examples_2.12-3.3.0.jar # <1>
10+
mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1>
1111
mainClass: org.apache.spark.examples.SparkPi # <2>
1212
s3connection: # <3>
1313
inline:

‎docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ metadata:
66
namespace: default
77
spec:
88
version: "1.0"
9-
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.3.0
9+
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.0.0-dev
1010
mode: cluster
1111
mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py
1212
args:

‎docs/modules/spark-k8s/examples/getting_started/getting_started.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ metadata:
5858
namespace: default
5959
spec:
6060
version: "1.0"
61-
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.3.0
61+
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.0.0-dev
6262
mode: cluster
6363
mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
6464
driver:

‎docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ metadata:
5858
namespace: default
5959
spec:
6060
version: "1.0"
61-
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.3.0
61+
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-0.0.0-dev
6262
mode: cluster
6363
mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
6464
driver:

‎docs/modules/spark-k8s/partials/supported-versions.adoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@
55
- 3.2.1-hadoop3.2
66
- 3.2.1-hadoop3.2-python39
77
- 3.3.0-hadoop3
8+
- 3.4.0-hadoop3

‎examples/ny-tlc-report-external-dependencies.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ metadata:
66
namespace: default
77
spec:
88
version: "1.0"
9-
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.3.0
9+
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.0.0-dev
1010
# Always | IfNotPresent | Never
1111
sparkImagePullPolicy: IfNotPresent
1212
mode: cluster

‎examples/ny-tlc-report-image.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ spec:
88
version: "1.0"
99
# everything under /jobs will be copied to /stackable/spark/jobs
1010
image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0
11-
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.3.0
11+
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.0.0-dev
1212
sparkImagePullPolicy: IfNotPresent
1313
mode: cluster
1414
mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py

‎examples/ny-tlc-report.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ metadata:
1313
name: spark-ny-cm
1414
spec:
1515
version: "1.0"
16-
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.3.0
16+
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.3.0-stackable0.0.0-dev
1717
mode: cluster
1818
mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.3.0.jar
1919
mainClass: tech.stackable.demo.spark.NYTLCReport

‎rust/crd/src/lib.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -618,7 +618,7 @@ impl SparkApplication {
618618
}
619619
// ...before being added to the command collection
620620
for (key, value) in submit_conf {
621-
submit_cmd.push(format!("--conf {key}={value}"));
621+
submit_cmd.push(format!("--conf \"{key}={value}\""));
622622
}
623623

624624
submit_cmd.extend(
@@ -939,7 +939,7 @@ spec:
939939
sparkImage: docker.stackable.tech/stackable/spark-k8s:3.2.1-hadoop3.2-python39-aws1.11.375-stackable0.3.0
940940
mode: cluster
941941
mainClass: org.apache.spark.examples.SparkPi
942-
mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/spark-examples_2.12-3.2.1.jar
942+
mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/spark-examples.jar
943943
sparkConf:
944944
"spark.hadoop.fs.s3a.aws.credentials.provider": "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
945945
driver:
@@ -960,7 +960,7 @@ spec:
960960
spark_application.spec.main_class
961961
);
962962
assert_eq!(
963-
Some("s3a://stackable-spark-k8s-jars/jobs/spark-examples_2.12-3.2.1.jar".to_string()),
963+
Some("s3a://stackable-spark-k8s-jars/jobs/spark-examples.jar".to_string()),
964964
spark_application.spec.main_application_file
965965
);
966966
assert_eq!(
@@ -1113,7 +1113,7 @@ spec:
11131113
- name: myregistrykey
11141114
mode: cluster
11151115
mainClass: org.apache.spark.examples.SparkPi
1116-
mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples_2.12-3.2.1.jar
1116+
mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples.jar
11171117
sparkConf:
11181118
spark.kubernetes.node.selector.node: "2"
11191119
driver:

‎rust/operator-binary/src/history_controller.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -536,9 +536,9 @@ fn command_args(s3logdir: &S3LogDir) -> Vec<String> {
536536

537537
if let Some(secret_dir) = s3logdir.credentials_mount_path() {
538538
command.extend(vec![
539-
format!("export AWS_ACCESS_KEY_ID=$(cat {secret_dir}/{ACCESS_KEY_ID})"),
539+
format!("export AWS_ACCESS_KEY_ID=\"$(cat {secret_dir}/{ACCESS_KEY_ID})\""),
540540
"&&".to_string(),
541-
format!("export AWS_SECRET_ACCESS_KEY=$(cat {secret_dir}/{SECRET_ACCESS_KEY})"),
541+
format!("export AWS_SECRET_ACCESS_KEY=\"$(cat {secret_dir}/{SECRET_ACCESS_KEY})\""),
542542
"&&".to_string(),
543543
]);
544544
}

‎tests/templates/kuttl/logging/05-deploy-automatic-log-config-spark-app.yaml.j2

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,11 @@ metadata:
66
spec:
77
version: "1.0"
88
sparkImage: docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
9-
sparkImagePullPolicy: IfNotPresent
109
image: docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
1110
vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
1211
mode: cluster
1312
mainClass: org.apache.spark.examples.SparkALS
14-
mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar
13+
mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples.jar
1514
job:
1615
logging:
1716
enableVectorAgent: true

‎tests/templates/kuttl/logging/06-deploy-custom-log-config-spark-app.yaml.j2

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,11 @@ metadata:
4040
spec:
4141
version: "1.0"
4242
sparkImage: docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
43-
sparkImagePullPolicy: IfNotPresent
4443
image: docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
4544
vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
4645
mode: cluster
4746
mainClass: org.apache.spark.examples.SparkALS
48-
mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar
47+
mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples.jar
4948
job:
5049
logging:
5150
enableVectorAgent: true

‎tests/templates/kuttl/logging/07-deploy-automatic-log-config-pyspark-app.yaml.j2

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ metadata:
66
spec:
77
version: "1.0"
88
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
9-
sparkImagePullPolicy: IfNotPresent
109
vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
1110
mode: cluster
1211
mainApplicationFile: local:///stackable/spark/examples/src/main/python/als.py

‎tests/templates/kuttl/logging/08-deploy-custom-log-config-pyspark-app.yaml.j2

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ metadata:
4040
spec:
4141
version: "1.0"
4242
sparkImage: docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'].split('-stackable')[0] }}-stackable{{ test_scenario['values']['spark'].split('-stackable')[1] }}
43-
sparkImagePullPolicy: IfNotPresent
4443
vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
4544
mode: cluster
4645
mainApplicationFile: local:///stackable/spark/examples/src/main/python/als.py

‎tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ spec:
1212
sparkImagePullPolicy: IfNotPresent
1313
mode: cluster
1414
mainClass: org.apache.spark.examples.SparkALS
15-
mainApplicationFile: "local:///stackable/spark/examples/jars/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar"
15+
mainApplicationFile: "local:///stackable/spark/examples/jars/spark-examples.jar"
1616
job:
1717
logging:
1818
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}

‎tests/templates/kuttl/spark-history-server/04-prepare-bucket.yaml.j2

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,9 @@ kind: TestStep
44
commands:
55
# give minio enough time to start
66
- command: sleep 10
7-
- command: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar minio-client:/tmp
7+
- command: kubectl cp -n $NAMESPACE spark-examples_{{ test_scenario['values']['spark'].split('-stackable')[0] }}.jar minio-client:/tmp/spark-examples.jar
88
- command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY'
99
- command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket
1010
- command: kubectl exec -n $NAMESPACE eventlog-minio-client -- sh -c 'mc alias set eventlog-minio http://eventlog-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY'
1111
- command: kubectl exec -n $NAMESPACE eventlog-minio-client -- mc mb eventlog-minio/spark-logs/eventlogs
12-
- script: >-
13-
kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar test-minio/my-bucket
12+
- command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples.jar test-minio/my-bucket

‎tests/templates/kuttl/spark-history-server/10-deploy-spark-app.yaml.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ spec:
1212
sparkImagePullPolicy: IfNotPresent
1313
mode: cluster
1414
mainClass: org.apache.spark.examples.SparkPi
15-
mainApplicationFile: "s3a://my-bucket/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar"
15+
mainApplicationFile: "s3a://my-bucket/spark-examples.jar"
1616
s3connection:
1717
reference: spark-data-s3-connection
1818
logFileDirectory:

‎tests/templates/kuttl/spark-history-server/12-deploy-spark-app.yaml.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ spec:
1212
sparkImagePullPolicy: IfNotPresent
1313
mode: cluster
1414
mainClass: org.apache.spark.examples.SparkPi
15-
mainApplicationFile: "s3a://my-bucket/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar"
15+
mainApplicationFile: "s3a://my-bucket/spark-examples.jar"
1616
s3connection:
1717
reference: spark-data-s3-connection
1818
logFileDirectory:

‎tests/templates/kuttl/spark-history-server/spark-examples_2.12-3.3.0.jar renamed to ‎tests/templates/kuttl/spark-history-server/spark-examples_3.3.0.jar

File renamed without changes.

‎tests/templates/kuttl/spark-history-server/spark-examples_3.4.0.jar

1.49 MB
Binary file not shown.

‎tests/templates/kuttl/spark-ny-public-s3/03-prepare-bucket.yaml.j2

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,10 @@ kind: TestStep
44
commands:
55
# give minio enough time to start
66
- command: sleep 10
7-
- command: kubectl cp -n $NAMESPACE ny-tlc-report-1.1.0-{{ test_scenario['values']['examples'] }}.jar minio-client:/tmp
7+
- command: kubectl cp -n $NAMESPACE ny-tlc-report-1.1.0-{{ test_scenario['values']['spark'].split('-stackable')[0] }}.jar minio-client:/tmp/ny-tlc-report.jar
88
- command: kubectl cp -n $NAMESPACE yellow_tripdata_2021-07.csv minio-client:/tmp
99
- command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY'
1010
- command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket
1111
- command: kubectl exec -n $NAMESPACE minio-client -- mc policy set public test-minio/my-bucket
12-
- script: >-
13-
kubectl exec -n $NAMESPACE minio-client --
14-
mc cp /tmp/ny-tlc-report-1.1.0-{{ test_scenario['values']['examples'] }}.jar test-minio/my-bucket
15-
- script: >-
16-
kubectl exec -n $NAMESPACE minio-client --
17-
mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket
12+
- command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/ny-tlc-report.jar test-minio/my-bucket
13+
- command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket

‎tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ spec:
2020
sparkImagePullPolicy: IfNotPresent
2121
mode: cluster
2222
mainClass: tech.stackable.demo.spark.NYTLCReport
23-
mainApplicationFile: "s3a://my-bucket/ny-tlc-report-1.1.0-{{ test_scenario['values']['examples'] }}.jar"
23+
mainApplicationFile: "s3a://my-bucket/ny-tlc-report.jar"
2424
volumes:
2525
- name: cm-job-arguments
2626
configMap:

‎tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.4.0.jar

132 KB
Binary file not shown.

‎tests/templates/kuttl/spark-pi-private-s3/04-prepare-bucket.yaml.j2

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ kind: TestStep
44
commands:
55
# give minio enough time to start
66
- command: sleep 10
7-
- command: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar minio-client:/tmp
7+
- command: kubectl cp -n $NAMESPACE spark-examples_{{ test_scenario['values']['spark'].split('-stackable')[0] }}.jar minio-client:/tmp/spark-examples.jar
88
- command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY'
99
- command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket
10-
- script: >-
11-
kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar test-minio/my-bucket
10+
- command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples.jar test-minio/my-bucket

‎tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ spec:
1212
sparkImagePullPolicy: IfNotPresent
1313
mode: cluster
1414
mainClass: org.apache.spark.examples.SparkPi
15-
mainApplicationFile: s3a://my-bucket/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar
15+
mainApplicationFile: s3a://my-bucket/spark-examples.jar
1616
s3connection:
1717
inline:
1818
host: test-minio

‎tests/templates/kuttl/spark-pi-private-s3/spark-examples_2.12-3.3.0.jar renamed to ‎tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.3.0.jar

File renamed without changes.

‎tests/templates/kuttl/spark-pi-private-s3/spark-examples_3.4.0.jar

1.49 MB
Binary file not shown.

‎tests/templates/kuttl/spark-pi-public-s3/03-prepare-bucket.yaml.j2

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,8 @@ kind: TestStep
44
commands:
55
# give minio enough time to start
66
- command: sleep 10
7-
- command: kubectl cp -n $NAMESPACE spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar minio-client:/tmp
7+
- command: kubectl cp -n $NAMESPACE spark-examples_{{ test_scenario['values']['spark'].split('-stackable')[0] }}.jar minio-client:/tmp/spark-examples.jar
88
- command: kubectl exec -n $NAMESPACE minio-client -- sh -c 'mc alias set test-minio http://test-minio:9000 $$MINIO_SERVER_ACCESS_KEY $$MINIO_SERVER_SECRET_KEY'
99
- command: kubectl exec -n $NAMESPACE minio-client -- mc mb test-minio/my-bucket
1010
- command: kubectl exec -n $NAMESPACE minio-client -- mc policy set public test-minio/my-bucket
11-
- script: >-
12-
kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar test-minio/my-bucket
11+
- command: kubectl exec -n $NAMESPACE minio-client -- mc cp /tmp/spark-examples.jar test-minio/my-bucket

‎tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ spec:
1212
sparkImagePullPolicy: IfNotPresent
1313
mode: cluster
1414
mainClass: org.apache.spark.examples.SparkPi
15-
mainApplicationFile: s3a://my-bucket/spark-examples_2.12-{{ test_scenario['values']['examples'] }}.jar
15+
mainApplicationFile: s3a://my-bucket/spark-examples.jar
1616
s3connection:
1717
inline:
1818
host: test-minio

‎tests/templates/kuttl/spark-pi-public-s3/spark-examples_2.12-3.3.0.jar renamed to ‎tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.3.0.jar

File renamed without changes.

‎tests/templates/kuttl/spark-pi-public-s3/spark-examples_3.4.0.jar

1.49 MB
Binary file not shown.

0 commit comments

Comments
 (0)