Merge pull request #27 from stackabletech/no-image-case

Refactor code and stabilize CRDs

Author: razvan

Cargo.lock

@@ -15,6 +15,8 @@
 from pyspark.sql import SparkSession, DataFrame
 from pyspark.sql.functions import dayofweek
 
+import tabulate
+
 
 def check_args() -> Namespace:
     """Parse the given CLI arguments"""
@@ -54,7 +56,7 @@ def build_report(spark: SparkSession, args: Namespace) -> DataFrame:
 
     try:
         report = build_report(spark, args)
-        report.show()
+        print(tabulate.tabulate(report.collect()))
         if args.output:
             report.coalesce(1).write.mode("overwrite").options(header=True).csv(
                 args.output

apps/ny_tlc_report.py

@@ -44,6 +44,4 @@ Environment variables can be (over)written by adding them to `spark-env.sh` file
 
 === Development
 
-export KUBERNETES_SERVICE_PORT_HTTPS=443
-export KUBERNETES_SERVICE_HOST=10.96.0.1
 kubectl create clusterrolebinding spark-role --clusterrole=edit  --serviceaccount=default:default

docs/modules/ROOT/pages/usage.adoc

@@ -6,7 +6,6 @@ metadata:
   namespace: default
 spec:
   version: "1.0"
-  image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0
   sparkImage: docker.stackable.tech/stackable/spark-k8s:3.2.1-hadoop3.2-python39-aws1.11.375-stackable0.3.0
   mode: cluster
   mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny_tlc_report.py

examples/ny-tlc-report-external-dependencies.yaml

@@ -4,8 +4,6 @@ kind: SparkApplication
 metadata:
   name: spark-examples-s3
 spec:
-  # TODO : remove the image
-  image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0
   version: "1.0"
   sparkImage: docker.stackable.tech/stackable/spark-k8s:3.2.1-hadoop3.2-python39-aws1.11.375-stackable0.3.0
   mode: cluster

examples/spark-examples-s3.yaml

@@ -8,7 +8,7 @@ repository = "https://github.com/stackabletech/spark-k8s-operator"
 version = "0.1.0-nightly"
 
 [dependencies]
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.13.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" }
 
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }

rust/crd/Cargo.toml

@@ -0,0 +1,16 @@
+pub const VOLUME_MOUNT_NAME_POD_TEMPLATES: &str = "pod-template";
+pub const VOLUME_MOUNT_PATH_POD_TEMPLATES: &str = "/stackable/spark/pod-templates";
+
+pub const CONTAINER_NAME_JOB: &str = "job";
+pub const VOLUME_MOUNT_NAME_JOB: &str = "job-files";
+pub const VOLUME_MOUNT_PATH_JOB: &str = "/stackable/spark/jobs";
+
+pub const CONTAINER_NAME_REQ: &str = "requirements";
+pub const VOLUME_MOUNT_NAME_REQ: &str = "req-files";
+pub const VOLUME_MOUNT_PATH_REQ: &str = "/stackable/spark/requirements";
+
+pub const CONTAINER_IMAGE_NAME_DRIVER: &str = "dummy-overwritten-by-command-line";
+pub const CONTAINER_NAME_DRIVER: &str = "spark-driver";
+
+pub const CONTAINER_IMAGE_NAME_EXECUTOR: &str = "dummy-overwritten-by-command-line";
+pub const CONTAINER_NAME_EXECUTOR: &str = "spark-executor";