Add pull policy (#75)

# Description

Added two optional fields to the custom resource to govern pull policy and secrets to be used when accessing images.
Closes #69.

Author: adwk67

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Added
+
+- Added new fields to govern image pull policy ([#75])
+
+[#75]: https://github.com/stackabletech/spark-k8s-operator/pull/75
+
 ### Changed
 
 - Updated examples ([#71])

Cargo.lock

@@ -356,6 +356,23 @@ spec:
                 sparkImage:
                   nullable: true
                   type: string
+                sparkImagePullPolicy:
+                  enum:
+                    - Always
+                    - IfNotPresent
+                    - Never
+                  nullable: true
+                  type: string
+                sparkImagePullSecrets:
+                  items:
+                    description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.
+                    properties:
+                      name:
+                        description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names"
+                        type: string
+                    type: object
+                  nullable: true
+                  type: array
                 stopped:
                   nullable: true
                   type: boolean

deploy/crd/sparkapplication.crd.yaml

@@ -358,6 +358,23 @@ spec:
                 sparkImage:
                   nullable: true
                   type: string
+                sparkImagePullPolicy:
+                  enum:
+                    - Always
+                    - IfNotPresent
+                    - Never
+                  nullable: true
+                  type: string
+                sparkImagePullSecrets:
+                  items:
+                    description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.
+                    properties:
+                      name:
+                        description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names"
+                        type: string
+                    type: object
+                  nullable: true
+                  type: array
                 stopped:
                   nullable: true
                   type: boolean

deploy/helm/spark-k8s-operator/crds/crds.yaml

@@ -359,6 +359,23 @@ spec:
                 sparkImage:
                   nullable: true
                   type: string
+                sparkImagePullPolicy:
+                  enum:
+                    - Always
+                    - IfNotPresent
+                    - Never
+                  nullable: true
+                  type: string
+                sparkImagePullSecrets:
+                  items:
+                    description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.
+                    properties:
+                      name:
+                        description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names"
+                        type: string
+                    type: object
+                  nullable: true
+                  type: array
                 stopped:
                   nullable: true
                   type: boolean

deploy/manifests/crds.yaml

@@ -199,6 +199,12 @@ Below are listed the CRD fields that can be defined by the user:
 |`spec.sparkImage`
 | Spark image which will be deployed to driver and executor pods, which must contain spark environment needed by the job e.g. `docker.stackable.tech/stackable/spark-k8s:3.2.1-hadoop3.2-stackable0.4.0`
 
+|`spec.sparkImagePullPolicy`
+| Optional Enum (one of `Always`, `IfNotPresent` or `Never`) that determines the pull policy of the spark job image
+
+|`spec.sparkImagePullSecrets`
+| An optional list of references to secrets in the same namespace to use for pulling any of the images used by a `SparkApplication` resource. Each reference has a single property (`name`) that must contain a reference to a valid secret
+
 |`spec.mainApplicationFile`
 |The actual application file that will be called by `spark-submit`
 

docs/modules/ROOT/pages/usage.adoc

@@ -7,6 +7,8 @@ metadata:
 spec:
   version: "1.0"
   sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.2.1-hadoop3.2-python39-stackable0.1.0
+  # Always | IfNotPresent | Never
+  sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainApplicationFile: s3a://my-bucket/ny-tlc-report.py
   args:

examples/ny-tlc-report-external-dependencies.yaml

@@ -9,6 +9,7 @@ spec:
   # everything under /jobs will be copied to /stackable/spark/jobs
   image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0
   sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.2.1-hadoop3.2-python39-stackable0.1.0
+  sparkImagePullPolicy: Always
   mode: cluster
   mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py
   args:

examples/ny-tlc-report-image.yaml

@@ -15,3 +15,4 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.8"
 snafu = "0.7"
+strum = { version = "0.24", features = ["derive"] }

rust/crd/Cargo.toml

@@ -5,7 +5,7 @@ pub mod constants;
 use constants::*;
 use stackable_operator::commons::s3::{InlinedS3BucketSpec, S3BucketDef};
 use stackable_operator::k8s_openapi::api::core::v1::{
-    EnvVar, EnvVarSource, SecretKeySelector, Volume, VolumeMount,
+    EnvVar, EnvVarSource, LocalObjectReference, SecretKeySelector, Volume, VolumeMount,
 };
 
 use std::collections::{BTreeMap, HashMap};
@@ -20,6 +20,7 @@ use stackable_operator::{
     role_utils::CommonConfiguration,
     schemars::{self, JsonSchema},
 };
+use strum::{Display, EnumString};
 
 #[derive(Snafu, Debug)]
 pub enum Error {
@@ -68,6 +69,10 @@ pub struct SparkApplicationSpec {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub spark_image: Option<String>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub spark_image_pull_policy: Option<ImagePullPolicy>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub spark_image_pull_secrets: Option<Vec<LocalObjectReference>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub driver: Option<DriverConfig>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub executor: Option<ExecutorConfig>,
@@ -89,6 +94,13 @@ pub struct SparkApplicationSpec {
     pub env: Option<Vec<EnvVar>>,
 }
 
+#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize, Display, EnumString)]
+pub enum ImagePullPolicy {
+    Always,
+    IfNotPresent,
+    Never,
+}
+
 #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct JobDependencies {
@@ -123,6 +135,14 @@ impl SparkApplication {
         self.spec.image.as_deref()
     }
 
+    pub fn spark_image_pull_policy(&self) -> Option<ImagePullPolicy> {
+        self.spec.spark_image_pull_policy.clone()
+    }
+
+    pub fn spark_image_pull_secrets(&self) -> Option<Vec<LocalObjectReference>> {
+        self.spec.spark_image_pull_secrets.clone()
+    }
+
     pub fn version(&self) -> Option<&str> {
         self.spec.version.as_deref()
     }
@@ -380,7 +400,10 @@ pub struct CommandStatus {
 
 #[cfg(test)]
 mod tests {
+    use crate::ImagePullPolicy;
+    use crate::LocalObjectReference;
     use crate::SparkApplication;
+    use std::str::FromStr;
 
     #[test]
     fn test_spark_examples_s3() {
@@ -542,4 +565,72 @@ spec:
         assert!(spark_application.spec.main_class.is_none());
         assert!(spark_application.spec.image.is_none());
     }
+
+    #[test]
+    fn test_image_actions() {
+        let spark_application = serde_yaml::from_str::<SparkApplication>(
+            r#"
+---
+apiVersion: spark.stackable.tech/v1alpha1
+kind: SparkApplication
+metadata:
+  name: spark-pi-local
+  namespace: default
+spec:
+  version: "1.0"
+  sparkImage: docker.stackable.tech/stackable/spark-k8s:3.2.1-hadoop3.2-stackable0.4.0
+  sparkImagePullPolicy: Always
+  sparkImagePullSecrets:
+    - name: myregistrykey
+  mode: cluster
+  mainClass: org.apache.spark.examples.SparkPi
+  mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples_2.12-3.2.1.jar
+  sparkConf:
+    spark.kubernetes.node.selector.node: "2"
+  driver:
+    cores: 1
+    coreLimit: "1200m"
+    memory: "512m"
+  executor:
+    cores: 1
+    instances: 1
+    memory: "512m"
+        "#,
+        )
+        .unwrap();
+
+        assert_eq!(
+            Some(vec![LocalObjectReference {
+                name: Some("myregistrykey".to_string())
+            }]),
+            spark_application.spark_image_pull_secrets()
+        );
+        assert_eq!(
+            Some(ImagePullPolicy::Always),
+            spark_application.spark_image_pull_policy()
+        );
+    }
+
+    #[test]
+    fn test_image_pull_policy_ser() {
+        assert_eq!("Never", ImagePullPolicy::Never.to_string());
+        assert_eq!("Always", ImagePullPolicy::Always.to_string());
+        assert_eq!("IfNotPresent", ImagePullPolicy::IfNotPresent.to_string());
+    }
+
+    #[test]
+    fn test_image_pull_policy_de() {
+        assert_eq!(
+            ImagePullPolicy::Always,
+            ImagePullPolicy::from_str("Always").unwrap()
+        );
+        assert_eq!(
+            ImagePullPolicy::Never,
+            ImagePullPolicy::from_str("Never").unwrap()
+        );
+        assert_eq!(
+            ImagePullPolicy::IfNotPresent,
+            ImagePullPolicy::from_str("IfNotPresent").unwrap()
+        );
+    }
 }