Merge pull request #77 from stackabletech/handle-spark-app-termination

Catch spark application failures

Author: razvan

CHANGELOG.md

@@ -8,9 +8,11 @@ All notable changes to this project will be documented in this file.
 
 - Added new fields to govern image pull policy ([#75])
 - New `nodeSelector` fields for both the driver and the excutors ([#76])
+- Mirror driver pod status to the corresponding spark application ([#77])
 
 [#75]: https://github.com/stackabletech/spark-k8s-operator/pull/75
 [#76]: https://github.com/stackabletech/spark-k8s-operator/pull/76
+[#77]: https://github.com/stackabletech/spark-k8s-operator/pull/77
 
 ### Changed
 

Cargo.lock

@@ -1,276 +1,6 @@
+---
+# Not used. Kept for compatibilty with Dockerfile.
 version: 0.1.0
 spec:
-  units:
-    - unit: &unitPassword
-        name: "password"
-        regex: "^[a-zA-Z]\\w{5,20}$"
-    - unit: &unitDirectory
-        name: "directory"
-        regex: "^(.*)/?([^/]+)$"
-        examples:
-          - "/tmp/xyz"
-    - unit: &unitMemory
-        name: "memory"
-        regex: "(^\\p{N}+)(?:\\s*)((?:b|k|m|g|t|p|kb|mb|gb|tb|pb)\\b$)"
-        examples:
-          - "1024b"
-          - "1024kb"
-          - "500m"
-          - "1g"
-
-properties:
-  - property: &sparkWorkerCores
-      propertyNames:
-        - name: "SPARK_WORKER_CORES"
-          kind:
-            type: "file"
-            file: "spark-env.sh"
-      datatype:
-         type: "integer"
-      roles:
-        - name: "slave"
-          required: false
-      asOfVersion: "0.6.2"
-      description: "Total number of cores to allow Spark jobs to use on the machine (default: all available cores)"
-
-  - property: &sparkWorkerMemory
-      propertyNames:
-        - name: "SPARK_WORKER_MEMORY"
-          kind:
-            type: "file"
-            file: "spark-env.sh"
-      datatype:
-        type: "string"
-        unit: *unitMemory
-      roles:
-        - name: "slave"
-          required: false
-      asOfVersion: "0.6.2"
-      description: "Total amount of memory to allow Spark jobs to use on the machine, e.g. 1000M, 2G (default: total memory minus 1 GB); note that each job's individual memory is configured using SPARK_MEM."
-
-  - property: &sparkDaemonMemory
-      propertyNames:
-        - name: "SPARK_DAEMON_MEMORY"
-          kind:
-            type: "file"
-            file: "spark-env.sh"
-      datatype:
-        type: "string"
-        unit: *unitMemory
-      defaultValues:
-        - fromVersion: "0.6.2"
-          to_version: "1.4.1"
-          value: "512m"
-        - fromVersion: "1.5.0"
-          value: "1g"
-      roles:
-        - name: "master"
-          required: false
-        - name: "slave"
-          required: false
-        - name: "history-server"
-          required: false
-      asOfVersion: "0.6.2"
-      description: "Memory to allocate to the Spark master and worker daemons themselves (default: 512m/1g)"
-
-  - property: &sparkEventLogEnabled
-      propertyNames:
-        - name: "spark.eventLog.enabled"
-          kind:
-            type: "file"
-            file: "spark-defaults.conf"
-      datatype:
-        type: "bool"
-      defaultValues:
-        - fromVersion: "1.0.0"
-          value: "false"
-      recommendedValues:
-        - fromVersion: "1.0.0"
-          value: "true"
-      roles:
-        - name: "master"
-          required: false
-        - name: "slave"
-          required: false
-        - name: "history-server"
-          required: false
-      asOfVersion: "1.0.0"
-      description: "Whether to log Spark events, useful for reconstructing the Web UI after the application has finished."
-
-  - property: &sparkEventLogDir
-      propertyNames:
-        - name: "spark.eventLog.dir"
-          kind:
-            type: "file"
-            file: "spark-defaults.conf"
-      datatype:
-        type: "string"
-        unit: *unitDirectory
-      defaultValues:
-        - fromVersion: "1.0.0"
-          value: "/tmp/spark-events"
-      recommendedValues:
-        - fromVersion: "1.0.0"
-          value: "/stackable/log"
-      roles:
-        - name: "master"
-          required: false
-        - name: "slave"
-          required: false
-      asOfVersion: "1.0.0"
-      expandsTo:
-        - property: *sparkEventLogEnabled
-          value: "true"
-      description: "Base directory in which Spark events are logged, if spark.eventLog.enabled is true. Within this base directory, Spark creates a sub-directory for each application, and logs the events specific to the application in this directory. Users may want to set this to a unified location like an HDFS directory so history files can be read by the history server."
-
-  - property: &sparkHistoryLogDirectory
-      propertyNames:
-        - name: "spark.history.fs.logDirectory"
-          kind:
-            type: "file"
-            file: "spark-defaults.conf"
-      datatype:
-        type: "string"
-        unit: *unitDirectory
-      defaultValues:
-        - fromVersion: "1.1.0"
-          value: "/tmp/spark-events"
-      recommendedValues:
-        - fromVersion: "1.0.0"
-          value: "/stackable/log"
-      roles:
-        - name: "history-server"
-          required: true
-      expandsTo:
-        - property: *sparkEventLogEnabled
-          value: "true"
-      asOfVersion: "1.1.0"
-      description: "For the filesystem history provider, the URL to the directory containing application event logs to load. This can be a local file://path, an HDFS path hdfs://namenode/shared/spark-logs or that of an alternative filesystem supported by the Hadoop APIs."
-
-  - property: &sparkHistoryStorePath
-      propertyNames:
-        - name: "spark.history.store.path"
-          kind:
-            type: "file"
-            file: "spark-defaults.conf"
-      datatype:
-        type: "string"
-        unit: *unitDirectory
-      roles:
-        - name: "history-server"
-          required: false
-      asOfVersion: "2.3.0"
-      description: "Local directory where to cache application history data. If set, the history server will store application data on disk instead of keeping it in memory. The data written to disk will be re-used in the event of a history server restart."
-
-  - property: &sparkAuthenticate
-      propertyNames:
-        - name: "spark.authenticate"
-          kind:
-            type: "file"
-            file: "spark-defaults.conf"
-      datatype:
-        type: "bool"
-      defaultValues:
-        - fromVersion: "1.0.0"
-          value: "false"
-      recommendedValues:
-        - fromVersion: "1.0.0"
-          value: "true"
-      roles:
-        - name: "master"
-          required: false
-        - name: "slave"
-          required: false
-        - name: "history-server"
-          required: false
-      asOfVersion: "1.0.0"
-      description: "Whether Spark authenticates its internal connections."
-
-  - property: &sparkAuthenticateSecret
-      propertyNames:
-        - name: "spark.authenticate.secret"
-          kind:
-            type: "file"
-            file: "spark-defaults.conf"
-      datatype:
-        type: "string"
-        unit: *unitPassword
-      recommendedValues:
-        - fromVersion: "1.0.0"
-          value: "secret"
-      roles:
-        - name: "master"
-          required: false
-        - name: "slave"
-          required: false
-        - name: "history-server"
-          required: false
-      asOfVersion: "1.0.0"
-      expandsTo:
-        - property: *sparkAuthenticate
-          value: "true"
-      description: "The secret key used in the authentication. SPARK_AUTHENTICATE must be set to true."
-
-  - property: &sparkPortMaxRetries
-      propertyNames:
-        - name: "spark.port.maxRetries"
-          kind:
-            type: "file"
-            file: "spark-defaults.conf"
-      datatype:
-        type: "integer"
-      defaultValues:
-        - fromVersion: "1.1.1"
-          value: "16"
-      recommendedValues:
-        - fromVersion: "1.1.1"
-          value: "0"
-      roles:
-        - name: "master"
-          required: true
-        - name: "slave"
-          required: true
-        - name: "history-server"
-          required: true
-      asOfVersion: "1.1.1"
-      description: "Maximum number of retries when binding to a port before giving up. When a port is given a specific value (non 0), each subsequent retry will increment the port used in the previous attempt by 1 before retrying. This essentially allows it to try a range of ports from the start port specified to port + maxRetries."
-
-  - property: &sparkNoDaemonize
-      propertyNames:
-        - name: "SPARK_NO_DAEMONIZE"
-          kind:
-            type: "env"
-      datatype:
-        type: "bool"
-      recommendedValues:
-        - fromVersion: "2.0.0"
-          value: "true"
-      roles:
-        - name: "master"
-          required: true
-        - name: "slave"
-          required: true
-        - name: "history-server"
-          required: true
-      asOfVersion: "2.0.0"
-      description: "Run spark processes in foreground if true. Useful for systemd (default: false)"
-
-  - property: &sparkConfDir
-      propertyNames:
-        - name: "SPARK_CONF_DIR"
-          kind:
-            type: "env"
-      datatype:
-        type: "string"
-      recommendedValues:
-        - fromVersion: "1.1.1"
-          value: "/stackable/config"
-      roles:
-        - name: "master"
-          required: true
-        - name: "slave"
-          required: true
-        - name: "history-server"
-          required: true
-      asOfVersion: "1.1.1"
-      description: "To specify a different configuration directory other than the default “SPARK_HOME/conf”, you can set SPARK_CONF_DIR. Spark will use the the configuration files (spark-defaults.conf, spark-env.sh, log4j.properties, etc) from this directory."
+  units: []
+properties: []