From f4cf2c841ebae8422b1e685f75d260f5ec6da824 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 11 Dec 2023 14:05:25 +0100 Subject: [PATCH 01/15] updated operator rs --- Cargo.lock | 9 +- Cargo.toml | 2 +- deploy/helm/spark-k8s-operator/crds/crds.yaml | 160 +++++++++++------- rust/operator-binary/src/main.rs | 4 +- 4 files changed, 110 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d836e415..388ebf86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1945,8 +1945,8 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "stackable-operator" -version = "0.56.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.56.1#beeb39436024fa5f61d840402c26ee56fc5fbd29" +version = "0.57.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.57.0#ab5c5c3f220ae9449e82f6861f44a4a9a6fb7b6b" dependencies = [ "chrono", "clap", @@ -1965,6 +1965,7 @@ dependencies = [ "rand", "regex", "schemars", + "semver", "serde", "serde_json", "serde_yaml", @@ -1980,8 +1981,8 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.56.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.56.1#beeb39436024fa5f61d840402c26ee56fc5fbd29" +version = "0.57.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.57.0#ab5c5c3f220ae9449e82f6861f44a4a9a6fb7b6b" dependencies = [ "darling", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index dc5d6550..032627b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.7" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.56.1" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.57.0" } strum = { version = "0.25", features = ["derive"] } tokio = { version = "1.29", features = ["full"] } tracing = "0.1" diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml index 581ea19e..4f01775e 100644 --- a/deploy/helm/spark-k8s-operator/crds/crds.yaml +++ b/deploy/helm/spark-k8s-operator/crds/crds.yaml @@ -70,6 +70,7 @@ spec: podAntiAffinity: null nodeAffinity: null nodeSelector: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). properties: nodeAffinity: description: Node affinity is a group of node affinity scheduling rules. @@ -539,7 +540,7 @@ spec: default: enableVectorAgent: null containers: {} - description: Logging configuration + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). properties: containers: additionalProperties: @@ -612,10 +613,10 @@ spec: description: Configuration per logger type: object type: object - description: Log configuration per container + description: Log configuration per container. type: object enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent + description: Wether or not to deploy a container with the Vector log agent. nullable: true type: boolean type: object @@ -628,6 +629,7 @@ spec: min: null max: null storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. properties: cpu: default: @@ -635,21 +637,22 @@ spec: max: null properties: max: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. nullable: true type: string min: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. nullable: true type: string type: object memory: properties: limit: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' nullable: true type: string runtimeLimits: + description: Additional options that can be specified. type: object type: object storage: @@ -690,15 +693,17 @@ spec: type: string type: object default: {} + description: The `configOverrides` can be used to configure properties in product config files that are not exposed in the CRD. Read the [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. type: object envOverrides: additionalProperties: type: string default: {} + description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object podOverrides: default: {} - description: See PodTemplateSpec (https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) for more details + description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. properties: metadata: properties: @@ -3588,6 +3593,7 @@ spec: podAntiAffinity: null nodeAffinity: null nodeSelector: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). properties: nodeAffinity: description: Node affinity is a group of node affinity scheduling rules. @@ -4057,7 +4063,7 @@ spec: default: enableVectorAgent: null containers: {} - description: Logging configuration + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). properties: containers: additionalProperties: @@ -4130,10 +4136,10 @@ spec: description: Configuration per logger type: object type: object - description: Log configuration per container + description: Log configuration per container. type: object enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent + description: Wether or not to deploy a container with the Vector log agent. nullable: true type: boolean type: object @@ -4146,6 +4152,7 @@ spec: min: null max: null storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. properties: cpu: default: @@ -4153,21 +4160,22 @@ spec: max: null properties: max: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. nullable: true type: string min: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. nullable: true type: string type: object memory: properties: limit: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' nullable: true type: string runtimeLimits: + description: Additional options that can be specified. type: object type: object storage: @@ -4208,15 +4216,17 @@ spec: type: string type: object default: {} + description: The `configOverrides` can be used to configure properties in product config files that are not exposed in the CRD. Read the [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. type: object envOverrides: additionalProperties: type: string default: {} + description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object podOverrides: default: {} - description: See PodTemplateSpec (https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) for more details + description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. properties: metadata: properties: @@ -7067,7 +7077,7 @@ spec: default: enableVectorAgent: null containers: {} - description: Logging configuration + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). properties: containers: additionalProperties: @@ -7140,10 +7150,10 @@ spec: description: Configuration per logger type: object type: object - description: Log configuration per container + description: Log configuration per container. type: object enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent + description: Wether or not to deploy a container with the Vector log agent. nullable: true type: boolean type: object @@ -7156,6 +7166,7 @@ spec: min: null max: null storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. properties: cpu: default: @@ -7163,21 +7174,22 @@ spec: max: null properties: max: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. nullable: true type: string min: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. nullable: true type: string type: object memory: properties: limit: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' nullable: true type: string runtimeLimits: + description: Additional options that can be specified. type: object type: object storage: @@ -7190,15 +7202,17 @@ spec: type: string type: object default: {} + description: The `configOverrides` can be used to configure properties in product config files that are not exposed in the CRD. Read the [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. type: object envOverrides: additionalProperties: type: string default: {} + description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object podOverrides: default: {} - description: See PodTemplateSpec (https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) for more details + description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. properties: metadata: properties: @@ -10004,7 +10018,7 @@ spec: s3: properties: bucket: - description: Operators are expected to define fields for this type in order to work with S3 buckets. + description: An S3 bucket definition, it can either be a reference to an explicit S3Bucket object, or it can be an inline defintion of a bucket. Read the [S3 resources concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3) to learn more. oneOf: - required: - inline @@ -10012,13 +10026,14 @@ spec: - reference properties: inline: - description: S3 bucket specification containing only the bucket name and an inlined or referenced connection specification. + description: An inline definition, containing the S3 bucket properties. properties: bucketName: + description: The name of the S3 bucket. nullable: true type: string connection: - description: Operators are expected to define fields for this type in order to work with S3 connections. + description: The definition of an S3 connection, either inline or as a reference. nullable: true oneOf: - required: @@ -10027,10 +10042,10 @@ spec: - reference properties: inline: - description: S3 connection definition as CRD. + description: Inline definition of an S3 connection. properties: accessStyle: - description: Which access style to use. Defaults to virtual hosted-style as most of the data products out there. Have a look at the official documentation on + description: Which access style to use. Defaults to virtual hosted-style as most of the data products out there. Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). enum: - Path - VirtualHosted @@ -10063,11 +10078,11 @@ spec: - secretClass type: object host: - description: Hostname of the S3 server without any protocol or port + description: 'Hostname of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' nullable: true type: string port: - description: Port the S3 server listens on. If not specified the products will determine the port to use. + description: Port the S3 server listens on. If not specified the product will determine the port to use. format: uint16 minimum: 0.0 nullable: true @@ -10114,10 +10129,12 @@ spec: type: object type: object reference: + description: A reference to an S3Connection resource. type: string type: object type: object reference: + description: A reference to an S3 bucket object. This is simply the name of the `S3Bucket` resource. type: string type: object prefix: @@ -10146,10 +10163,10 @@ spec: - reference properties: inline: - description: S3 connection definition as CRD. + description: Inline definition of an S3 connection. properties: accessStyle: - description: Which access style to use. Defaults to virtual hosted-style as most of the data products out there. Have a look at the official documentation on + description: Which access style to use. Defaults to virtual hosted-style as most of the data products out there. Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). enum: - Path - VirtualHosted @@ -10182,11 +10199,11 @@ spec: - secretClass type: object host: - description: Hostname of the S3 server without any protocol or port + description: 'Hostname of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' nullable: true type: string port: - description: Port the S3 server listens on. If not specified the products will determine the port to use. + description: Port the S3 server listens on. If not specified the product will determine the port to use. format: uint16 minimum: 0.0 nullable: true @@ -10233,6 +10250,7 @@ spec: type: object type: object reference: + description: A reference to an S3Connection resource. type: string type: object sparkConf: @@ -10247,6 +10265,10 @@ spec: - productVersion - required: - productVersion + description: |- + Specify which image to use, the easiest way is to only configure the `productVersion`. You can also configure a custom image registry to pull from, as well as completely custom images. + + Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) for details. properties: custom: description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0` @@ -10256,14 +10278,14 @@ spec: type: string pullPolicy: default: Always - description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the Images' + description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' enum: - IfNotPresent - Always - Never type: string pullSecrets: - description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry' + description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' items: description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. properties: @@ -11443,6 +11465,10 @@ spec: - productVersion - required: - productVersion + description: |- + Specify which image to use, the easiest way is to only configure the `productVersion`. You can also configure a custom image registry to pull from, as well as completely custom images. + + Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) for details. properties: custom: description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0` @@ -11452,14 +11478,14 @@ spec: type: string pullPolicy: default: Always - description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the Images' + description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' enum: - IfNotPresent - Always - Never type: string pullSecrets: - description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry' + description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' items: description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. properties: @@ -11486,7 +11512,7 @@ spec: s3: properties: bucket: - description: Operators are expected to define fields for this type in order to work with S3 buckets. + description: An S3 bucket definition, it can either be a reference to an explicit S3Bucket object, or it can be an inline defintion of a bucket. Read the [S3 resources concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3) to learn more. oneOf: - required: - inline @@ -11494,13 +11520,14 @@ spec: - reference properties: inline: - description: S3 bucket specification containing only the bucket name and an inlined or referenced connection specification. + description: An inline definition, containing the S3 bucket properties. properties: bucketName: + description: The name of the S3 bucket. nullable: true type: string connection: - description: Operators are expected to define fields for this type in order to work with S3 connections. + description: The definition of an S3 connection, either inline or as a reference. nullable: true oneOf: - required: @@ -11509,10 +11536,10 @@ spec: - reference properties: inline: - description: S3 connection definition as CRD. + description: Inline definition of an S3 connection. properties: accessStyle: - description: Which access style to use. Defaults to virtual hosted-style as most of the data products out there. Have a look at the official documentation on + description: Which access style to use. Defaults to virtual hosted-style as most of the data products out there. Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). enum: - Path - VirtualHosted @@ -11545,11 +11572,11 @@ spec: - secretClass type: object host: - description: Hostname of the S3 server without any protocol or port + description: 'Hostname of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' nullable: true type: string port: - description: Port the S3 server listens on. If not specified the products will determine the port to use. + description: Port the S3 server listens on. If not specified the product will determine the port to use. format: uint16 minimum: 0.0 nullable: true @@ -11596,10 +11623,12 @@ spec: type: object type: object reference: + description: A reference to an S3Connection resource. type: string type: object type: object reference: + description: A reference to an S3 bucket object. This is simply the name of the `S3Bucket` resource. type: string type: object prefix: @@ -11626,6 +11655,7 @@ spec: podAntiAffinity: null nodeAffinity: null nodeSelector: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). properties: nodeAffinity: description: Node affinity is a group of node affinity scheduling rules. @@ -12098,7 +12128,7 @@ spec: default: enableVectorAgent: null containers: {} - description: Logging configuration + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). properties: containers: additionalProperties: @@ -12171,10 +12201,10 @@ spec: description: Configuration per logger type: object type: object - description: Log configuration per container + description: Log configuration per container. type: object enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent + description: Wether or not to deploy a container with the Vector log agent. nullable: true type: boolean type: object @@ -12187,6 +12217,7 @@ spec: min: null max: null storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. properties: cpu: default: @@ -12194,21 +12225,22 @@ spec: max: null properties: max: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. nullable: true type: string min: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. nullable: true type: string type: object memory: properties: limit: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' nullable: true type: string runtimeLimits: + description: Additional options that can be specified. type: object type: object storage: @@ -12221,15 +12253,17 @@ spec: type: string type: object default: {} + description: The `configOverrides` can be used to configure properties in product config files that are not exposed in the CRD. Read the [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. type: object envOverrides: additionalProperties: type: string default: {} + description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object podOverrides: default: {} - description: See PodTemplateSpec (https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) for more details + description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. properties: metadata: properties: @@ -15036,7 +15070,12 @@ spec: default: enabled: true maxUnavailable: null - description: 'This struct is used to configure: 1.) If PodDisruptionBudgets are created by the operator 2.) The allowed number of Pods to be unavailable (`maxUnavailable`)' + description: |- + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). properties: enabled: default: true @@ -15067,6 +15106,7 @@ spec: podAntiAffinity: null nodeAffinity: null nodeSelector: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). properties: nodeAffinity: description: Node affinity is a group of node affinity scheduling rules. @@ -15539,7 +15579,7 @@ spec: default: enableVectorAgent: null containers: {} - description: Logging configuration + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). properties: containers: additionalProperties: @@ -15612,10 +15652,10 @@ spec: description: Configuration per logger type: object type: object - description: Log configuration per container + description: Log configuration per container. type: object enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent + description: Wether or not to deploy a container with the Vector log agent. nullable: true type: boolean type: object @@ -15628,6 +15668,7 @@ spec: min: null max: null storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. properties: cpu: default: @@ -15635,21 +15676,22 @@ spec: max: null properties: max: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. nullable: true type: string min: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. nullable: true type: string type: object memory: properties: limit: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' nullable: true type: string runtimeLimits: + description: Additional options that can be specified. type: object type: object storage: @@ -15662,15 +15704,17 @@ spec: type: string type: object default: {} + description: The `configOverrides` can be used to configure properties in product config files that are not exposed in the CRD. Read the [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. type: object envOverrides: additionalProperties: type: string default: {} + description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object podOverrides: default: {} - description: See PodTemplateSpec (https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) for more details + description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. properties: metadata: properties: diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 65045550..96f4d248 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -51,8 +51,8 @@ async fn main() -> anyhow::Result<()> { let opts = Opts::parse(); match opts.cmd { Command::Crd => { - SparkApplication::print_yaml_schema()?; - SparkHistoryServer::print_yaml_schema()?; + SparkApplication::print_yaml_schema(built_info::CARGO_PKG_VERSION)?; + SparkHistoryServer::print_yaml_schema(built_info::CARGO_PKG_VERSION)?; } Command::Run(ProductOperatorRun { product_config, From b827a48678968bc4d7c35006f4864dc226398ee6 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 11 Dec 2023 14:06:33 +0100 Subject: [PATCH 02/15] update Changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1aad75f..0dec5966 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- Various documentation of the CRD ([#319]). + +[#319]: https://github.com/stackabletech/spark-k8s-operator/pull/319 + ### Fixed - Add missing `deletecollection` RBAC permission for Spark drivers. Previously this caused confusing error From 7fb84a4f32cf0148f75a6f8083f47583f1d40757 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 11 Dec 2023 14:38:24 +0100 Subject: [PATCH 03/15] docs --- deploy/helm/spark-k8s-operator/crds/crds.yaml | 16 ++++++- rust/crd/src/history.rs | 1 + rust/crd/src/lib.rs | 42 ++++++++++++++++++- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml index 4f01775e..0745e3bc 100644 --- a/deploy/helm/spark-k8s-operator/crds/crds.yaml +++ b/deploy/helm/spark-k8s-operator/crds/crds.yaml @@ -23,6 +23,10 @@ spec: description: Auto-generated derived type for SparkApplicationSpec via `CustomResource` properties: spec: + description: |- + A Spark cluster stacklet. This resource is managed by the Stackable operator for Apache Spark. Find more information on how to use it and the resources that the operator generates in the [operator documentation](https://docs.stackable.tech/home/nightly/spark-k8s/). + + The SparkApplication CRD looks a little different than the CRDs of the other products on the Stackable Data Platform. properties: args: items: @@ -33,21 +37,25 @@ spec: nullable: true properties: excludePackages: + description: A list of excluded packages that is passed directly to `spark-submit`. items: type: string nullable: true type: array packages: + description: A list of packages that is passed directly to `spark-submit`. items: type: string nullable: true type: array repositories: + description: A list of repositories that is passed directly to `spark-submit`. items: type: string nullable: true type: array requirements: + description: 'Under the `requirements` you can specify Python dependencies that will be installed with `pip`. Example: `tabulate==0.8.9`' items: type: string nullable: true @@ -7060,6 +7068,7 @@ spec: type: object type: object image: + description: 'User-supplied image containing spark-job dependencies that will be copied to the specified volume mount. TODO: Where will they be copied from? Where to? docs link?' nullable: true type: string job: @@ -10145,6 +10154,7 @@ spec: type: object type: object mainApplicationFile: + description: 'The actual application file that will be called by `spark-submit`. TODO: What is the default?' nullable: true type: string mainClass: @@ -10154,7 +10164,7 @@ spec: nullable: true type: string s3connection: - description: Operators are expected to define fields for this type in order to work with S3 connections. + description: Configure an S3 connection that the SparkApplication has access to. Read more in the [Spark S3 usage guide](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/s3). nullable: true oneOf: - required: @@ -10308,10 +10318,11 @@ spec: nullable: true type: boolean vectorAggregatorConfigMapName: - description: Name of the Vector aggregator discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. + description: Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). It must contain the key `ADDRESS` with the address of the Vector aggregator. Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) to learn how to configure log aggregation with Vector. nullable: true type: string version: + description: Application version. TODO what is the default? nullable: true type: string volumes: @@ -11434,6 +11445,7 @@ spec: description: Auto-generated derived type for SparkHistoryServerSpec via `CustomResource` properties: spec: + description: TODO properties: clusterConfig: default: diff --git a/rust/crd/src/history.rs b/rust/crd/src/history.rs index 026b7cef..0d0d5a28 100644 --- a/rust/crd/src/history.rs +++ b/rust/crd/src/history.rs @@ -49,6 +49,7 @@ pub enum Error { CannotRetrieveRoleGroup { role_group: String }, } +/// TODO #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, Serialize)] #[kube( group = "spark.stackable.tech", diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index ca771d0c..1b507ecb 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -82,6 +82,12 @@ pub struct SparkApplicationStatus { pub phase: String, } +/// A Spark cluster stacklet. This resource is managed by the Stackable operator for Apache Spark. +/// Find more information on how to use it and the resources that the operator generates in the +/// [operator documentation](DOCS_BASE_URL_PLACEHOLDER/spark-k8s/). +/// +/// The SparkApplication CRD looks a little different than the CRDs of the other products on the +/// Stackable Data Platform. #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, Serialize)] #[kube( group = "spark.stackable.tech", @@ -98,41 +104,67 @@ pub struct SparkApplicationStatus { )] #[serde(rename_all = "camelCase")] pub struct SparkApplicationSpec { + /// Application version. TODO what is the default? #[serde(default, skip_serializing_if = "Option::is_none")] pub version: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub mode: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub main_class: Option, + + /// The actual application file that will be called by `spark-submit`. + /// TODO: What is the default? #[serde(default, skip_serializing_if = "Option::is_none")] pub main_application_file: Option, + + /// User-supplied image containing spark-job dependencies that will be copied to the specified volume mount. + /// TODO: Where will they be copied from? Where to? docs link? #[serde(default, skip_serializing_if = "Option::is_none")] pub image: Option, + + // no doc - docs in ProductImage struct. pub spark_image: ProductImage, - /// Name of the Vector aggregator discovery ConfigMap. + + /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). /// It must contain the key `ADDRESS` with the address of the Vector aggregator. + /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) + /// to learn how to configure log aggregation with Vector. #[serde(skip_serializing_if = "Option::is_none")] pub vector_aggregator_config_map_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub job: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub driver: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub executor: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub stopped: Option, + + /// #[serde(default, skip_serializing_if = "Option::is_none")] pub spark_conf: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub deps: Option, + + /// Configure an S3 connection that the SparkApplication has access to. + /// Read more in the [Spark S3 usage guide](DOCS_BASE_URL_PLACEHOLDER/spark-k8s/usage-guide/s3). #[serde(default, skip_serializing_if = "Option::is_none")] pub s3connection: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub args: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub volumes: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub env: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub log_file_directory: Option, } @@ -140,12 +172,20 @@ pub struct SparkApplicationSpec { #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub struct JobDependencies { + /// Under the `requirements` you can specify Python dependencies that will be installed with `pip`. + /// Example: `tabulate==0.8.9` #[serde(default, skip_serializing_if = "Option::is_none")] pub requirements: Option>, + + /// A list of packages that is passed directly to `spark-submit`. #[serde(default, skip_serializing_if = "Option::is_none")] pub packages: Option>, + + /// A list of repositories that is passed directly to `spark-submit`. #[serde(default, skip_serializing_if = "Option::is_none")] pub repositories: Option>, + + /// A list of excluded packages that is passed directly to `spark-submit`. #[serde(default, skip_serializing_if = "Option::is_none")] pub exclude_packages: Option>, } From 3303668afff01a2727321aed836ccc45b5bc9f2b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 Dec 2023 14:19:50 +0100 Subject: [PATCH 04/15] removed version and stopped fields and others mandatory --- deploy/helm/spark-k8s-operator/crds/crds.yaml | 18 ++--- rust/crd/src/lib.rs | 53 +++++++------- .../src/spark_k8s_controller.rs | 73 ++++++++++++------- 3 files changed, 80 insertions(+), 64 deletions(-) diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml index 0745e3bc..6b6682ee 100644 --- a/deploy/helm/spark-k8s-operator/crds/crds.yaml +++ b/deploy/helm/spark-k8s-operator/crds/crds.yaml @@ -7068,7 +7068,7 @@ spec: type: object type: object image: - description: 'User-supplied image containing spark-job dependencies that will be copied to the specified volume mount. TODO: Where will they be copied from? Where to? docs link?' + description: User-supplied image containing spark-job dependencies that will be copied to the specified volume mount. See the [examples](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/examples). nullable: true type: string job: @@ -10154,14 +10154,14 @@ spec: type: object type: object mainApplicationFile: - description: 'The actual application file that will be called by `spark-submit`. TODO: What is the default?' - nullable: true + description: The actual application file that will be called by `spark-submit`. type: string mainClass: + description: The main class - i.e. entry point - for JVM artifacts. nullable: true type: string mode: - nullable: true + description: 'Mode: cluster or client. Currently only cluster is supported.' type: string s3connection: description: Configure an S3 connection that the SparkApplication has access to. Read more in the [Spark S3 usage guide](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/s3). @@ -10266,6 +10266,7 @@ spec: sparkConf: additionalProperties: type: string + description: A map of key/value strings that will be passed directly to spark-submit. nullable: true type: object sparkImage: @@ -10314,17 +10315,10 @@ spec: nullable: true type: string type: object - stopped: - nullable: true - type: boolean vectorAggregatorConfigMapName: description: Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). It must contain the key `ADDRESS` with the address of the Vector aggregator. Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) to learn how to configure log aggregation with Vector. nullable: true type: string - version: - description: Application version. TODO what is the default? - nullable: true - type: string volumes: items: description: Volume represents a named volume in a pod that may be accessed by any container in the pod. @@ -11402,6 +11396,8 @@ spec: nullable: true type: array required: + - mainApplicationFile + - mode - sparkImage type: object status: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 1b507ecb..08710bc1 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -104,21 +104,18 @@ pub struct SparkApplicationStatus { )] #[serde(rename_all = "camelCase")] pub struct SparkApplicationSpec { - /// Application version. TODO what is the default? - #[serde(default, skip_serializing_if = "Option::is_none")] - pub version: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub mode: Option, + /// Mode: cluster or client. Currently only cluster is supported. + pub mode: String, + + /// The main class - i.e. entry point - for JVM artifacts. #[serde(default, skip_serializing_if = "Option::is_none")] pub main_class: Option, /// The actual application file that will be called by `spark-submit`. - /// TODO: What is the default? - #[serde(default, skip_serializing_if = "Option::is_none")] - pub main_application_file: Option, + pub main_application_file: String, /// User-supplied image containing spark-job dependencies that will be copied to the specified volume mount. - /// TODO: Where will they be copied from? Where to? docs link? + /// See the [examples](DOCS_BASE_URL_PLACEHOLDER/spark-k8s/usage-guide/examples). #[serde(default, skip_serializing_if = "Option::is_none")] pub image: Option, @@ -141,10 +138,7 @@ pub struct SparkApplicationSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub executor: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stopped: Option, - - /// + /// A map of key/value strings that will be passed directly to spark-submit. #[serde(default, skip_serializing_if = "Option::is_none")] pub spark_conf: Option>, @@ -199,20 +193,16 @@ impl SparkApplication { format!("{app_name}-{role}-pod-template", app_name = self.name_any()) } - pub fn mode(&self) -> Option<&str> { - self.spec.mode.as_deref() + pub fn mode(&self) -> &str { + self.spec.mode.as_ref() } pub fn image(&self) -> Option<&str> { self.spec.image.as_deref() } - pub fn version(&self) -> Option<&str> { - self.spec.version.as_deref() - } - - pub fn application_artifact(&self) -> Option<&str> { - self.spec.main_application_file.as_deref() + pub fn application_artifact(&self) -> &str { + self.spec.main_application_file.as_ref() } pub fn requirements(&self) -> Option { @@ -418,11 +408,15 @@ impl SparkApplication { mounts } - pub fn build_recommended_labels<'a>(&'a self, role: &'a str) -> ObjectLabels { + pub fn build_recommended_labels<'a>( + &'a self, + app_version: &'a str, + role: &'a str, + ) -> ObjectLabels { ObjectLabels { owner: self, app_name: APP_NAME, - app_version: self.version().unwrap(), + app_version, // TODO &resolved_product_image.app_version_label, operator_name: OPERATOR_NAME, controller_name: CONTROLLER_NAME, role, @@ -438,7 +432,7 @@ impl SparkApplication { spark_image: &str, ) -> Result, Error> { // mandatory properties - let mode = self.mode().context(ObjectHasNoDeployModeSnafu)?; + let mode = self.mode(); let name = self.metadata.name.clone().context(ObjectHasNoNameSnafu)?; let mut submit_cmd: Vec = vec![]; @@ -585,9 +579,7 @@ impl SparkApplication { .map(|mc| format! {"--class {mc}"}), ); - let artifact = self - .application_artifact() - .context(ObjectHasNoArtifactSnafu)?; + let artifact = self.application_artifact(); submit_cmd.push(artifact.to_string()); if let Some(job_args) = self.spec.args.clone() { @@ -951,7 +943,10 @@ mod tests { kind: SparkApplication metadata: name: spark-examples + spec: + mode: cluster + mainApplicationFile: test.py sparkImage: productVersion: 1.2.3 "}) @@ -979,6 +974,8 @@ mod tests { metadata: name: spark-examples spec: + mode: cluster + mainApplicationFile: test.py sparkImage: productVersion: 1.2.3 job: @@ -1152,6 +1149,8 @@ mod tests { metadata: name: spark-examples spec: + mode: cluster + mainApplicationFile: test.py sparkImage: productVersion: 1.2.3 "#}) diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index 9e8dfb54..f1c6b72d 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -191,7 +191,8 @@ pub async fn reconcile(spark_application: Arc, ctx: Arc) .validated_role_config(&resolved_product_image, &ctx.product_config) .context(InvalidProductConfigSnafu)?; - let (serviceaccount, rolebinding) = build_spark_role_serviceaccount(&spark_application)?; + let (serviceaccount, rolebinding) = + build_spark_role_serviceaccount(&spark_application, &resolved_product_image)?; client .apply_patch(CONTROLLER_NAME, &serviceaccount, &serviceaccount) .await @@ -298,6 +299,7 @@ pub async fn reconcile(spark_application: Arc, ctx: Arc) submit_product_config, vector_aggregator_address.as_deref(), &submit_config.logging, + &resolved_product_image, )?; client .apply_patch( @@ -482,7 +484,10 @@ fn pod_template( // cleanly (specifically driver pods and related config maps) when the spark application is deleted. .ownerreference_from_resource(spark_application, None, None) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(spark_application.build_recommended_labels(&container_name)) + .with_recommended_labels( + spark_application + .build_recommended_labels(&spark_image.app_version_label, &container_name), + ) .build(), ) .add_container(cb.build()) @@ -580,7 +585,8 @@ fn pod_template_config_map( .ownerreference_from_resource(spark_application, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels( - spark_application.build_recommended_labels("pod-templates"), + spark_application + .build_recommended_labels(&spark_image.app_version_label, "pod-templates"), ) .build(), ) @@ -628,6 +634,7 @@ fn submit_job_config_map( product_config: Option<&HashMap>>, vector_aggregator_address: Option<&str>, logging: &Logging, + spark_image: &ResolvedProductImage, ) -> Result { let cm_name = spark_application.submit_job_config_map_name(); @@ -639,7 +646,10 @@ fn submit_job_config_map( .name(&cm_name) .ownerreference_from_resource(spark_application, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(spark_application.build_recommended_labels("spark-submit")) + .with_recommended_labels( + spark_application + .build_recommended_labels(&spark_image.app_version_label, "spark-submit"), + ) .build(), ); @@ -767,25 +777,27 @@ fn spark_job( )); } - let mut pod = PodTemplateSpec { - metadata: Some( - ObjectMetaBuilder::new() - .name("spark-submit") - .with_recommended_labels( - spark_application.build_recommended_labels("spark-job-template"), - ) - .build(), - ), - spec: Some(PodSpec { - containers, - restart_policy: Some("Never".to_string()), - service_account_name: serviceaccount.metadata.name.clone(), - volumes: Some(volumes), - image_pull_secrets: spark_image.pull_secrets.clone(), - security_context: Some(security_context()), - ..PodSpec::default() - }), - }; + let mut pod = + PodTemplateSpec { + metadata: Some( + ObjectMetaBuilder::new() + .name("spark-submit") + .with_recommended_labels(spark_application.build_recommended_labels( + &spark_image.app_version_label, + "spark-job-template", + )) + .build(), + ), + spec: Some(PodSpec { + containers, + restart_policy: Some("Never".to_string()), + service_account_name: serviceaccount.metadata.name.clone(), + volumes: Some(volumes), + image_pull_secrets: spark_image.pull_secrets.clone(), + security_context: Some(security_context()), + ..PodSpec::default() + }), + }; if let Some(submit_pod_overrides) = spark_application.pod_overrides(SparkApplicationRole::Submit) @@ -798,7 +810,10 @@ fn spark_job( .name_and_namespace(spark_application) .ownerreference_from_resource(spark_application, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(spark_application.build_recommended_labels("spark-job")) + .with_recommended_labels( + spark_application + .build_recommended_labels(&spark_image.app_version_label, "spark-job"), + ) .build(), spec: Some(JobSpec { template: pod, @@ -817,6 +832,7 @@ fn spark_job( /// They are deleted when the job is deleted. fn build_spark_role_serviceaccount( spark_app: &SparkApplication, + spark_image: &ResolvedProductImage, ) -> Result<(ServiceAccount, RoleBinding)> { let sa_name = spark_app.metadata.name.as_ref().unwrap().to_string(); let sa = ServiceAccount { @@ -825,7 +841,10 @@ fn build_spark_role_serviceaccount( .name(&sa_name) .ownerreference_from_resource(spark_app, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(spark_app.build_recommended_labels("service-account")) + .with_recommended_labels( + spark_app + .build_recommended_labels(&spark_image.app_version_label, "service-account"), + ) .build(), ..ServiceAccount::default() }; @@ -836,7 +855,9 @@ fn build_spark_role_serviceaccount( .name(binding_name) .ownerreference_from_resource(spark_app, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(spark_app.build_recommended_labels("role-binding")) + .with_recommended_labels( + spark_app.build_recommended_labels(&spark_image.app_version_label, "role-binding"), + ) .build(), role_ref: RoleRef { api_group: ClusterRole::GROUP.to_string(), From 774186ee7282ee48b1e1782b693c646de0253660 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 Dec 2023 15:29:11 +0100 Subject: [PATCH 05/15] added comments --- rust/crd/src/lib.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 08710bc1..f810055d 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -129,12 +129,18 @@ pub struct SparkApplicationSpec { #[serde(skip_serializing_if = "Option::is_none")] pub vector_aggregator_config_map_name: Option, + /// The job builds a spark-submit command, complete with arguments and referenced dependencies + /// such as templates, and passes it on to Spark. #[serde(default, skip_serializing_if = "Option::is_none")] pub job: Option>, + /// The driver role specifies the configuration that, together with the driver pod template, is used by + /// Spark to create driver pods. #[serde(default, skip_serializing_if = "Option::is_none")] pub driver: Option>, + /// The executor role specifies the configuration that, together with the driver pod template, is used by + /// Spark to create the executor pods. #[serde(default, skip_serializing_if = "Option::is_none")] pub executor: Option>, @@ -142,6 +148,9 @@ pub struct SparkApplicationSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub spark_conf: Option>, + /// Job dependencies: a list of python packages that will be installed via pip, a list of packages + /// or repositories that is passed directly to spark-submit, or a list of excluded packages + /// (also passed directly to spark-submit). #[serde(default, skip_serializing_if = "Option::is_none")] pub deps: Option, @@ -150,15 +159,21 @@ pub struct SparkApplicationSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub s3connection: Option, + /// Arguments passed directly to the job artifact. #[serde(default, skip_serializing_if = "Option::is_none")] pub args: Option>, + /// A list of volumes that can be made available to the job, driver or executors via their volume mounts. #[serde(default, skip_serializing_if = "Option::is_none")] pub volumes: Option>, + /// A list of environment variables that will be set in the job pod and the driver and executor + /// pod templates. #[serde(default, skip_serializing_if = "Option::is_none")] pub env: Option>, + /// The log file directory definition used by the Spark history server. + /// Currently only S3 buckets are supported. #[serde(default, skip_serializing_if = "Option::is_none")] pub log_file_directory: Option, } @@ -943,7 +958,6 @@ mod tests { kind: SparkApplication metadata: name: spark-examples - spec: mode: cluster mainApplicationFile: test.py From d63b78ec1aaae11a5c5ec5dd549535450f4f047e Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 Dec 2023 17:12:56 +0100 Subject: [PATCH 06/15] history server docs. Removed test step that does not work following the deletecollection fix --- rust/crd/src/history.rs | 15 +++++++++++++-- tests/templates/kuttl/resources/20-assert.yaml.j2 | 7 ------- 2 files changed, 13 insertions(+), 9 deletions(-) delete mode 100644 tests/templates/kuttl/resources/20-assert.yaml.j2 diff --git a/rust/crd/src/history.rs b/rust/crd/src/history.rs index 0d0d5a28..2de608b2 100644 --- a/rust/crd/src/history.rs +++ b/rust/crd/src/history.rs @@ -49,7 +49,9 @@ pub enum Error { CannotRetrieveRoleGroup { role_group: String }, } -/// TODO +/// A Spark cluster history server component. This resource is managed by the Stackable operator +/// for Apache Spark. Find more information on how to use it in the +/// [operator documentation](DOCS_BASE_URL_PLACEHOLDER/spark-k8s/usage-guide/history-server). #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, Serialize)] #[kube( group = "spark.stackable.tech", @@ -66,16 +68,25 @@ pub enum Error { #[serde(rename_all = "camelCase")] pub struct SparkHistoryServerSpec { pub image: ProductImage, - /// Global Spark history server configuration that applies to all roles and role groups + + /// Global Spark history server configuration that applies to all roles and role groups. #[serde(default)] pub cluster_config: SparkHistoryServerClusterConfig, + /// Name of the Vector aggregator discovery ConfigMap. /// It must contain the key `ADDRESS` with the address of the Vector aggregator. #[serde(skip_serializing_if = "Option::is_none")] pub vector_aggregator_config_map_name: Option, + + /// The log file directory definition used by the Spark history server. + /// Currently only S3 buckets are supported. pub log_file_directory: LogFileDirectorySpec, + + /// A map of key/value strings that will be passed directly to Spark when deploying the history server. #[serde(default)] pub spark_conf: BTreeMap, + + /// A history server node role definition. pub nodes: Role, } diff --git a/tests/templates/kuttl/resources/20-assert.yaml.j2 b/tests/templates/kuttl/resources/20-assert.yaml.j2 deleted file mode 100644 index 0fd00fea..00000000 --- a/tests/templates/kuttl/resources/20-assert.yaml.j2 +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 120 -commands: - - script: kubectl get pods -n $NAMESPACE resources-crd-exec-1 -o json | jq '.spec.containers[].env[] | select(.name == "SPARK_EXECUTOR_CORES").value' | grep -w "2" - - script: kubectl get pods -n $NAMESPACE resources-sparkconf-exec-1 -o json | jq '.spec.containers[].env[] | select(.name == "SPARK_EXECUTOR_CORES").value' | grep -w "2" From d32eb7213360d417647e122bd900e9f83b82ed2d Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 Dec 2023 17:17:25 +0100 Subject: [PATCH 07/15] regenerate charts --- deploy/helm/spark-k8s-operator/crds/crds.yaml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml index 6b6682ee..63df5d91 100644 --- a/deploy/helm/spark-k8s-operator/crds/crds.yaml +++ b/deploy/helm/spark-k8s-operator/crds/crds.yaml @@ -29,11 +29,13 @@ spec: The SparkApplication CRD looks a little different than the CRDs of the other products on the Stackable Data Platform. properties: args: + description: Arguments passed directly to the job artifact. items: type: string nullable: true type: array deps: + description: 'Job dependencies: a list of python packages that will be installed via pip, a list of packages or repositories that is passed directly to spark-submit, or a list of excluded packages (also passed directly to spark-submit).' nullable: true properties: excludePackages: @@ -62,6 +64,7 @@ spec: type: array type: object driver: + description: The driver role specifies the configuration that, together with the driver pod template, is used by Spark to create driver pods. nullable: true properties: cliOverrides: @@ -3509,6 +3512,7 @@ spec: type: object type: object env: + description: A list of environment variables that will be set in the job pod and the driver and executor pod templates. items: description: EnvVar represents an environment variable present in a Container. properties: @@ -3585,6 +3589,7 @@ spec: nullable: true type: array executor: + description: The executor role specifies the configuration that, together with the driver pod template, is used by Spark to create the executor pods. nullable: true properties: cliOverrides: @@ -7072,6 +7077,7 @@ spec: nullable: true type: string job: + description: The job builds a spark-submit command, complete with arguments and referenced dependencies such as templates, and passes it on to Spark. nullable: true properties: cliOverrides: @@ -10019,6 +10025,7 @@ spec: type: object type: object logFileDirectory: + description: The log file directory definition used by the Spark history server. Currently only S3 buckets are supported. nullable: true oneOf: - required: @@ -10320,6 +10327,7 @@ spec: nullable: true type: string volumes: + description: A list of volumes that can be made available to the job, driver or executors via their volume mounts. items: description: Volume represents a named volume in a pod that may be accessed by any container in the pod. properties: @@ -11441,12 +11449,12 @@ spec: description: Auto-generated derived type for SparkHistoryServerSpec via `CustomResource` properties: spec: - description: TODO + description: A Spark cluster history server component. This resource is managed by the Stackable operator for Apache Spark. Find more information on how to use it in the [operator documentation](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/history-server). properties: clusterConfig: default: listenerClass: cluster-internal - description: Global Spark history server configuration that applies to all roles and role groups + description: Global Spark history server configuration that applies to all roles and role groups. properties: listenerClass: default: cluster-internal @@ -11513,6 +11521,7 @@ spec: type: string type: object logFileDirectory: + description: The log file directory definition used by the Spark history server. Currently only S3 buckets are supported. oneOf: - required: - s3 @@ -11647,7 +11656,7 @@ spec: type: object type: object nodes: - description: This struct represents a role - e.g. HDFS datanodes or Trino workers. It has a [`HashMap`] containing all the roleGroups that are part of this role. Additionally, there is a `config`, which is configurable at the role *and* roleGroup level. Everything at roleGroup level is merged on top of what is configured on role level using the [`Merge`] trait. There is also a second form of config, which can only be configured at role level, the `roleConfig`. + description: A history server node role definition. properties: cliOverrides: additionalProperties: @@ -18563,6 +18572,7 @@ spec: additionalProperties: type: string default: {} + description: A map of key/value strings that will be passed directly to Spark when deploying the history server. type: object vectorAggregatorConfigMapName: description: Name of the Vector aggregator discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. From 1b440e1cc6a5a7cfae835d720a7f7daa7dda26f8 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 12 Dec 2023 22:04:12 +0100 Subject: [PATCH 08/15] updated docs --- .../spark-k8s/pages/crd-reference.adoc | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/docs/modules/spark-k8s/pages/crd-reference.adoc b/docs/modules/spark-k8s/pages/crd-reference.adoc index 7a8b1135..98b235b5 100644 --- a/docs/modules/spark-k8s/pages/crd-reference.adoc +++ b/docs/modules/spark-k8s/pages/crd-reference.adoc @@ -12,49 +12,46 @@ Below are listed the CRD fields that can be defined by the user: |`SparkApplication` |`metadata.name` -|Application name - -|`spec.version` -|Application version +|Application name. Mandatory. |`spec.mode` -| `cluster` or `client`. Currently only `cluster` is supported +| `cluster` or `client`. Currently only `cluster` is supported. Mandatory. |`spec.image` -|User-supplied image containing spark-job dependencies that will be copied to the specified volume mount +|User-supplied image containing spark-job dependencies that will be copied to the specified volume mount. |`spec.sparkImage` -| Spark image which will be deployed to driver and executor pods, which must contain spark environment needed by the job e.g. `docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable0.0.0-dev` +| Spark image which will be deployed to driver and executor pods, which must contain spark environment needed by the job e.g. `docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable0.0.0-dev`. Mandatory. |`spec.sparkImagePullPolicy` -| Optional Enum (one of `Always`, `IfNotPresent` or `Never`) that determines the pull policy of the spark job image +| Optional Enum (one of `Always`, `IfNotPresent` or `Never`) that determines the pull policy of the spark job image. |`spec.sparkImagePullSecrets` -| An optional list of references to secrets in the same namespace to use for pulling any of the images used by a `SparkApplication` resource. Each reference has a single property (`name`) that must contain a reference to a valid secret +| An optional list of references to secrets in the same namespace to use for pulling any of the images used by a `SparkApplication` resource. Each reference has a single property (`name`) that must contain a reference to a valid secret. |`spec.mainApplicationFile` -|The actual application file that will be called by `spark-submit` +|The actual application file that will be called by `spark-submit`. Mandatory. |`spec.mainClass` -|The main class i.e. entry point for JVM artifacts +|The main class/entry point for JVM artifacts. |`spec.args` -|Arguments passed directly to the job artifact +|Arguments passed directly to the job artifact. |`spec.s3connection` |S3 connection specification. See the xref:concepts:s3.adoc[] for more details. |`spec.sparkConf` -|A map of key/value strings that will be passed directly to `spark-submit` +|A map of key/value strings that will be passed directly to `spark-submit. |`spec.deps.requirements` -|A list of python packages that will be installed via `pip` +|A list of python packages that will be installed via `pip`. |`spec.deps.packages` -|A list of packages that is passed directly to `spark-submit` +|A list of packages that is passed directly to `spark-submit`. |`spec.deps.excludePackages` -|A list of excluded packages that is passed directly to `spark-submit` +|A list of excluded packages that is passed directly to `spark-submit`. |`spec.deps.repositories` |A list of repositories that is passed directly to `spark-submit` @@ -92,23 +89,23 @@ Below are listed the CRD fields that can be defined by the user: |`spec.executor.resources` |Resources specification for the executor Pods -|`spec.executor.instances` -|Number of executor instances launched for this job +|`spec.executor.replicas` +|Number of executor instances launched for this job. |`spec.executor.volumeMounts` -|A list of mounted volumes for each executor +|A list of mounted volumes for each executor. |`spec.executor.volumeMounts.name` -|Name of mount +|Name of mount. |`spec.executor.volumeMounts.mountPath` -|Volume mount path +|Volume mount path. |`spec.executor.affinity` |Driver Pod placement affinity. See xref:usage-guide/operations/pod-placement.adoc[] for details. |`spec.executor.logging` -|Logging aggregation for the executor Pods. See xref:concepts:logging.adoc[] for details +|Logging aggregation for the executor Pods. See xref:concepts:logging.adoc[] for details. |`spec.logFileDirectory.bucket` |S3 bucket definition where applications should publish events for the Spark History server. From bad86310c6d82b27bee9a7ded90f8ea2cd8a0661 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 13 Dec 2023 08:51:46 +0100 Subject: [PATCH 09/15] updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dec5966..fcec0336 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. ### Added - Various documentation of the CRD ([#319]). +- [BREAKING] Removed version field ([#319]). [#319]: https://github.com/stackabletech/spark-k8s-operator/pull/319 From 27410e5af1e06779434a109b457555c8d41d7a08 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 13 Dec 2023 14:43:53 +0100 Subject: [PATCH 10/15] review suggestions --- Cargo.lock | 120 +++++++++--------- Cargo.toml | 2 +- deploy/helm/spark-k8s-operator/crds/crds.yaml | 79 +++++++----- rust/crd/src/lib.rs | 80 +++++------- 4 files changed, 139 insertions(+), 142 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 388ebf86..d6206a22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -673,12 +673,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.1" @@ -840,16 +834,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.0.2" @@ -857,7 +841,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown", ] [[package]] @@ -1045,7 +1029,7 @@ dependencies = [ "backoff", "derivative", "futures 0.3.28", - "hashbrown 0.14.1", + "hashbrown", "json-patch", "k8s-openapi", "kube-client", @@ -1205,71 +1189,62 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "opentelemetry" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" +checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ - "opentelemetry_api", - "opentelemetry_sdk", + "futures-core", + "futures-sink", + "indexmap", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", ] [[package]] name = "opentelemetry-jaeger" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876958ba9084f390f913fcf04ddf7bbbb822898867bb0a51cc28f2b9e5c1b515" +checksum = "e617c66fd588e40e0dbbd66932fdc87393095b125d4459b1a3a10feb1712f8a1" dependencies = [ "async-trait", "futures-core", "futures-util", "opentelemetry", "opentelemetry-semantic-conventions", + "opentelemetry_sdk", "thrift", "tokio", ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269" +checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" dependencies = [ "opentelemetry", ] -[[package]] -name = "opentelemetry_api" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" -dependencies = [ - "futures-channel", - "futures-util", - "indexmap 1.9.3", - "js-sys", - "once_cell", - "pin-project-lite", - "thiserror", - "urlencoding", -] - [[package]] name = "opentelemetry_sdk" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" +checksum = "968ba3f2ca03e90e5187f5e4f46c791ef7f2c163ae87789c8ce5f5ca3b7b7de5" dependencies = [ "async-trait", "crossbeam-channel", "futures-channel", "futures-executor", "futures-util", + "glob", "once_cell", - "opentelemetry_api", - "ordered-float 3.9.1", + "opentelemetry", + "ordered-float 4.2.0", "percent-encoding", "rand", - "regex", "thiserror", "tokio", "tokio-stream", @@ -1286,9 +1261,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "3.9.1" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" +checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" dependencies = [ "num-traits", ] @@ -1802,7 +1777,7 @@ version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ - "indexmap 2.0.2", + "indexmap", "itoa", "ryu", "serde", @@ -1823,7 +1798,7 @@ version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ - "indexmap 2.0.2", + "indexmap", "itoa", "ryu", "serde", @@ -1945,8 +1920,8 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "stackable-operator" -version = "0.57.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.57.0#ab5c5c3f220ae9449e82f6861f44a4a9a6fb7b6b" +version = "0.58.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.58.1#ab309d577e1937834f2adcbcd647822aa9c2ae43" dependencies = [ "chrono", "clap", @@ -1961,6 +1936,7 @@ dependencies = [ "lazy_static", "opentelemetry", "opentelemetry-jaeger", + "opentelemetry_sdk", "product-config", "rand", "regex", @@ -1977,12 +1953,13 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "url", ] [[package]] name = "stackable-operator-derive" -version = "0.57.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.57.0#ab5c5c3f220ae9449e82f6861f44a4a9a6fb7b6b" +version = "0.58.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.58.1#ab309d577e1937834f2adcbcd647822aa9c2ae43" dependencies = [ "darling", "proc-macro2", @@ -2249,7 +2226,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.0.2", + "indexmap", "serde", "serde_spanned", "toml_datetime", @@ -2363,20 +2340,33 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-opentelemetry" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8" +checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" dependencies = [ + "js-sys", "once_cell", "opentelemetry", "opentelemetry_sdk", "smallvec", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.2.0", "tracing-subscriber", + "web-time", ] [[package]] @@ -2394,7 +2384,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.1.3", ] [[package]] @@ -2583,6 +2573,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57099a701fb3a8043f993e8228dc24229c7b942e2b009a1b962e54489ba1d3bf" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 032627b6..fe5ad768 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.7" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.57.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.58.1" } strum = { version = "0.25", features = ["derive"] } tokio = { version = "1.29", features = ["full"] } tracing = "0.1" diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml index 63df5d91..d643b79f 100644 --- a/deploy/helm/spark-k8s-operator/crds/crds.yaml +++ b/deploy/helm/spark-k8s-operator/crds/crds.yaml @@ -29,38 +29,38 @@ spec: The SparkApplication CRD looks a little different than the CRDs of the other products on the Stackable Data Platform. properties: args: + default: [] description: Arguments passed directly to the job artifact. items: type: string - nullable: true type: array deps: description: 'Job dependencies: a list of python packages that will be installed via pip, a list of packages or repositories that is passed directly to spark-submit, or a list of excluded packages (also passed directly to spark-submit).' nullable: true properties: excludePackages: + default: [] description: A list of excluded packages that is passed directly to `spark-submit`. items: type: string - nullable: true type: array packages: + default: [] description: A list of packages that is passed directly to `spark-submit`. items: type: string - nullable: true type: array repositories: + default: [] description: A list of repositories that is passed directly to `spark-submit`. items: type: string - nullable: true type: array requirements: + default: [] description: 'Under the `requirements` you can specify Python dependencies that will be installed with `pip`. Example: `tabulate==0.8.9`' items: type: string - nullable: true type: array type: object driver: @@ -3512,6 +3512,7 @@ spec: type: object type: object env: + default: [] description: A list of environment variables that will be set in the job pod and the driver and executor pod templates. items: description: EnvVar represents an environment variable present in a Container. @@ -3586,7 +3587,6 @@ spec: required: - name type: object - nullable: true type: array executor: description: The executor role specifies the configuration that, together with the driver pod template, is used by Spark to create the executor pods. @@ -10068,27 +10068,30 @@ spec: nullable: true type: string credentials: - description: If the S3 uses authentication you have to specify you S3 credentials. In the most cases a SecretClass providing `accessKey` and `secretKey` is sufficient. + description: If the S3 uses authentication you have to specify you S3 credentials. In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) providing `accessKey` and `secretKey` is sufficient. nullable: true properties: scope: - description: '[Scope](https://docs.stackable.tech/secret-operator/scope.html) of the [SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html)' + description: '[Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass).' nullable: true properties: node: default: false + description: The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. This will typically be the DNS name of the node. type: boolean pod: default: false + description: The pod scope is resolved to the name of the Kubernetes Pod. This allows the secret to differentiate between StatefulSet replicas. type: boolean services: default: [] + description: The service scope allows Pod objects to specify custom scopes. This should typically correspond to Service objects that the Pod participates in. items: type: string type: array type: object secretClass: - description: '[SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html) containing the LDAP bind credentials' + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' type: string required: - secretClass @@ -10108,7 +10111,7 @@ spec: nullable: true properties: verification: - description: The verification method used to verify the certificates of the server and/or the client + description: The verification method used to verify the certificates of the server and/or the client. oneOf: - required: - none @@ -10116,13 +10119,13 @@ spec: - server properties: none: - description: Use TLS but don't verify certificates + description: Use TLS but don't verify certificates. type: object server: - description: Use TLS and ca certificate to verify the server + description: Use TLS and a CA certificate to verify the server. properties: caCert: - description: Ca cert to verify the server + description: CA cert to verify the server. oneOf: - required: - webPki @@ -10130,10 +10133,10 @@ spec: - secretClass properties: secretClass: - description: Name of the SecretClass which will provide the ca cert. Note that a SecretClass does not need to have a key but can also work with just a ca cert. So if you got provided with a ca cert but don't have access to the key you can still use this method. + description: Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. Note that a SecretClass does not need to have a key but can also work with just a CA certificate, so if you got provided with a CA cert but don't have access to the key you can still use this method. type: string webPki: - description: Use TLS and the ca certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. + description: Use TLS and the CA certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. type: object type: object required: @@ -10190,27 +10193,30 @@ spec: nullable: true type: string credentials: - description: If the S3 uses authentication you have to specify you S3 credentials. In the most cases a SecretClass providing `accessKey` and `secretKey` is sufficient. + description: If the S3 uses authentication you have to specify you S3 credentials. In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) providing `accessKey` and `secretKey` is sufficient. nullable: true properties: scope: - description: '[Scope](https://docs.stackable.tech/secret-operator/scope.html) of the [SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html)' + description: '[Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass).' nullable: true properties: node: default: false + description: The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. This will typically be the DNS name of the node. type: boolean pod: default: false + description: The pod scope is resolved to the name of the Kubernetes Pod. This allows the secret to differentiate between StatefulSet replicas. type: boolean services: default: [] + description: The service scope allows Pod objects to specify custom scopes. This should typically correspond to Service objects that the Pod participates in. items: type: string type: array type: object secretClass: - description: '[SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html) containing the LDAP bind credentials' + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' type: string required: - secretClass @@ -10230,7 +10236,7 @@ spec: nullable: true properties: verification: - description: The verification method used to verify the certificates of the server and/or the client + description: The verification method used to verify the certificates of the server and/or the client. oneOf: - required: - none @@ -10238,13 +10244,13 @@ spec: - server properties: none: - description: Use TLS but don't verify certificates + description: Use TLS but don't verify certificates. type: object server: - description: Use TLS and ca certificate to verify the server + description: Use TLS and a CA certificate to verify the server. properties: caCert: - description: Ca cert to verify the server + description: CA cert to verify the server. oneOf: - required: - webPki @@ -10252,10 +10258,10 @@ spec: - secretClass properties: secretClass: - description: Name of the SecretClass which will provide the ca cert. Note that a SecretClass does not need to have a key but can also work with just a ca cert. So if you got provided with a ca cert but don't have access to the key you can still use this method. + description: Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. Note that a SecretClass does not need to have a key but can also work with just a CA certificate, so if you got provided with a CA cert but don't have access to the key you can still use this method. type: string webPki: - description: Use TLS and the ca certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. + description: Use TLS and the CA certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. type: object type: object required: @@ -10273,8 +10279,8 @@ spec: sparkConf: additionalProperties: type: string + default: {} description: A map of key/value strings that will be passed directly to spark-submit. - nullable: true type: object sparkImage: anyOf: @@ -10327,6 +10333,7 @@ spec: nullable: true type: string volumes: + default: [] description: A list of volumes that can be made available to the job, driver or executors via their volume mounts. items: description: Volume represents a named volume in a pod that may be accessed by any container in the pod. @@ -11401,7 +11408,6 @@ spec: required: - name type: object - nullable: true type: array required: - mainApplicationFile @@ -11563,27 +11569,30 @@ spec: nullable: true type: string credentials: - description: If the S3 uses authentication you have to specify you S3 credentials. In the most cases a SecretClass providing `accessKey` and `secretKey` is sufficient. + description: If the S3 uses authentication you have to specify you S3 credentials. In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) providing `accessKey` and `secretKey` is sufficient. nullable: true properties: scope: - description: '[Scope](https://docs.stackable.tech/secret-operator/scope.html) of the [SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html)' + description: '[Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass).' nullable: true properties: node: default: false + description: The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. This will typically be the DNS name of the node. type: boolean pod: default: false + description: The pod scope is resolved to the name of the Kubernetes Pod. This allows the secret to differentiate between StatefulSet replicas. type: boolean services: default: [] + description: The service scope allows Pod objects to specify custom scopes. This should typically correspond to Service objects that the Pod participates in. items: type: string type: array type: object secretClass: - description: '[SecretClass](https://docs.stackable.tech/secret-operator/secretclass.html) containing the LDAP bind credentials' + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' type: string required: - secretClass @@ -11603,7 +11612,7 @@ spec: nullable: true properties: verification: - description: The verification method used to verify the certificates of the server and/or the client + description: The verification method used to verify the certificates of the server and/or the client. oneOf: - required: - none @@ -11611,13 +11620,13 @@ spec: - server properties: none: - description: Use TLS but don't verify certificates + description: Use TLS but don't verify certificates. type: object server: - description: Use TLS and ca certificate to verify the server + description: Use TLS and a CA certificate to verify the server. properties: caCert: - description: Ca cert to verify the server + description: CA cert to verify the server. oneOf: - required: - webPki @@ -11625,10 +11634,10 @@ spec: - secretClass properties: secretClass: - description: Name of the SecretClass which will provide the ca cert. Note that a SecretClass does not need to have a key but can also work with just a ca cert. So if you got provided with a ca cert but don't have access to the key you can still use this method. + description: Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. Note that a SecretClass does not need to have a key but can also work with just a CA certificate, so if you got provided with a CA cert but don't have access to the key you can still use this method. type: string webPki: - description: Use TLS and the ca certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. + description: Use TLS and the CA certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. type: object type: object required: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index f810055d..a0c9bb25 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -145,8 +145,8 @@ pub struct SparkApplicationSpec { pub executor: Option>, /// A map of key/value strings that will be passed directly to spark-submit. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub spark_conf: Option>, + #[serde(default)] + pub spark_conf: HashMap, /// Job dependencies: a list of python packages that will be installed via pip, a list of packages /// or repositories that is passed directly to spark-submit, or a list of excluded packages @@ -160,17 +160,17 @@ pub struct SparkApplicationSpec { pub s3connection: Option, /// Arguments passed directly to the job artifact. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub args: Option>, + #[serde(default)] + pub args: Vec, /// A list of volumes that can be made available to the job, driver or executors via their volume mounts. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub volumes: Option>, + #[serde(default)] + pub volumes: Vec, /// A list of environment variables that will be set in the job pod and the driver and executor /// pod templates. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub env: Option>, + #[serde(default)] + pub env: Vec, /// The log file directory definition used by the Spark history server. /// Currently only S3 buckets are supported. @@ -183,20 +183,20 @@ pub struct SparkApplicationSpec { pub struct JobDependencies { /// Under the `requirements` you can specify Python dependencies that will be installed with `pip`. /// Example: `tabulate==0.8.9` - #[serde(default, skip_serializing_if = "Option::is_none")] - pub requirements: Option>, + #[serde(default)] + pub requirements: Vec, /// A list of packages that is passed directly to `spark-submit`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub packages: Option>, + #[serde(default)] + pub packages: Vec, /// A list of repositories that is passed directly to `spark-submit`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub repositories: Option>, + #[serde(default)] + pub repositories: Vec, /// A list of excluded packages that is passed directly to `spark-submit`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub exclude_packages: Option>, + #[serde(default)] + pub exclude_packages: Vec, } impl SparkApplication { @@ -224,16 +224,15 @@ impl SparkApplication { self.spec .deps .as_ref() - .and_then(|deps| deps.requirements.as_ref()) + .map(|deps| &deps.requirements) .map(|req| req.join(" ")) } pub fn packages(&self) -> Vec { - self.spec - .deps - .as_ref() - .and_then(|deps| deps.packages.clone()) - .unwrap_or_default() + match self.spec.deps.as_ref() { + Some(deps) => deps.clone().packages, + None => Vec::new(), + } } pub fn volumes( @@ -242,14 +241,7 @@ impl SparkApplication { s3logdir: &Option, log_config_map: &str, ) -> Vec { - let mut result: Vec = self - .spec - .volumes - .as_ref() - .iter() - .flat_map(|v| v.iter()) - .cloned() - .collect(); + let mut result: Vec = self.spec.volumes.clone(); if self.spec.image.is_some() { result.push( @@ -527,14 +519,14 @@ impl SparkApplication { // repositories and packages arguments if let Some(deps) = self.spec.deps.clone() { - submit_cmd.extend( - deps.repositories - .map(|r| format!("--repositories {}", r.join(","))), - ); - submit_cmd.extend( - deps.packages - .map(|p| format!("--conf spark.jars.packages={}", p.join(","))), - ); + submit_cmd.extend(vec![format!( + "--repositories {}", + deps.repositories.join(",") + )]); + submit_cmd.extend(vec![format!( + "--conf spark.jars.packages={}", + deps.packages.join(",") + )]); } // some command elements need to be initially stored in a map (to allow overwrites) and @@ -579,9 +571,8 @@ impl SparkApplication { } // conf arguments: these should follow - and thus override - values set from resource limits above - if let Some(spark_conf) = self.spec.spark_conf.clone() { - submit_conf.extend(spark_conf); - } + submit_conf.extend(self.spec.spark_conf.clone()); + // ...before being added to the command collection for (key, value) in submit_conf { submit_cmd.push(format!("--conf \"{key}={value}\"")); @@ -597,9 +588,7 @@ impl SparkApplication { let artifact = self.application_artifact(); submit_cmd.push(artifact.to_string()); - if let Some(job_args) = self.spec.args.clone() { - submit_cmd.extend(job_args); - } + submit_cmd.extend(self.spec.args.clone()); Ok(submit_cmd) } @@ -609,8 +598,7 @@ impl SparkApplication { s3conn: &Option, s3logdir: &Option, ) -> Vec { - let tmp = self.spec.env.as_ref(); - let mut e: Vec = tmp.iter().flat_map(|e| e.iter()).cloned().collect(); + let mut e: Vec = self.spec.env.clone(); if self.requirements().is_some() { e.push(EnvVar { name: "PYTHONPATH".to_string(), From e4e67730e35f45f02789332f372361587b91edc2 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 13 Dec 2023 14:49:58 +0100 Subject: [PATCH 11/15] expanded changelog comment --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcec0336..b0d99e3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ All notable changes to this project will be documented in this file. ### Added - Various documentation of the CRD ([#319]). -- [BREAKING] Removed version field ([#319]). +- [BREAKING] Removed version field. Several attributes have been changed to mandatory. While this change is technically breaking, + existing Spark jobs would not have worked before as these attributes were necessary ([#319]). [#319]: https://github.com/stackabletech/spark-k8s-operator/pull/319 From 2a1bb778852ddad452e83593939a51294765afa5 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 13 Dec 2023 17:05:43 +0100 Subject: [PATCH 12/15] changed mode to enum, make JobDependencies non-optional --- CHANGELOG.md | 9 ++++----- rust/crd/src/lib.rs | 37 ++++++++++++++++--------------------- rust/crd/src/roles.rs | 6 ++++++ 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0d99e3c..3f55b6a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,11 @@ All notable changes to this project will be documented in this file. ## [Unreleased] -### Added +### Changed - Various documentation of the CRD ([#319]). -- [BREAKING] Removed version field. Several attributes have been changed to mandatory. While this change is technically breaking, - existing Spark jobs would not have worked before as these attributes were necessary ([#319]). - -[#319]: https://github.com/stackabletech/spark-k8s-operator/pull/319 +- [BREAKING] Removed version field. Several attributes have been changed to mandatory. While this change is + technically breaking, existing Spark jobs would not have worked before as these attributes were necessary ([#319]). ### Fixed @@ -18,6 +16,7 @@ All notable changes to this project will be documented in this file. messages in the spark driver log (`User "system:serviceaccount:default:my-spark-app" cannot deletecollection resource "configmaps" in API group "" in the namespace "default".`) ([#313]). [#313]: https://github.com/stackabletech/spark-k8s-operator/pull/313 +[#319]: https://github.com/stackabletech/spark-k8s-operator/pull/319 ## [23.11.0] - 2023-11-24 diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index a0c9bb25..2a12958b 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -105,7 +105,7 @@ pub struct SparkApplicationStatus { #[serde(rename_all = "camelCase")] pub struct SparkApplicationSpec { /// Mode: cluster or client. Currently only cluster is supported. - pub mode: String, + pub mode: SparkMode, /// The main class - i.e. entry point - for JVM artifacts. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -151,8 +151,8 @@ pub struct SparkApplicationSpec { /// Job dependencies: a list of python packages that will be installed via pip, a list of packages /// or repositories that is passed directly to spark-submit, or a list of excluded packages /// (also passed directly to spark-submit). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub deps: Option, + #[serde(default)] + pub deps: JobDependencies, /// Configure an S3 connection that the SparkApplication has access to. /// Read more in the [Spark S3 usage guide](DOCS_BASE_URL_PLACEHOLDER/spark-k8s/usage-guide/s3). @@ -208,10 +208,6 @@ impl SparkApplication { format!("{app_name}-{role}-pod-template", app_name = self.name_any()) } - pub fn mode(&self) -> &str { - self.spec.mode.as_ref() - } - pub fn image(&self) -> Option<&str> { self.spec.image.as_deref() } @@ -221,18 +217,14 @@ impl SparkApplication { } pub fn requirements(&self) -> Option { - self.spec - .deps - .as_ref() - .map(|deps| &deps.requirements) - .map(|req| req.join(" ")) + if !self.spec.deps.requirements.is_empty() { + return Some(self.spec.deps.requirements.join(" ")); + } + None } pub fn packages(&self) -> Vec { - match self.spec.deps.as_ref() { - Some(deps) => deps.clone().packages, - None => Vec::new(), - } + self.spec.deps.packages.clone() } pub fn volumes( @@ -423,7 +415,7 @@ impl SparkApplication { ObjectLabels { owner: self, app_name: APP_NAME, - app_version, // TODO &resolved_product_image.app_version_label, + app_version, operator_name: OPERATOR_NAME, controller_name: CONTROLLER_NAME, role, @@ -439,7 +431,7 @@ impl SparkApplication { spark_image: &str, ) -> Result, Error> { // mandatory properties - let mode = self.mode(); + let mode = &self.spec.mode; let name = self.metadata.name.clone().context(ObjectHasNoNameSnafu)?; let mut submit_cmd: Vec = vec![]; @@ -518,14 +510,17 @@ impl SparkApplication { ]); // repositories and packages arguments - if let Some(deps) = self.spec.deps.clone() { + if !self.spec.deps.repositories.is_empty() { submit_cmd.extend(vec![format!( "--repositories {}", - deps.repositories.join(",") + self.spec.deps.repositories.join(",") )]); + } + + if !self.spec.deps.packages.is_empty() { submit_cmd.extend(vec![format!( "--conf spark.jars.packages={}", - deps.packages.join(",") + self.spec.deps.packages.join(",") )]); } diff --git a/rust/crd/src/roles.rs b/rust/crd/src/roles.rs index 42f0e275..cdd94508 100644 --- a/rust/crd/src/roles.rs +++ b/rust/crd/src/roles.rs @@ -88,6 +88,12 @@ pub enum SparkContainer { Vector, Tls, } +#[derive(Clone, Debug, Deserialize, Display, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum SparkMode { + Cluster, + Client, +} #[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] #[fragment_attrs( From da2c940619c4227bc03a3df4c9c72eac001f0118 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 13 Dec 2023 17:12:16 +0100 Subject: [PATCH 13/15] regenerate charts --- deploy/helm/spark-k8s-operator/crds/crds.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml index d643b79f..2e4f7ce4 100644 --- a/deploy/helm/spark-k8s-operator/crds/crds.yaml +++ b/deploy/helm/spark-k8s-operator/crds/crds.yaml @@ -35,8 +35,12 @@ spec: type: string type: array deps: + default: + requirements: [] + packages: [] + repositories: [] + excludePackages: [] description: 'Job dependencies: a list of python packages that will be installed via pip, a list of packages or repositories that is passed directly to spark-submit, or a list of excluded packages (also passed directly to spark-submit).' - nullable: true properties: excludePackages: default: [] @@ -10172,6 +10176,9 @@ spec: type: string mode: description: 'Mode: cluster or client. Currently only cluster is supported.' + enum: + - cluster + - client type: string s3connection: description: Configure an S3 connection that the SparkApplication has access to. Read more in the [Spark S3 usage guide](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/s3). From 2b1b0934e3e027943cce34580266ba0f680b9d2b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 13 Dec 2023 17:20:18 +0100 Subject: [PATCH 14/15] removed all mentions of removed version field --- docs/modules/spark-k8s/examples/example-encapsulated.yaml | 1 - docs/modules/spark-k8s/examples/example-history-app.yaml | 1 - docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml | 1 - .../examples/example-sparkapp-external-dependencies.yaml | 1 - docs/modules/spark-k8s/examples/example-sparkapp-image.yaml | 1 - docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml | 1 - docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml | 1 - docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml | 1 - .../spark-k8s/examples/getting_started/getting_started.sh | 1 - .../spark-k8s/examples/getting_started/getting_started.sh.j2 | 1 - examples/ny-tlc-report-external-dependencies.yaml | 1 - examples/ny-tlc-report-image.yaml | 1 - examples/ny-tlc-report.yaml | 1 - tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 | 1 - .../logging/05-deploy-automatic-log-config-spark-app.yaml.j2 | 1 - .../kuttl/logging/06-deploy-custom-log-config-spark-app.yaml.j2 | 1 - .../logging/07-deploy-automatic-log-config-pyspark-app.yaml.j2 | 1 - .../logging/08-deploy-custom-log-config-pyspark-app.yaml.j2 | 1 - tests/templates/kuttl/pod_overrides/10-deploy-spark-app.yaml.j2 | 1 - .../kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 | 1 - .../kuttl/pyspark-ny-public-s3/10-deploy-spark-app.yaml.j2 | 1 - tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 | 1 - tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 | 1 - tests/templates/kuttl/smoke/10-deploy-spark-app.yaml.j2 | 1 - tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 | 1 - .../kuttl/spark-history-server/10-deploy-spark-app.yaml.j2 | 1 - .../kuttl/spark-history-server/12-deploy-spark-app.yaml.j2 | 1 - .../kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 | 1 - .../kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 | 1 - .../kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 | 1 - 30 files changed, 30 deletions(-) diff --git a/docs/modules/spark-k8s/examples/example-encapsulated.yaml b/docs/modules/spark-k8s/examples/example-encapsulated.yaml index bdc6d3f4..e8edac3c 100644 --- a/docs/modules/spark-k8s/examples/example-encapsulated.yaml +++ b/docs/modules/spark-k8s/examples/example-encapsulated.yaml @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: spark-pi spec: - version: "1.0" sparkImage: productVersion: 3.5.0 # <1> mode: cluster diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml index 95ed5350..cfd37074 100644 --- a/docs/modules/spark-k8s/examples/example-history-app.yaml +++ b/docs/modules/spark-k8s/examples/example-history-app.yaml @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: spark-pi-s3-1 spec: - version: "1.0" sparkImage: productVersion: 3.5.0 pullPolicy: IfNotPresent diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml index a5c90a8f..27173724 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml @@ -5,7 +5,6 @@ metadata: name: ny-tlc-report-configmap namespace: default spec: - version: "1.0" sparkImage: productVersion: 3.5.0 mode: cluster diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml index b1c798c5..c9a700ba 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-external-dependencies.yaml @@ -5,7 +5,6 @@ metadata: name: example-sparkapp-external-dependencies namespace: default spec: - version: "1.0" sparkImage: productVersion: 3.5.0 mode: cluster diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml index 13e7fe9c..15428794 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml @@ -5,7 +5,6 @@ metadata: name: example-sparkapp-image namespace: default spec: - version: "1.0" image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0 # <1> sparkImage: productVersion: 3.5.0 diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml index 0bce8d0f..322fd315 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml @@ -5,7 +5,6 @@ metadata: name: example-sparkapp-pvc namespace: default spec: - version: "1.0" sparkImage: productVersion: 3.5.0 mode: cluster diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml index b9b06157..0c0b6167 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: example-sparkapp-s3-private spec: - version: "1.0" sparkImage: productVersion: 3.5.0 mode: cluster diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml index 34213995..572bb7e5 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml @@ -5,7 +5,6 @@ metadata: name: pyspark-streaming namespace: default spec: - version: "1.0" sparkImage: productVersion: 3.5.0 mode: cluster diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh index db4b9127..e6b1ef29 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh @@ -59,7 +59,6 @@ metadata: name: pyspark-pi namespace: default spec: - version: "1.0" sparkImage: productVersion: 3.5.0 mode: cluster diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 index b2fbc2fc..a7e2a954 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 @@ -59,7 +59,6 @@ metadata: name: pyspark-pi namespace: default spec: - version: "1.0" sparkImage: productVersion: 3.5.0 mode: cluster diff --git a/examples/ny-tlc-report-external-dependencies.yaml b/examples/ny-tlc-report-external-dependencies.yaml index f5d3bc69..965e3122 100644 --- a/examples/ny-tlc-report-external-dependencies.yaml +++ b/examples/ny-tlc-report-external-dependencies.yaml @@ -5,7 +5,6 @@ metadata: name: spark-ny-ext namespace: default spec: - version: "1.0" sparkImage: productVersion: 3.5.0 pullPolicy: IfNotPresent diff --git a/examples/ny-tlc-report-image.yaml b/examples/ny-tlc-report-image.yaml index d2a8dfa6..5adb3328 100644 --- a/examples/ny-tlc-report-image.yaml +++ b/examples/ny-tlc-report-image.yaml @@ -5,7 +5,6 @@ metadata: name: spark-ny-image namespace: default spec: - version: "1.0" # everything under /jobs will be copied to /stackable/spark/jobs image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0 sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable0.0.0-dev diff --git a/examples/ny-tlc-report.yaml b/examples/ny-tlc-report.yaml index dfd14139..eda7a0cd 100644 --- a/examples/ny-tlc-report.yaml +++ b/examples/ny-tlc-report.yaml @@ -12,7 +12,6 @@ kind: SparkApplication metadata: name: spark-ny-cm spec: - version: "1.0" sparkImage: productVersion: 3.5.0 mode: cluster diff --git a/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 index 943e63e0..ede72f0d 100644 --- a/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/iceberg/10-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: pyspark-iceberg spec: - version: "1.0" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/logging/05-deploy-automatic-log-config-spark-app.yaml.j2 b/tests/templates/kuttl/logging/05-deploy-automatic-log-config-spark-app.yaml.j2 index 353f1ca6..1ea2e310 100644 --- a/tests/templates/kuttl/logging/05-deploy-automatic-log-config-spark-app.yaml.j2 +++ b/tests/templates/kuttl/logging/05-deploy-automatic-log-config-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: spark-automatic-log-config spec: - version: "1.0" sparkImage: {% if test_scenario['values']['spark'].find(",") > 0 %} custom: "{{ test_scenario['values']['spark'].split(',')[1] }}" diff --git a/tests/templates/kuttl/logging/06-deploy-custom-log-config-spark-app.yaml.j2 b/tests/templates/kuttl/logging/06-deploy-custom-log-config-spark-app.yaml.j2 index ac323f21..a5227b37 100644 --- a/tests/templates/kuttl/logging/06-deploy-custom-log-config-spark-app.yaml.j2 +++ b/tests/templates/kuttl/logging/06-deploy-custom-log-config-spark-app.yaml.j2 @@ -38,7 +38,6 @@ kind: SparkApplication metadata: name: spark-custom-log-config spec: - version: "1.0" sparkImage: {% if test_scenario['values']['spark'].find(",") > 0 %} custom: "{{ test_scenario['values']['spark'].split(',')[1] }}" diff --git a/tests/templates/kuttl/logging/07-deploy-automatic-log-config-pyspark-app.yaml.j2 b/tests/templates/kuttl/logging/07-deploy-automatic-log-config-pyspark-app.yaml.j2 index f87b1160..85d903cd 100644 --- a/tests/templates/kuttl/logging/07-deploy-automatic-log-config-pyspark-app.yaml.j2 +++ b/tests/templates/kuttl/logging/07-deploy-automatic-log-config-pyspark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: pyspark-automatic-log-config spec: - version: "1.0" sparkImage: {% if test_scenario['values']['spark'].find(",") > 0 %} custom: "{{ test_scenario['values']['spark'].split(',')[1] }}" diff --git a/tests/templates/kuttl/logging/08-deploy-custom-log-config-pyspark-app.yaml.j2 b/tests/templates/kuttl/logging/08-deploy-custom-log-config-pyspark-app.yaml.j2 index c3646e0a..f85328a1 100644 --- a/tests/templates/kuttl/logging/08-deploy-custom-log-config-pyspark-app.yaml.j2 +++ b/tests/templates/kuttl/logging/08-deploy-custom-log-config-pyspark-app.yaml.j2 @@ -38,7 +38,6 @@ kind: SparkApplication metadata: name: pyspark-custom-log-config spec: - version: "1.0" sparkImage: {% if test_scenario['values']['spark'].find(",") > 0 %} custom: "{{ test_scenario['values']['spark'].split(',')[1] }}" diff --git a/tests/templates/kuttl/pod_overrides/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/pod_overrides/10-deploy-spark-app.yaml.j2 index b606aa8a..cf66b309 100644 --- a/tests/templates/kuttl/pod_overrides/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/pod_overrides/10-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: spark-pi-s3-1 spec: - version: "1.0" sparkImage: {% if test_scenario['values']['spark'].find(",") > 0 %} custom: "{{ test_scenario['values']['spark'].split(',')[1] }}" diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 index 8c1d9a34..ce607ef2 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: pyspark-ny-public-s3-image spec: - version: "1.0" # everything under /jobs will be copied to /stackable/spark/jobs image: "docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}" {% if lookup('env', 'VECTOR_AGGREGATOR') %} diff --git a/tests/templates/kuttl/pyspark-ny-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3/10-deploy-spark-app.yaml.j2 index 7cc98360..26f9a795 100644 --- a/tests/templates/kuttl/pyspark-ny-public-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/pyspark-ny-public-s3/10-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: pyspark-ny-public-s3 spec: - version: "1.0" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 index dd2acd53..23f885d2 100644 --- a/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: resources-crd spec: - version: "1.0" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 index dc48fe9e..d04aec51 100644 --- a/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: resources-sparkconf spec: - version: "1.0" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/smoke/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/smoke/10-deploy-spark-app.yaml.j2 index bd26ac35..b806109c 100644 --- a/tests/templates/kuttl/smoke/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: spark-pi-s3-1 spec: - version: "1.0" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 index 2bab38db..0658c426 100644 --- a/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: spark-examples spec: - version: "1.0" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/spark-history-server/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-history-server/10-deploy-spark-app.yaml.j2 index 93a96d25..54508763 100644 --- a/tests/templates/kuttl/spark-history-server/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-history-server/10-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: spark-pi-s3-1 spec: - version: "1.0" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/spark-history-server/12-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-history-server/12-deploy-spark-app.yaml.j2 index 286e1a56..9bc1f4fc 100644 --- a/tests/templates/kuttl/spark-history-server/12-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-history-server/12-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: spark-pi-s3-2 spec: - version: "1.0" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 index 99510674..756e13fa 100644 --- a/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-ny-public-s3/10-deploy-spark-app.yaml.j2 @@ -12,7 +12,6 @@ kind: SparkApplication metadata: name: spark-ny-cm spec: - version: "1.0" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 index 4adda1e8..953f3945 100644 --- a/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-pi-private-s3/10-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: spark-pi-private-s3 spec: - version: "1.0" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 index aa38ed4a..a5dcf506 100644 --- a/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-pi-public-s3/10-deploy-spark-app.yaml.j2 @@ -4,7 +4,6 @@ kind: SparkApplication metadata: name: spark-pi-public-s3 spec: - version: "1.0" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} From 5f5164327e31f7f71058e14049cb449050964260 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 13 Dec 2023 17:50:19 +0100 Subject: [PATCH 15/15] added strum directive --- rust/crd/src/roles.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/crd/src/roles.rs b/rust/crd/src/roles.rs index cdd94508..9ac3241f 100644 --- a/rust/crd/src/roles.rs +++ b/rust/crd/src/roles.rs @@ -90,6 +90,7 @@ pub enum SparkContainer { } #[derive(Clone, Debug, Deserialize, Display, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] pub enum SparkMode { Cluster, Client,