diff --git a/CHANGELOG.md b/CHANGELOG.md index 614b3c4f..3561d175 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,12 @@ All notable changes to this project will be documented in this file. - Bumped image to `3.3.0-stackable0.2.0` in tests and docs ([#145]) - BREAKING: use resource limit struct instead of passing spark configuration arguments ([#147]) - Fixed resources test ([#151]) +- Fixed inconsistencies with resources usage ([#166]) [#145]: https://github.com/stackabletech/spark-k8s-operator/pull/145 [#147]: https://github.com/stackabletech/spark-k8s-operator/pull/147 [#151]: https://github.com/stackabletech/spark-k8s-operator/pull/151 +[#166]: https://github.com/stackabletech/spark-k8s-operator/pull/166 ## [0.5.0] - 2022-09-06 diff --git a/Cargo.lock b/Cargo.lock index 2d4971e1..25e5a749 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -548,6 +548,12 @@ version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + [[package]] name = "futures-util" version = "0.3.24" @@ -1437,6 +1443,40 @@ dependencies = [ "winapi", ] +[[package]] +name = "rstest" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9c9dc66cc29792b663ffb5269be669f1613664e69ad56441fdb895c2347b930" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5015e68a0685a95ade3eee617ff7101ab6a3fc689203101ca16ebc16f2b89c66" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustversion" version = "1.0.9" @@ -1710,6 +1750,7 @@ dependencies = [ name = "stackable-spark-k8s-crd" version = "0.6.0-nightly" dependencies = [ + "rstest", "semver", "serde", "serde_json", diff --git a/deploy/crd/sparkapplication.crd.yaml b/deploy/crd/sparkapplication.crd.yaml index 008eb14c..68565c64 100644 --- a/deploy/crd/sparkapplication.crd.yaml +++ b/deploy/crd/sparkapplication.crd.yaml @@ -116,50 +116,6 @@ spec: type: object type: object storage: - properties: - data: - default: - capacity: ~ - properties: - capacity: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n ::= \n (Note that may be empty, from the \"\" case in .)\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n ::= m | \"\" | k | M | G | T | P | E\n (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n ::= \"e\" | \"E\" \n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n a. No precision is lost\n b. No fractional digits will be emitted\n c. The exponent (or suffix) is as large as possible.\nThe sign will be omitted unless the number is negative.\n\nExamples:\n 1.5 will be serialized as \"1500m\"\n 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." - nullable: true - type: string - selectors: - description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values." - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist." - type: string - values: - description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch." - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed." - type: object - type: object - storageClass: - nullable: true - type: string - type: object type: object type: object volumeMounts: @@ -312,50 +268,6 @@ spec: type: object type: object storage: - properties: - data: - default: - capacity: ~ - properties: - capacity: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n ::= \n (Note that may be empty, from the \"\" case in .)\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n ::= m | \"\" | k | M | G | T | P | E\n (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n ::= \"e\" | \"E\" \n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n a. No precision is lost\n b. No fractional digits will be emitted\n c. The exponent (or suffix) is as large as possible.\nThe sign will be omitted unless the number is negative.\n\nExamples:\n 1.5 will be serialized as \"1500m\"\n 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." - nullable: true - type: string - selectors: - description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values." - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist." - type: string - values: - description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch." - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed." - type: object - type: object - storageClass: - nullable: true - type: string - type: object type: object type: object volumeMounts: @@ -420,50 +332,6 @@ spec: type: object type: object storage: - properties: - data: - default: - capacity: ~ - properties: - capacity: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n ::= \n (Note that may be empty, from the \"\" case in .)\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n ::= m | \"\" | k | M | G | T | P | E\n (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n ::= \"e\" | \"E\" \n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n a. No precision is lost\n b. No fractional digits will be emitted\n c. The exponent (or suffix) is as large as possible.\nThe sign will be omitted unless the number is negative.\n\nExamples:\n 1.5 will be serialized as \"1500m\"\n 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." - nullable: true - type: string - selectors: - description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values." - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist." - type: string - values: - description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch." - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed." - type: object - type: object - storageClass: - nullable: true - type: string - type: object type: object type: object type: object diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml index 0f723c65..745191c3 100644 --- a/deploy/helm/spark-k8s-operator/crds/crds.yaml +++ b/deploy/helm/spark-k8s-operator/crds/crds.yaml @@ -118,50 +118,6 @@ spec: type: object type: object storage: - properties: - data: - default: - capacity: ~ - properties: - capacity: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n ::= \n (Note that may be empty, from the \"\" case in .)\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n ::= m | \"\" | k | M | G | T | P | E\n (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n ::= \"e\" | \"E\" \n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n a. No precision is lost\n b. No fractional digits will be emitted\n c. The exponent (or suffix) is as large as possible.\nThe sign will be omitted unless the number is negative.\n\nExamples:\n 1.5 will be serialized as \"1500m\"\n 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." - nullable: true - type: string - selectors: - description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values." - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist." - type: string - values: - description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch." - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed." - type: object - type: object - storageClass: - nullable: true - type: string - type: object type: object type: object volumeMounts: @@ -314,50 +270,6 @@ spec: type: object type: object storage: - properties: - data: - default: - capacity: ~ - properties: - capacity: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n ::= \n (Note that may be empty, from the \"\" case in .)\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n ::= m | \"\" | k | M | G | T | P | E\n (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n ::= \"e\" | \"E\" \n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n a. No precision is lost\n b. No fractional digits will be emitted\n c. The exponent (or suffix) is as large as possible.\nThe sign will be omitted unless the number is negative.\n\nExamples:\n 1.5 will be serialized as \"1500m\"\n 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." - nullable: true - type: string - selectors: - description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values." - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist." - type: string - values: - description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch." - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed." - type: object - type: object - storageClass: - nullable: true - type: string - type: object type: object type: object volumeMounts: @@ -422,50 +334,6 @@ spec: type: object type: object storage: - properties: - data: - default: - capacity: ~ - properties: - capacity: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n ::= \n (Note that may be empty, from the \"\" case in .)\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n ::= m | \"\" | k | M | G | T | P | E\n (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n ::= \"e\" | \"E\" \n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n a. No precision is lost\n b. No fractional digits will be emitted\n c. The exponent (or suffix) is as large as possible.\nThe sign will be omitted unless the number is negative.\n\nExamples:\n 1.5 will be serialized as \"1500m\"\n 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." - nullable: true - type: string - selectors: - description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values." - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist." - type: string - values: - description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch." - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed." - type: object - type: object - storageClass: - nullable: true - type: string - type: object type: object type: object type: object diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 87b171a5..d8b84d3d 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -119,50 +119,6 @@ spec: type: object type: object storage: - properties: - data: - default: - capacity: ~ - properties: - capacity: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n ::= \n (Note that may be empty, from the \"\" case in .)\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n ::= m | \"\" | k | M | G | T | P | E\n (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n ::= \"e\" | \"E\" \n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n a. No precision is lost\n b. No fractional digits will be emitted\n c. The exponent (or suffix) is as large as possible.\nThe sign will be omitted unless the number is negative.\n\nExamples:\n 1.5 will be serialized as \"1500m\"\n 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." - nullable: true - type: string - selectors: - description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values." - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist." - type: string - values: - description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch." - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed." - type: object - type: object - storageClass: - nullable: true - type: string - type: object type: object type: object volumeMounts: @@ -315,50 +271,6 @@ spec: type: object type: object storage: - properties: - data: - default: - capacity: ~ - properties: - capacity: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n ::= \n (Note that may be empty, from the \"\" case in .)\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n ::= m | \"\" | k | M | G | T | P | E\n (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n ::= \"e\" | \"E\" \n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n a. No precision is lost\n b. No fractional digits will be emitted\n c. The exponent (or suffix) is as large as possible.\nThe sign will be omitted unless the number is negative.\n\nExamples:\n 1.5 will be serialized as \"1500m\"\n 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." - nullable: true - type: string - selectors: - description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values." - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist." - type: string - values: - description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch." - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed." - type: object - type: object - storageClass: - nullable: true - type: string - type: object type: object type: object volumeMounts: @@ -423,50 +335,6 @@ spec: type: object type: object storage: - properties: - data: - default: - capacity: ~ - properties: - capacity: - description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n ::= \n (Note that may be empty, from the \"\" case in .)\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n ::= m | \"\" | k | M | G | T | P | E\n (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n ::= \"e\" | \"E\" \n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n a. No precision is lost\n b. No fractional digits will be emitted\n c. The exponent (or suffix) is as large as possible.\nThe sign will be omitted unless the number is negative.\n\nExamples:\n 1.5 will be serialized as \"1500m\"\n 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." - nullable: true - type: string - selectors: - description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. - nullable: true - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values." - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist." - type: string - values: - description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch." - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed." - type: object - type: object - storageClass: - nullable: true - type: string - type: object type: object type: object type: object diff --git a/docs/modules/ROOT/examples/example-sparkapp-streaming.yaml b/docs/modules/ROOT/examples/example-sparkapp-streaming.yaml new file mode 100644 index 00000000..ca28770c --- /dev/null +++ b/docs/modules/ROOT/examples/example-sparkapp-streaming.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: pyspark-streaming + namespace: default +spec: + version: "1.0" + sparkImage: docker.stackable.tech/stackable/pyspark-k8s:3.3.0-stackable0.1.0 + mode: cluster + mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py + args: + - "/tmp2" + sparkConf: + spark.kubernetes.submission.waitAppCompletion: "false" + spark.kubernetes.driver.pod.name: "pyspark-streaming-driver" + spark.kubernetes.executor.podNamePrefix: "pyspark-streaming" + driver: + resources: + cpu: + min: "1" + max: "2" + memory: + limit: "1Gi" + executor: + instances: 1 + resources: + cpu: + min: "1700m" + max: "3" + memory: + limit: "2Gi" diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index f62ed545..4b477a84 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -180,6 +180,11 @@ executor: WARNING: The default values are _most likely_ not sufficient to run a proper cluster in production. Please adapt according to your requirements. For more details regarding Kubernetes CPU limits see: https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/[Assign CPU Resources to Containers and Pods]. +Spark allocates a default amount of non-heap memory based on the type of job (JVM or non-JVM). This is taken into account when defining memory settings based exclusively on the resource limits, so that the "declared" value is the actual total value (i.e. including memory overhead). This may result in minor deviations from the stated resource value due to rounding differences. + +NOTE: It is possible to define Spark resources either directly by setting configuration properties listed under `sparkConf`, or by using resource limits. If both are used, then `sparkConf` properties take precedence. It is recommended for the sake of clarity to use *_either_* one *_or_* the other. + + == CRD argument coverage Below are listed the CRD fields that can be defined by the user: diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index 8f5f0b4f..66b8a7e2 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -17,3 +17,6 @@ serde_json = "1.0" serde_yaml = "0.8" snafu = "0.7" strum = { version = "0.24", features = ["derive"] } + +[dev-dependencies] +rstest = "0.15.0" diff --git a/rust/crd/src/constants.rs b/rust/crd/src/constants.rs index a28462b0..2da409bf 100644 --- a/rust/crd/src/constants.rs +++ b/rust/crd/src/constants.rs @@ -22,3 +22,6 @@ pub const SECRET_ACCESS_KEY: &str = "secretAccessKey"; pub const S3_SECRET_DIR_NAME: &str = "/stackable/secrets"; pub const SPARK_UID: i64 = 1000; +pub const MIN_MEMORY_OVERHEAD: u32 = 384; +pub const JVM_OVERHEAD_FACTOR: f32 = 0.1; +pub const NON_JVM_OVERHEAD_FACTOR: f32 = 0.4; diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 7c324f29..79af03a1 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -10,14 +10,14 @@ use stackable_operator::commons::s3::{ use stackable_operator::k8s_openapi::api::core::v1::{ EmptyDirVolumeSource, EnvVar, LocalObjectReference, Volume, VolumeMount, }; +use stackable_operator::memory::{to_java_heap_value, BinaryMultiple}; +use std::cmp::max; use std::collections::{BTreeMap, HashMap}; use serde::{Deserialize, Serialize}; -use snafu::{OptionExt, Snafu}; -use stackable_operator::commons::resources::{ - CpuLimits, MemoryLimits, NoRuntimeLimits, PvcConfig, Resources, -}; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::commons::resources::{CpuLimits, MemoryLimits, NoRuntimeLimits, Resources}; use stackable_operator::kube::ResourceExt; use stackable_operator::labels; use stackable_operator::{ @@ -45,6 +45,15 @@ pub enum Error { ObjectHasNoName, #[snafu(display("application has no Spark image"))] NoSparkImage, + #[snafu(display("failed to convert java heap config to unit [{unit}]"))] + FailedToConvertJavaHeap { + source: stackable_operator::error::Error, + unit: String, + }, + #[snafu(display("failed to convert to quantity"))] + FailedQuantityConversion, + #[snafu(display("failed to parse value"))] + FailedParseToFloatConversion, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] @@ -53,12 +62,9 @@ pub struct SparkApplicationStatus { pub phase: String, } -#[derive(Clone, Debug, Default, Deserialize, Merge, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Merge, JsonSchema, Eq, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] -pub struct SparkStorageConfig { - #[serde(default)] - pub data: PvcConfig, -} +pub struct SparkStorageConfig {} #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] @@ -77,13 +83,7 @@ impl SparkConfig { limit: Some(Quantity("1Gi".to_owned())), runtime_limits: NoRuntimeLimits {}, }, - storage: SparkStorageConfig { - data: PvcConfig { - capacity: Some(Quantity("2Gi".to_owned())), - storage_class: None, - selectors: None, - }, - }, + storage: SparkStorageConfig {}, } } } @@ -382,13 +382,6 @@ impl SparkApplication { } } - // conf arguments that are not driver or executor specific - if let Some(spark_conf) = self.spec.spark_conf.clone() { - for (key, value) in spark_conf { - submit_cmd.push(format!("--conf {key}={value}")); - } - } - // repositories and packages arguments if let Some(deps) = self.spec.deps.clone() { submit_cmd.extend( @@ -398,6 +391,103 @@ impl SparkApplication { submit_cmd.extend(deps.packages.map(|p| format!("--packages {}", p.join(",")))); } + // some command elements need to be initially stored in a map (to allow overwrites) and + // then added to the vector once complete. + let mut submit_conf: BTreeMap = BTreeMap::new(); + + // resource limits, either declared or taken from defaults + if let Some(Resources { + cpu: CpuLimits { max: Some(max), .. }, + .. + }) = &self.driver_resources() + { + submit_conf.insert( + "spark.kubernetes.driver.limit.cores".to_string(), + max.0.clone(), + ); + let cores = + cores_from_quantity(max.0.clone()).map_err(|_| Error::FailedQuantityConversion)?; + // will have default value from resources to apply if nothing set specifically + submit_conf.insert("spark.driver.cores".to_string(), cores); + } + if let Some(Resources { + cpu: CpuLimits { min: Some(min), .. }, + .. + }) = &self.driver_resources() + { + submit_conf.insert( + "spark.kubernetes.driver.request.cores".to_string(), + min.0.clone(), + ); + } + if let Some(Resources { + memory: MemoryLimits { + limit: Some(limit), .. + }, + .. + }) = &self.driver_resources() + { + let memory = self + .subtract_spark_memory_overhead(limit) + .map_err(|_| Error::FailedQuantityConversion)?; + submit_conf.insert("spark.driver.memory".to_string(), memory); + } + + if let Some(Resources { + cpu: CpuLimits { max: Some(max), .. }, + .. + }) = &self.executor_resources() + { + submit_conf.insert( + "spark.kubernetes.executor.limit.cores".to_string(), + max.0.clone(), + ); + let cores = + cores_from_quantity(max.0.clone()).map_err(|_| Error::FailedQuantityConversion)?; + // will have default value from resources to apply if nothing set specifically + submit_conf.insert("spark.executor.cores".to_string(), cores); + } + if let Some(Resources { + cpu: CpuLimits { min: Some(min), .. }, + .. + }) = &self.executor_resources() + { + submit_conf.insert( + "spark.kubernetes.executor.request.cores".to_string(), + min.0.clone(), + ); + } + if let Some(Resources { + memory: MemoryLimits { + limit: Some(limit), .. + }, + .. + }) = &self.executor_resources() + { + let memory = self + .subtract_spark_memory_overhead(limit) + .map_err(|_| Error::FailedQuantityConversion)?; + submit_conf.insert("spark.executor.memory".to_string(), memory); + } + + if let Some(executors) = &self.spec.executor { + if let Some(instances) = executors.instances { + submit_conf.insert( + "spark.executor.instances".to_string(), + instances.to_string(), + ); + } + } + + // conf arguments: these should follow - and thus override - values set from resource limits above + if let Some(spark_conf) = self.spec.spark_conf.clone() { + submit_conf.extend(spark_conf); + } + // ...before being added to the command collection + for (key, value) in submit_conf { + submit_cmd.push(format!("--conf {key}={value}")); + } + submit_cmd.extend( self.spec .main_class @@ -417,6 +507,32 @@ impl SparkApplication { Ok(submit_cmd) } + /// A memory overhead will be applied using a factor of 0.1 (JVM jobs) or 0.4 (non-JVM jobs), + /// being not less than 384MB. The resource limit should keep this transparent by reducing the + /// declared memory limit accordingly. + fn subtract_spark_memory_overhead(&self, limit: &Quantity) -> Result { + // determine job-type using class name: scala/java will declare an application and main class; + // R and python will just declare the application name/file (for python this could be .zip/.py/.egg). + // Spark itself just checks the application name - See e.g. + // https://github.com/apache/spark/blob/01c7a46f24fb4bb4287a184a3d69e0e5c904bc50/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala#L1092 + let non_jvm_factor = if self.spec.main_class.is_some() { + 1.0 / (1.0 + JVM_OVERHEAD_FACTOR) + } else { + 1.0 / (1.0 + NON_JVM_OVERHEAD_FACTOR) + }; + let original_memory = to_java_heap_value(limit, 1.0, BinaryMultiple::Mebi).context( + FailedToConvertJavaHeapSnafu { + unit: BinaryMultiple::Mebi.to_java_memory_unit(), + }, + )?; + let reduced_memory = to_java_heap_value(limit, non_jvm_factor, BinaryMultiple::Mebi) + .context(FailedToConvertJavaHeapSnafu { + unit: BinaryMultiple::Mebi.to_java_memory_unit(), + })?; + let deduction = max(MIN_MEMORY_OVERHEAD, original_memory - reduced_memory); + Ok(format!("{}m", original_memory - deduction)) + } + pub fn env(&self) -> Vec { let tmp = self.spec.env.as_ref(); let mut e: Vec = tmp.iter().flat_map(|e| e.iter()).cloned().collect(); @@ -477,6 +593,28 @@ impl SparkApplication { } } +/// CPU Limits can be defined as integer, decimal, or unitised values (see +/// ) +/// of which only "m" (milli-units) is allowed. The parsed value will be rounded up to the next +/// integer value. +// TODO: Move to operator-rs when needed in multiple operators +fn cores_from_quantity(q: String) -> Result { + let start_of_unit = q.find('m'); + let cores = if let Some(start_of_unit) = start_of_unit { + let (prefix, _) = q.split_at(start_of_unit); + (prefix + .parse::() + .map_err(|_| Error::FailedParseToFloatConversion)? + / 1000.0) + .ceil() + } else { + q.parse::() + .map_err(|_| Error::FailedParseToFloatConversion)? + .ceil() + }; + Ok((cores as u32).to_string()) +} + #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct CommonConfig { @@ -508,22 +646,16 @@ impl DriverConfig { limit: Some(Quantity("2Gi".to_owned())), runtime_limits: NoRuntimeLimits {}, }, - storage: SparkStorageConfig { - data: PvcConfig { - capacity: Some(Quantity("2Gi".to_owned())), - storage_class: None, - selectors: None, - }, - }, + storage: SparkStorageConfig {}, } } pub fn spark_config(&self) -> Option> { - let conf = DriverConfig::default_resources(); + let default_resources = DriverConfig::default_resources(); let mut resources = self.resources.clone().unwrap_or_default(); - resources.merge(&conf); + resources.merge(&default_resources); Some(resources) } } @@ -551,31 +683,27 @@ impl ExecutorConfig { limit: Some(Quantity("4Gi".to_owned())), runtime_limits: NoRuntimeLimits {}, }, - storage: SparkStorageConfig { - data: PvcConfig { - capacity: Some(Quantity("2Gi".to_owned())), - storage_class: None, - selectors: None, - }, - }, + storage: SparkStorageConfig {}, } } pub fn spark_config(&self) -> Option> { - let conf = ExecutorConfig::default_resources(); + let default_resources = ExecutorConfig::default_resources(); let mut resources = self.resources.clone().unwrap_or_default(); - resources.merge(&conf); + resources.merge(&default_resources); Some(resources) } } #[cfg(test)] mod tests { - use crate::ImagePullPolicy; use crate::LocalObjectReference; + use crate::Quantity; use crate::SparkApplication; + use crate::{cores_from_quantity, ImagePullPolicy}; + use rstest::rstest; use stackable_operator::builder::ObjectMetaBuilder; use stackable_operator::commons::s3::{ S3AccessStyle, S3BucketSpec, S3ConnectionDef, S3ConnectionSpec, @@ -943,4 +1071,15 @@ spec: .0 ); } + + #[rstest] + #[case("1800m", "2")] + #[case("100m", "1")] + #[case("1.5", "2")] + #[case("2", "2")] + fn test_quantity_to_cores(#[case] input: &str, #[case] output: &str) { + let q = &Quantity(input.to_string()); + let cores = cores_from_quantity(q.0.clone()).unwrap(); + assert_eq!(output, cores); + } } diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index 7e181e5a..2cb51bd8 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -231,6 +231,8 @@ fn pod_template( cb.add_volume_mounts(volume_mounts.to_vec()) .add_env_vars(env.to_vec()); + // N.B. this may be ignored by spark as preference is given to spark + // configuration settings. let resources = match container_name { CONTAINER_NAME_DRIVER => spark_application .driver_resources() diff --git a/tests/templates/kuttl/resources/10-assert.yaml b/tests/templates/kuttl/resources/10-assert.yaml index e40a762e..b0333036 100644 --- a/tests/templates/kuttl/resources/10-assert.yaml +++ b/tests/templates/kuttl/resources/10-assert.yaml @@ -1,46 +1,53 @@ --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 900 ---- -apiVersion: spark.stackable.tech/v1alpha1 -kind: SparkApplication -metadata: - name: spark-resources -status: - phase: Succeeded +timeout: 240 --- apiVersion: v1 kind: Pod metadata: labels: - spark-role: driver + job-name: resources-crd app.kubernetes.io/managed-by: spark-k8s-operator +spec: + containers: + - name: spark-submit + resources: + limits: + cpu: 200m + memory: 1Gi + requests: + cpu: 100m + memory: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: resources-crd-driver spec: containers: - name: spark-driver resources: limits: - cpu: 1500m - # N.B. spark adds memoryOverhead of 10% by default - memory: 1408Mi + cpu: "2" + memory: "1Gi" requests: cpu: "1" - memory: 1408Mi + memory: "1Gi" --- apiVersion: v1 kind: Pod metadata: - labels: - job-name: spark-resources - app.kubernetes.io/managed-by: spark-k8s-operator + name: resources-crd-exec-1 spec: containers: - - name: spark-submit + - name: spark-executor resources: limits: - cpu: 200m - memory: 1Gi + cpu: "3" + # rounding error (vs. 2 Gi) + memory: 2046Mi requests: - cpu: 100m - memory: 1Gi + cpu: 1500m + # rounding error (vs. 2 Gi) + memory: 2046Mi diff --git a/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 index 3a48fef6..7c62a0af 100644 --- a/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 @@ -2,14 +2,18 @@ apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication metadata: - name: spark-resources + name: resources-crd spec: version: "1.0" - sparkImage: docker.stackable.tech/stackable/spark-k8s:{{ test_scenario['values']['spark'] }}-stackable{{ test_scenario['values']['stackable'] }} - sparkImagePullPolicy: IfNotPresent + sparkImage: docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'] }}-stackable{{ test_scenario['values']['stackable'] }} mode: cluster - mainClass: org.apache.spark.examples.SparkALS - mainApplicationFile: local:///stackable/spark/examples/jars/spark-examples_2.12-{{ test_scenario['values']['spark'] }}.jar + mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py + args: + - "/tmp2" + sparkConf: + spark.kubernetes.submission.waitAppCompletion: "false" + spark.kubernetes.driver.pod.name: "resources-crd-driver" + spark.kubernetes.executor.podNamePrefix: "resources-crd" job: resources: cpu: @@ -21,14 +25,14 @@ spec: resources: cpu: min: "1" - max: "1500m" + max: "2" memory: limit: "1Gi" executor: instances: 1 resources: cpu: - min: "1" - max: "2" + min: "1500m" + max: "3" memory: - limit: "1Gi" + limit: "2Gi" diff --git a/tests/templates/kuttl/resources/12-assert.yaml b/tests/templates/kuttl/resources/12-assert.yaml new file mode 100644 index 00000000..be10f9b8 --- /dev/null +++ b/tests/templates/kuttl/resources/12-assert.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 240 +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + job-name: resources-sparkconf + app.kubernetes.io/managed-by: spark-k8s-operator +spec: + containers: + - name: spark-submit + resources: + limits: + cpu: 100m + memory: 1Gi + requests: + cpu: 50m + memory: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: resources-sparkconf-driver +spec: + containers: + - name: spark-driver + resources: + limits: + cpu: "3" + memory: "1433Mi" + requests: + cpu: "2" + memory: "1433Mi" +--- +apiVersion: v1 +kind: Pod +metadata: + name: resources-sparkconf-exec-1 +spec: + containers: + - name: spark-executor + resources: + limits: + cpu: "3" + memory: "2867Mi" + requests: + cpu: "2" + memory: "2867Mi" diff --git a/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 new file mode 100644 index 00000000..4021f1d8 --- /dev/null +++ b/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 @@ -0,0 +1,28 @@ +--- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: resources-sparkconf +spec: + version: "1.0" + sparkImage: docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'] }}-stackable{{ test_scenario['values']['stackable'] }} + mode: cluster + mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py + args: + - "/tmp2" + sparkConf: + spark.kubernetes.submission.waitAppCompletion: "false" + spark.kubernetes.driver.pod.name: "resources-sparkconf-driver" + spark.kubernetes.executor.podNamePrefix: "resources-sparkconf" + spark.kubernetes.driver.request.cores: "2" + spark.kubernetes.driver.limit.cores: "3" + spark.driver.cores: "3" + spark.driver.memory: "1g" + spark.driver.memoryOverheadFactor: "0.4" + spark.kubernetes.executor.request.cores: "2" + spark.kubernetes.executor.limit.cores: "3" + spark.executor.cores: "3" + spark.executor.memory: "2g" + spark.executor.memoryOverheadFactor: "0.4" + spark.executor.instances: "1" + diff --git a/tests/templates/kuttl/resources/14-assert.yaml b/tests/templates/kuttl/resources/14-assert.yaml new file mode 100644 index 00000000..e121a4dc --- /dev/null +++ b/tests/templates/kuttl/resources/14-assert.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 240 +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + job-name: resources-defaults + app.kubernetes.io/managed-by: spark-k8s-operator +spec: + containers: + - name: spark-submit + resources: + limits: + cpu: 100m + memory: 1Gi + requests: + cpu: 50m + memory: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: resources-defaults-driver +spec: + containers: + - name: spark-driver + resources: + limits: + cpu: "2" + memory: "2046Mi" + requests: + cpu: "1" + memory: "2046Mi" +--- +apiVersion: v1 +kind: Pod +metadata: + name: resources-defaults-exec-1 +spec: + containers: + - name: spark-executor + resources: + limits: + cpu: "4" + memory: "4095Mi" + requests: + cpu: "1" + memory: "4095Mi" +--- +apiVersion: v1 +kind: Pod +metadata: + name: resources-defaults-exec-2 +spec: + containers: + - name: spark-executor + resources: + limits: + cpu: "4" + memory: "4095Mi" + requests: + cpu: "1" + memory: "4095Mi" diff --git a/tests/templates/kuttl/resources/14-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/resources/14-deploy-spark-app.yaml.j2 new file mode 100644 index 00000000..4ef2a823 --- /dev/null +++ b/tests/templates/kuttl/resources/14-deploy-spark-app.yaml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: resources-defaults +spec: + version: "1.0" + sparkImage: docker.stackable.tech/stackable/pyspark-k8s:{{ test_scenario['values']['spark'] }}-stackable{{ test_scenario['values']['stackable'] }} + mode: cluster + mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py + args: + - "/tmp2" + sparkConf: + spark.kubernetes.submission.waitAppCompletion: "false" + spark.kubernetes.driver.pod.name: "resources-defaults-driver" + spark.kubernetes.executor.podNamePrefix: "resources-defaults" +