diff --git a/CHANGELOG.md b/CHANGELOG.md index b1749441..e827a8c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to this project will be documented in this file. - Run a `containerdebug` process in the background of each Spark container to collect debugging information ([#508]). - Aggregate emitted Kubernetes events on the CustomResources ([#515]). - Support configuring JVM arguments ([#532]). +- Support for S3 region ([#528]). ### Changed @@ -20,6 +21,7 @@ All notable changes to this project will be documented in this file. [#508]: https://github.com/stackabletech/spark-k8s-operator/pull/508 [#514]: https://github.com/stackabletech/spark-k8s-operator/pull/514 [#515]: https://github.com/stackabletech/spark-k8s-operator/pull/515 +[#528]: https://github.com/stackabletech/spark-k8s-operator/pull/528 [#532]: https://github.com/stackabletech/spark-k8s-operator/pull/532 ## [24.11.1] - 2025-01-10 diff --git a/Cargo.lock b/Cargo.lock index dc7bddf4..34780b65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2539,8 +2539,8 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stackable-operator" -version = "0.85.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" +version = "0.87.2" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" dependencies = [ "chrono", "clap", @@ -2578,7 +2578,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" dependencies = [ "darling", "proc-macro2", @@ -2589,7 +2589,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#bc176bfc23f15533cdb3b7a7e7a773d4f29891e1" dependencies = [ "kube 0.98.0", "semver", @@ -2655,18 +2655,18 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.3" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.26.4" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ "heck", "proc-macro2", diff --git a/Cargo.nix b/Cargo.nix index 07445eb1..98648d7e 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1594,7 +1594,7 @@ rec { "default" = [ "Debug" "Clone" "Copy" "PartialEq" "Eq" "PartialOrd" "Ord" "Hash" "Default" "Deref" "DerefMut" "Into" ]; "full" = [ "syn/full" ]; }; - resolvedDefaultFeatures = [ "Clone" "Debug" "Default" "Hash" "PartialEq" ]; + resolvedDefaultFeatures = [ "Clone" "Debug" "Default" "Eq" "Hash" "PartialEq" ]; }; "either" = rec { crateName = "either"; @@ -6841,13 +6841,10 @@ rec { }; "ring" = rec { crateName = "ring"; - version = "0.17.8"; + version = "0.17.13"; edition = "2021"; - links = "ring_core_0_17_8"; - sha256 = "03fwlb1ssrmfxdckvqv033pfmk01rhx9ynwi7r186dcfcp5s8zy1"; - authors = [ - "Brian Smith " - ]; + links = "ring_core_0_17_13_"; + sha256 = "1vjhhlmpqqd9lc53ffjj1yk203188n2km27g3myvssm15a1mvb3h"; dependencies = [ { name = "cfg-if"; @@ -6862,14 +6859,13 @@ rec { name = "libc"; packageId = "libc"; usesDefaultFeatures = false; - target = { target, features }: ((("android" == target."os" or null) || ("linux" == target."os" or null)) && (("aarch64" == target."arch" or null) || ("arm" == target."arch" or null))); + target = { target, features }: ((("aarch64" == target."arch" or null) && ("little" == target."endian" or null)) && ("apple" == target."vendor" or null) && (("ios" == target."os" or null) || ("macos" == target."os" or null) || ("tvos" == target."os" or null) || ("visionos" == target."os" or null) || ("watchos" == target."os" or null))); } { - name = "spin"; - packageId = "spin"; + name = "libc"; + packageId = "libc"; usesDefaultFeatures = false; - target = { target, features }: (("aarch64" == target."arch" or null) || ("arm" == target."arch" or null) || ("x86" == target."arch" or null) || ("x86_64" == target."arch" or null)); - features = [ "once" ]; + target = { target, features }: (((("aarch64" == target."arch" or null) && ("little" == target."endian" or null)) || (("arm" == target."arch" or null) && ("little" == target."endian" or null))) && (("android" == target."os" or null) || ("linux" == target."os" or null))); } { name = "untrusted"; @@ -6878,7 +6874,7 @@ rec { { name = "windows-sys"; packageId = "windows-sys 0.52.0"; - target = { target, features }: (("aarch64" == target."arch" or null) && ("windows" == target."os" or null)); + target = { target, features }: ((("aarch64" == target."arch" or null) && ("little" == target."endian" or null)) && ("windows" == target."os" or null)); features = [ "Win32_Foundation" "Win32_System_Threading" ]; } ]; @@ -8061,31 +8057,6 @@ rec { }; resolvedDefaultFeatures = [ "all" ]; }; - "spin" = rec { - crateName = "spin"; - version = "0.9.8"; - edition = "2015"; - sha256 = "0rvam5r0p3a6qhc18scqpvpgb3ckzyqxpgdfyjnghh8ja7byi039"; - authors = [ - "Mathijs van de Nes " - "John Ericson " - "Joshua Barretto " - ]; - features = { - "barrier" = [ "mutex" ]; - "default" = [ "lock_api" "mutex" "spin_mutex" "rwlock" "once" "lazy" "barrier" ]; - "fair_mutex" = [ "mutex" ]; - "lazy" = [ "once" ]; - "lock_api" = [ "lock_api_crate" ]; - "lock_api_crate" = [ "dep:lock_api_crate" ]; - "portable-atomic" = [ "dep:portable-atomic" ]; - "portable_atomic" = [ "portable-atomic" ]; - "spin_mutex" = [ "mutex" ]; - "ticket_mutex" = [ "mutex" ]; - "use_ticket_mutex" = [ "mutex" "ticket_mutex" ]; - }; - resolvedDefaultFeatures = [ "once" ]; - }; "stable_deref_trait" = rec { crateName = "stable_deref_trait"; version = "1.2.0"; @@ -8102,13 +8073,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.85.0"; + version = "0.87.2"; edition = "2021"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "bc176bfc23f15533cdb3b7a7e7a773d4f29891e1"; + sha256 = "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a"; }; libName = "stackable_operator"; authors = [ @@ -8141,7 +8112,7 @@ rec { name = "educe"; packageId = "educe"; usesDefaultFeatures = false; - features = [ "Clone" "Debug" "Default" "PartialEq" ]; + features = [ "Clone" "Debug" "Default" "PartialEq" "Eq" ]; } { name = "either"; @@ -8267,8 +8238,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "bc176bfc23f15533cdb3b7a7e7a773d4f29891e1"; + sha256 = "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8302,8 +8273,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "59506c6202778889a27b6ae8153457e60a49c68d"; - sha256 = "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5"; + rev = "bc176bfc23f15533cdb3b7a7e7a773d4f29891e1"; + sha256 = "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a"; }; libName = "stackable_shared"; authors = [ @@ -8555,9 +8526,9 @@ rec { }; "strum" = rec { crateName = "strum"; - version = "0.26.3"; - edition = "2018"; - sha256 = "01lgl6jvrf4j28v5kmx9bp480ygf1nhvac8b4p7rcj9hxw50zv4g"; + version = "0.27.1"; + edition = "2021"; + sha256 = "0cic9r2sc2h17nnpjm2yfp7rsd35gkbcbqvrhl553jaiih4fykgn"; authors = [ "Peter Glotfelty " ]; @@ -8568,12 +8539,6 @@ rec { optional = true; } ]; - devDependencies = [ - { - name = "strum_macros"; - packageId = "strum_macros"; - } - ]; features = { "default" = [ "std" ]; "derive" = [ "strum_macros" ]; @@ -8584,9 +8549,9 @@ rec { }; "strum_macros" = rec { crateName = "strum_macros"; - version = "0.26.4"; - edition = "2018"; - sha256 = "1gl1wmq24b8md527cpyd5bw9rkbqldd7k1h38kf5ajd2ln2ywssc"; + version = "0.27.1"; + edition = "2021"; + sha256 = "1s7x07nkrgjfvxrvcdjw6qanad4c55yjnd32bph9q3xgpid8qyn7"; procMacro = true; authors = [ "Peter Glotfelty " @@ -8611,7 +8576,7 @@ rec { { name = "syn"; packageId = "syn 2.0.96"; - features = [ "parsing" "extra-traits" ]; + features = [ "parsing" ]; } ]; diff --git a/Cargo.toml b/Cargo.toml index 00d1ed01..7661e019 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/spark-k8s-operator" [workspace.dependencies] stackable-versioned = { git = "https://github.com/stackabletech/operator-rs.git", features = ["k8s"], tag = "stackable-versioned-0.5.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.87.2" } product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } anyhow = "1.0" @@ -25,7 +25,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -strum = { version = "0.26", features = ["derive"] } +strum = { version = "0.27", features = ["derive"] } tokio = { version = "1.39", features = ["full"] } tracing = "0.1" tracing-futures = { version = "0.2", features = ["futures-03"] } diff --git a/crate-hashes.json b/crate-hashes.json index c7d32c3a..e02a942d 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,7 +1,7 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator-derive@0.3.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator@0.85.0": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-shared@0.0.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#stackable-operator-derive@0.3.1": "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#stackable-operator@0.87.2": "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.2#stackable-shared@0.0.1": "0cqz1xmj3vbm5hm9x6wbgg2l265s30j5j5609wmg68p6giywh82a", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#k8s-version@0.1.2": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#stackable-versioned-macros@0.5.0": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#stackable-versioned@0.5.0": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml index f6672b00..cbdcb3f7 100644 --- a/deploy/helm/spark-k8s-operator/crds/crds.yaml +++ b/deploy/helm/spark-k8s-operator/crds/crds.yaml @@ -794,6 +794,20 @@ spec: minimum: 0.0 nullable: true type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object tls: description: Use a TLS connection. If not specified no TLS will be used. nullable: true @@ -930,6 +944,20 @@ spec: minimum: 0.0 nullable: true type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object tls: description: Use a TLS connection. If not specified no TLS will be used. nullable: true @@ -1249,6 +1277,20 @@ spec: minimum: 0.0 nullable: true type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object tls: description: Use a TLS connection. If not specified no TLS will be used. nullable: true diff --git a/rust/operator-binary/src/crd/history.rs b/rust/operator-binary/src/crd/history.rs index a2fd0dde..d9a4e857 100644 --- a/rust/operator-binary/src/crd/history.rs +++ b/rust/operator-binary/src/crd/history.rs @@ -484,6 +484,7 @@ mod test { access_style: Default::default(), credentials: None, tls: TlsClientDetails { tls: None }, + region: Default::default(), }, }, prefix: "prefix".to_string(), diff --git a/rust/operator-binary/src/crd/logdir.rs b/rust/operator-binary/src/crd/logdir.rs index bf8e2c81..c9ef10ab 100644 --- a/rust/operator-binary/src/crd/logdir.rs +++ b/rust/operator-binary/src/crd/logdir.rs @@ -182,7 +182,7 @@ impl S3LogDir { pub fn history_server_spark_config(&self) -> Result, Error> { let connection = &self.bucket.connection; - Ok(BTreeMap::from([ + let config = BTreeMap::from([ ("spark.history.fs.logDirectory".to_string(), self.url()), ( "spark.hadoop.fs.s3a.endpoint".to_string(), @@ -192,7 +192,13 @@ impl S3LogDir { "spark.hadoop.fs.s3a.path.style.access".to_string(), (connection.access_style == S3AccessStyle::Path).to_string(), ), - ])) + ( + "spark.hadoop.fs.s3a.endpoint.region".to_string(), + connection.region.name.clone(), + ), + ]); + + Ok(config) } pub fn application_spark_config(&self) -> Result, Error> { @@ -211,6 +217,10 @@ impl S3LogDir { format!("spark.hadoop.fs.s3a.bucket.{bucket_name}.path.style.access"), (connection.access_style == S3AccessStyle::Path).to_string(), ); + result.insert( + format!("spark.hadoop.fs.s3a.bucket.{bucket_name}.region"), + connection.region.name.clone(), + ); if let Some(secret_dir) = self.credentials_mount_path() { // We don't use the credentials at all here but assume they are available result.insert( diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 1ba79aa5..2bc5cb2a 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -572,6 +572,10 @@ impl v1alpha1::SparkApplication { "--conf spark.hadoop.fs.s3a.path.style.access={}", s3conn.access_style == S3AccessStyle::Path )); + submit_cmd.push(format!( + "--conf spark.hadoop.fs.s3a.endpoint.region=\"{region_name}\"", + region_name = s3conn.region.name + )); if let Some(credentials) = &s3conn.credentials { let secret_class_name = credentials.secret_class.clone(); let secret_dir = format!("{S3_SECRET_DIR_NAME}/{secret_class_name}");