diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..63f72bfc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +--- +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: detect-aws-credentials + args: ["--allow-missing-credentials"] + - id: detect-private-key + - repo: https://github.com/doublify/pre-commit-rust + rev: v1.0 + hooks: + - id: fmt + - id: clippy + args: ["--all-targets"] + - repo: https://github.com/adrienverge/yamllint + rev: v1.26.3 + hooks: + - id: yamllint diff --git a/CHANGELOG.md b/CHANGELOG.md index 296414ef..408d3554 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,9 @@ All notable changes to this project will be documented in this file. - ServiceAccount, ClusterRole and RoleBinding for Spark driver ([#39]) - S3 credentials can be provided via a Secret ([#42]) - Job information can be passed via a configuration map ([#50]) +- Update S3 bucket specification to be conform with the corresponding ADR ([#55]) [#39]: https://github.com/stackabletech/spark-k8s-operator/pull/39 [#42]: https://github.com/stackabletech/spark-k8s-operator/pull/42 [#50]: https://github.com/stackabletech/spark-k8s-operator/pull/50 +[#55]: https://github.com/stackabletech/spark-k8s-operator/pull/55 diff --git a/Cargo.lock b/Cargo.lock index b5f73157..3bbd203e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,9 +33,20 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" + +[[package]] +name = "async-trait" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "atty" @@ -103,6 +114,18 @@ dependencies = [ "git2", ] +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "bytes" version = "1.1.0" @@ -111,9 +134,9 @@ checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" [[package]] name = "cargo-lock" -version = "7.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb04b88bd5b2036e30704f95c6ee16f3b5ca3b4ca307da2889d9006648e5c88" +checksum = "6c408da54db4c50d4693f7e649c299bc9de9c23ead86249e5368830bb32a734b" dependencies = [ "semver", "serde", @@ -152,16 +175,16 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.6" +version = "3.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8c93436c21e4698bacadf42917db28b23017027a4deccb35dbe47a7e7840123" +checksum = "85a35a599b11c089a7f49105658d089b8f2cf0882993c17daf6de15285c2c35d" dependencies = [ "atty", "bitflags", "clap_derive", + "clap_lex", "indexmap", "lazy_static", - "os_str_bytes", "strsim", "termcolor", "textwrap", @@ -169,22 +192,31 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.1.4" +version = "3.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da95d038ede1a964ce99f49cbe27a7fb538d1da595e4b4f70b8c8f338d17bf16" +checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro-error", "proc-macro2", "quote", "syn", ] +[[package]] +name = "clap_lex" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "const_format" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22bc6cd49b0ec407b680c3e380182b6ac63b73991cb7602de350352fc309b614" +checksum = "0936ffe6d0c8d6a51b3b0a73b2acbe925d786f346cf45bfddc8341d79fb7dc8a" dependencies = [ "const_format_proc_macros", ] @@ -216,11 +248,31 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if", + "lazy_static", +] + [[package]] name = "darling" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ "darling_core", "darling_macro", @@ -228,9 +280,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", @@ -242,9 +294,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core", "quote", @@ -513,9 +565,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if", "libc", @@ -541,15 +593,6 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.4.0" @@ -567,9 +610,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" dependencies = [ "bytes", "fnv", @@ -595,9 +638,9 @@ checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" [[package]] name = "httparse" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" [[package]] name = "httpdate" @@ -628,6 +671,24 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-openssl" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ee5d7a8f718585d1c3c61dfde28ef5b0bb14734b4db13f5ada856cdc6c612b" +dependencies = [ + "http", + "hyper", + "linked_hash_set", + "once_cell", + "openssl", + "openssl-sys", + "parking_lot", + "tokio", + "tokio-openssl", + "tower-layer", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -672,9 +733,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg", "hashbrown", @@ -689,6 +750,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "integer-encoding" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e85a1509a128c855368e135cffcde7eac17d8e1083f41e2b98c58bc1a5074be" + [[package]] name = "itoa" version = "1.0.1" @@ -715,6 +782,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "json-patch" version = "0.2.6" @@ -754,9 +830,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcc72fdf0c491160a34d4a1bfb03f96da8a5054288d61c816d514b5c2fa49ea" +checksum = "342744dfeb81fe186b84f485b33f12c6a15d3396987d933b06a566a3db52ca38" dependencies = [ "k8s-openapi", "kube-client", @@ -767,9 +843,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b01c722d55ffedec74cbc259b4508d8a59bf19540006ec87618f76ab156579" +checksum = "3f69a504997799340408635d6e351afb8aab2c34ca3165e162f41b3b34a69a79" dependencies = [ "base64", "bytes", @@ -780,6 +856,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-openssl", "hyper-timeout", "hyper-tls", "jsonpath_lib", @@ -803,9 +880,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd9e3535777edd122cc26fe3fe6357066b33eff63d8b919862edbe7a956a679" +checksum = "a4a247487699941baaf93438d65b12d4e32450bea849d619d19ed394e8a4a645" dependencies = [ "chrono", "form_urlencoded", @@ -821,9 +898,9 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1322e25c20dd6f18ca6baecc88bb130331e99d988df9d7a9a207f15819e05bff" +checksum = "203f7c5acf9d0dfb0b08d44ec1d66ace3d1dfe0cdd82e65e274f3f96615d666c" dependencies = [ "darling", "proc-macro2", @@ -834,9 +911,9 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816c8c086f8bbcf9a4db0b7a68db90b784ef6292a57de35c64cccb90d5edfbe5" +checksum = "02ea50e6ed56578e1d1d02548901b12fe6d3edbf110269a396955e285d487973" dependencies = [ "ahash", "backoff", @@ -864,9 +941,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.121" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" +checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" [[package]] name = "libgit2-sys" @@ -882,9 +959,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" +checksum = "92e7e15d7610cce1d9752e137625f14e61a28cd45929b6e12e47b50fe154ee2e" dependencies = [ "cc", "libc", @@ -898,20 +975,30 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +[[package]] +name = "linked_hash_set" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47186c6da4d81ca383c7c47c1bfc80f4b95f4720514d860a5407aaf4233f9588" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lock_api" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ + "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", ] @@ -933,9 +1020,9 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "mio" @@ -962,9 +1049,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" dependencies = [ "lazy_static", "libc", @@ -989,9 +1076,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", @@ -999,9 +1086,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] @@ -1024,18 +1111,30 @@ checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" [[package]] name = "openssl" -version = "0.10.38" +version = "0.10.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" dependencies = [ "bitflags", "cfg-if", "foreign-types", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -1044,9 +1143,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.72" +version = "0.9.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0" dependencies = [ "autocfg", "cc", @@ -1055,6 +1154,60 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "js-sys", + "lazy_static", + "percent-encoding", + "pin-project", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c0b12cd9e3f9b35b52f6e0dac66866c519b26f424f4bbf96e3fe8bfbdc5229" +dependencies = [ + "async-trait", + "lazy_static", + "opentelemetry", + "opentelemetry-semantic-conventions", + "thiserror", + "thrift", + "tokio", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985cc35d832d412224b2cffe2f9194b1b89b6aa5d0bef76d080dce09d90e62bd" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "ordered-float" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "2.10.0" @@ -1069,9 +1222,6 @@ name = "os_str_bytes" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" -dependencies = [ - "memchr", -] [[package]] name = "parking_lot" @@ -1085,9 +1235,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if", "libc", @@ -1133,9 +1283,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -1145,9 +1295,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "ppv-lite86" @@ -1181,17 +1331,17 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" dependencies = [ "unicode-xid", ] [[package]] name = "product-config" -version = "0.3.1" -source = "git+https://github.com/stackabletech/product-config.git?tag=0.3.1#40c93e5283beef100c9fecdb6368f1e1480db3e8" +version = "0.4.0" +source = "git+https://github.com/stackabletech/product-config.git?tag=0.4.0#e1e5938b4f6120f85a088194e86d22433fdba731" dependencies = [ "fancy-regex", "java-properties", @@ -1206,9 +1356,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.16" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] @@ -1245,18 +1395,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.11" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", "redox_syscall", @@ -1385,18 +1535,18 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" dependencies = [ "serde", ] [[package]] name = "serde" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] @@ -1407,15 +1557,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" dependencies = [ - "ordered-float", + "ordered-float 2.10.0", "serde", ] [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -1435,9 +1585,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "indexmap", "itoa", @@ -1447,9 +1597,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a521f2940385c165a24ee286aa8599633d162077a54bdcae2a6fd5a7bfa7a0" +checksum = "707d15895415db6628332b737c838b88c598522e4dc70647e59b72312924aebc" dependencies = [ "indexmap", "ryu", @@ -1477,9 +1627,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "smallvec" @@ -1489,9 +1639,9 @@ checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "snafu" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eba135d2c579aa65364522eb78590cdf703176ef71ad4c32b00f58f7afb2df5" +checksum = "5177903bf45656592d9eb5c0e22f408fc023aae51dbe2088889b71633ba451f2" dependencies = [ "doc-comment", "snafu-derive", @@ -1499,11 +1649,11 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a7fe9b0669ef117c5cabc5549638528f36771f058ff977d7689deb517833a75" +checksum = "410b26ed97440d90ced3e2488c868d56a86e2064f5d7d6f417909b286afe25e5" dependencies = [ - "heck 0.3.3", + "heck", "proc-macro2", "quote", "syn", @@ -1521,8 +1671,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.15.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.15.0#c7c408d476c0b7ba06833c19e5c9d2aefa5875ae" +version = "0.19.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.19.0#f8f2d5527b3463cc40f3d851172af7ae81db75c1" dependencies = [ "backoff", "chrono", @@ -1535,6 +1685,8 @@ dependencies = [ "k8s-openapi", "kube", "lazy_static", + "opentelemetry", + "opentelemetry-jaeger", "product-config", "rand", "regex", @@ -1542,13 +1694,26 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "stackable-operator-derive", "strum", "thiserror", "tokio", "tracing", + "tracing-opentelemetry", "tracing-subscriber", ] +[[package]] +name = "stackable-operator-derive" +version = "0.17.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.19.0#f8f2d5527b3463cc40f3d851172af7ae81db75c1" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "stackable-spark-k8s-crd" version = "0.1.0-nightly" @@ -1601,7 +1766,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro2", "quote", "rustversion", @@ -1610,9 +1775,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" +checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52" dependencies = [ "proc-macro2", "quote", @@ -1650,18 +1815,18 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -1677,6 +1842,28 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "thrift" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b82ca8f46f95b3ce96081fe3dd89160fdea970c254bb72925255d1b62aae692e" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float 1.1.1", + "threadpool", +] + [[package]] name = "time" version = "0.1.43" @@ -1689,9 +1876,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -1704,9 +1891,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.17.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "dce653fb475565de9f6fb0614b28bca8df2c430c0cf84bcd9c843f15de5414cc" dependencies = [ "libc", "mio", @@ -1750,26 +1937,49 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-openssl" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" +dependencies = [ + "futures-util", + "openssl", + "openssl-sys", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64910e1b9c1901aaf5375561e35b9c057d95ff41a44ede043a03e09279eabaf1" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" dependencies = [ "bytes", "futures-core", "futures-sink", - "log", "pin-project-lite", "slab", "tokio", + "tracing", ] [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] @@ -1825,9 +2035,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.32" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" dependencies = [ "cfg-if", "log", @@ -1838,9 +2048,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" dependencies = [ "proc-macro2", "quote", @@ -1849,9 +2059,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" dependencies = [ "lazy_static", "valuable", @@ -1859,20 +2069,33 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" dependencies = [ "lazy_static", "log", "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f9378e96a9361190ae297e7f3a8ff644aacd2897f244b1ff81f381669196fa6" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-subscriber" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0ab7bdc962035a87fba73f3acca9b8a8d0034c2e6f60b84aeaaddddc155dce" +checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" dependencies = [ "ansi_term", "lazy_static", @@ -1903,9 +2126,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "unicode-bidi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-normalization" @@ -1916,17 +2139,11 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" - [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "url" @@ -1980,6 +2197,60 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + [[package]] name = "winapi" version = "0.3.9" @@ -2013,9 +2284,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -2026,33 +2297,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "xml-rs" @@ -2071,6 +2342,6 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb5728b8afd3f280a869ce1d4c554ffaed35f45c231fc41bfbd0381bef50317" +checksum = "94693807d016b2f2d2e14420eb3bfcca689311ff775dcf113d74ea624b7cdf07" diff --git a/Cargo.toml b/Cargo.toml index 2182c5f7..9915447f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,4 +2,4 @@ members = [ "rust/crd", "rust/operator-binary" -] \ No newline at end of file +] diff --git a/deploy/crd/sparkapplication.crd.yaml b/deploy/crd/sparkapplication.crd.yaml index b79fe882..4fbb0187 100644 --- a/deploy/crd/sparkapplication.crd.yaml +++ b/deploy/crd/sparkapplication.crd.yaml @@ -263,16 +263,90 @@ spec: mode: nullable: true type: string - s3: + s3bucket: + description: Operators are expected to define fields for this type in order to work with S3 buckets. nullable: true + oneOf: + - required: + - inline + - required: + - reference properties: - credentialsSecret: - type: string - endpoint: - nullable: true + inline: + description: S3 bucket specification containing only the bucket name and an inlined or referenced connection specification. + properties: + bucketName: + nullable: true + type: string + connection: + description: Operators are expected to define fields for this type in order to work with S3 connections. + nullable: true + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: S3 connection definition as CRD. + properties: + host: + nullable: true + type: string + port: + format: uint16 + minimum: 0.0 + nullable: true + type: integer + secretClass: + nullable: true + type: string + tls: + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: "Use TLS but don't verify certificates" + type: object + server: + description: Use TLS and ca certificate to verify the server + properties: + caCert: + description: Ca cert to verify the server + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: "Name of the SecretClass which will provide the ca cert. Note that a SecretClass does not need to have a key but can also work with just a ca cert. So if you got provided with a ca cert but don't have access to the key you can still use this method." + type: string + webPki: + description: Use TLS and the ca certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + type: object + reference: + type: string + type: object + type: object + reference: type: string - required: - - credentialsSecret type: object sparkConf: additionalProperties: diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml index ef22569f..28d4ce91 100644 --- a/deploy/helm/spark-k8s-operator/crds/crds.yaml +++ b/deploy/helm/spark-k8s-operator/crds/crds.yaml @@ -265,16 +265,90 @@ spec: mode: nullable: true type: string - s3: + s3bucket: + description: Operators are expected to define fields for this type in order to work with S3 buckets. nullable: true + oneOf: + - required: + - inline + - required: + - reference properties: - credentialsSecret: - type: string - endpoint: - nullable: true + inline: + description: S3 bucket specification containing only the bucket name and an inlined or referenced connection specification. + properties: + bucketName: + nullable: true + type: string + connection: + description: Operators are expected to define fields for this type in order to work with S3 connections. + nullable: true + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: S3 connection definition as CRD. + properties: + host: + nullable: true + type: string + port: + format: uint16 + minimum: 0.0 + nullable: true + type: integer + secretClass: + nullable: true + type: string + tls: + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: "Use TLS but don't verify certificates" + type: object + server: + description: Use TLS and ca certificate to verify the server + properties: + caCert: + description: Ca cert to verify the server + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: "Name of the SecretClass which will provide the ca cert. Note that a SecretClass does not need to have a key but can also work with just a ca cert. So if you got provided with a ca cert but don't have access to the key you can still use this method." + type: string + webPki: + description: Use TLS and the ca certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + type: object + reference: + type: string + type: object + type: object + reference: type: string - required: - - credentialsSecret type: object sparkConf: additionalProperties: diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 615c40d6..53fd9d65 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -266,16 +266,90 @@ spec: mode: nullable: true type: string - s3: + s3bucket: + description: Operators are expected to define fields for this type in order to work with S3 buckets. nullable: true + oneOf: + - required: + - inline + - required: + - reference properties: - credentialsSecret: - type: string - endpoint: - nullable: true + inline: + description: S3 bucket specification containing only the bucket name and an inlined or referenced connection specification. + properties: + bucketName: + nullable: true + type: string + connection: + description: Operators are expected to define fields for this type in order to work with S3 connections. + nullable: true + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: S3 connection definition as CRD. + properties: + host: + nullable: true + type: string + port: + format: uint16 + minimum: 0.0 + nullable: true + type: integer + secretClass: + nullable: true + type: string + tls: + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: "Use TLS but don't verify certificates" + type: object + server: + description: Use TLS and ca certificate to verify the server + properties: + caCert: + description: Ca cert to verify the server + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: "Name of the SecretClass which will provide the ca cert. Note that a SecretClass does not need to have a key but can also work with just a ca cert. So if you got provided with a ca cert but don't have access to the key you can still use this method." + type: string + webPki: + description: Use TLS and the ca certificates trusted by the common web browsers to verify the server. This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + type: object + reference: + type: string + type: object + type: object + reference: type: string - required: - - credentialsSecret type: object sparkConf: additionalProperties: diff --git a/docs/modules/ROOT/examples/example-sparkapp-s3-private.yaml b/docs/modules/ROOT/examples/example-sparkapp-s3-private.yaml index 662ae74e..761b5de0 100644 --- a/docs/modules/ROOT/examples/example-sparkapp-s3-private.yaml +++ b/docs/modules/ROOT/examples/example-sparkapp-s3-private.yaml @@ -9,9 +9,14 @@ spec: mode: cluster mainApplicationFile: s3a://my-bucket/spark-examples_2.12-3.2.1.jar # <1> mainClass: org.apache.spark.examples.SparkPi # <2> - s3: # <3> - credentialsSecret: minio-credentials # <4> - endpoint: http://test-minio:9000/ + s3bucket: # <3> + inline: + bucketName: my-bucket + connection: + inline: + host: test-minio + port: 9000 + secretClass: minio-credentials # <4> sparkConf: # <5> spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider" # <6> spark.hadoop.fs.s3a.path.style.access: "true" diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 62991404..1d26df25 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -119,6 +119,58 @@ include::example$example-sparkapp-configmap.yaml[] <6> The name of the volume backed by the configuration map that will be mounted to the driver/executor <7> The mount location of the volume (this will contain a file `/arguments/job-args.txt`) +== S3 bucket specification + +You can specify S3 connection details directly inside the `SparkApplication` specification or by refering to an external `S3Bucket` custom resource. + +To specify S3 connection details directly as part of the `SparkApplication` resource you add an inline bucket configuration as shown below. + +[source,yaml] +---- +s3bucket: # <1> + inline: + bucketName: my-bucket # <2> + connection: + inline: + host: test-minio # <3> + port: 9000 # <4> + secretClass: minio-credentials # <5> +---- +<1> Entry point for the bucket configuration. +<2> Bucket name. +<3> Bucket host. +<4> Optional bucket port. +<5> Name of the `Secret` object expected to contain the following keys: `ACCESS_KEY_ID` and `SECRET_ACCESS_KEY` + +It is also possible to configure the bucket connection details as a separate Kubernetes resource and only refer to that object from the `SparkApplication` like this: + +[source,yaml] +---- +s3bucket: + reference: my-bucket-resource # <1> +---- +<1> Name of the bucket resource with connection details. + +The resource named `my-bucket-resource` is then defined as shown below: + +[source,yaml] +---- +--- +apiVersion: s3.stackable.tech/v1alpha1 +kind: S3Bucket +metadata: + name: my-bucket-resource +spec: + bucketName: my-bucket-name + connection: + inline: + host: test-minio + port: 9000 + secretClass: minio-credentials +---- + +This has the advantage that bucket configuration can be shared across `SparkApplication`s and reduces the cost of updating these details. + == CRD argument coverage Below are listed the CRD fields that can be defined by the user: @@ -156,11 +208,8 @@ Below are listed the CRD fields that can be defined by the user: |`spec.args` |Arguments passed directly to the job artifact -|`spec.s3.credentialsSecret` -|Name of the credentials secret for S3 access - -|`spec.s3.endpoint` -|S3 endpoint +|`spec.s3bucket` +|S3 bucket and connection specification. See the <> for more details. |`spec.sparkConf` |A map of key/value strings that will be passed directly to `spark-submit` diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index debabc8d..69a2aa1e 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/stackabletech/spark-k8s-operator" version = "0.1.0-nightly" [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag="0.19.0" } semver = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 96f9710d..bcc0183c 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -3,6 +3,7 @@ pub mod constants; use constants::*; +use stackable_operator::commons::s3::{InlinedS3BucketSpec, S3BucketDef}; use stackable_operator::k8s_openapi::api::core::v1::{ EnvVar, EnvVarSource, SecretKeySelector, Volume, VolumeMount, }; @@ -79,7 +80,7 @@ pub struct SparkApplicationSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub deps: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub s3: Option, + pub s3bucket: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub args: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -101,14 +102,6 @@ pub struct JobDependencies { pub exclude_packages: Option>, } -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct S3 { - pub credentials_secret: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub endpoint: Option, -} - impl SparkApplication { pub fn enable_monitoring(&self) -> Option { let spec: &SparkApplicationSpec = &self.spec; @@ -146,24 +139,6 @@ impl SparkApplication { .map(|req| req.join(" ")) } - pub fn env(&self) -> Vec { - let tmp = self.spec.env.as_ref(); - let mut e: Vec = tmp.iter().flat_map(|e| e.iter()).cloned().collect(); - if let Some(s3) = self.spec.s3.as_ref() { - e.push(env_var_from_secret( - ENV_AWS_ACCESS_KEY_ID, - &s3.credentials_secret, - ACCESS_KEY_ID, - )); - e.push(env_var_from_secret( - ENV_AWS_SECRET_ACCESS_KEY, - &s3.credentials_secret, - SECRET_ACCESS_KEY, - )); - } - e - } - pub fn volumes(&self) -> Vec { let tmp = self.spec.volumes.as_ref(); tmp.iter().flat_map(|v| v.iter()).cloned().collect() @@ -201,7 +176,11 @@ impl SparkApplication { ls } - pub fn build_command(&self, serviceaccount_name: &str) -> Result, Error> { + pub fn build_command( + &self, + serviceaccount_name: &str, + s3bucket: &Option, + ) -> Result, Error> { // mandatory properties let mode = self.mode().context(ObjectHasNoDeployModeSnafu)?; let name = self.metadata.name.clone().context(ObjectHasNoNameSnafu)?; @@ -222,9 +201,23 @@ impl SparkApplication { format!("--conf spark.kubernetes.authenticate.driver.serviceAccountName={}", serviceaccount_name), ]; - if let Some(endpoint) = self.spec.s3.as_ref().and_then(|s3| s3.endpoint.as_ref()) { + // See https://spark.apache.org/docs/latest/running-on-kubernetes.html#dependency-management + // for possible S3 related properties + if let Some(endpoint) = s3bucket.as_ref().and_then(|s3| s3.endpoint()) { submit_cmd.push(format!("--conf spark.hadoop.fs.s3a.endpoint={}", endpoint)); } + if s3bucket.as_ref().and_then(|s3| s3.secret_class()).is_some() { + // We don't use the secret at all here, instead we assume the Self::env() has been + // called and this environment variables are availables. + submit_cmd.push(format!( + "--conf spark.hadoop.fs.s3a.access.key=${}", + ENV_AWS_ACCESS_KEY_ID + )); + submit_cmd.push(format!( + "--conf spark.hadoop.fs.s3a.secret.key=${}", + ENV_AWS_SECRET_ACCESS_KEY + )); + } // conf arguments that are not driver or executor specific if let Some(spark_conf) = self.spec.spark_conf.clone() { @@ -268,6 +261,41 @@ impl SparkApplication { Ok(submit_cmd) } + + pub fn env(&self, s3bucket: &Option) -> Vec { + let tmp = self.spec.env.as_ref(); + let mut e: Vec = tmp.iter().flat_map(|e| e.iter()).cloned().collect(); + if let Some(s3) = s3bucket { + if let Some(secret) = s3.secret_class() { + e.push(Self::env_var_from_secret( + ENV_AWS_ACCESS_KEY_ID, + secret.as_ref(), + ACCESS_KEY_ID, + )); + e.push(Self::env_var_from_secret( + ENV_AWS_SECRET_ACCESS_KEY, + secret.as_ref(), + SECRET_ACCESS_KEY, + )); + } + } + e + } + + fn env_var_from_secret(var_name: &str, secret: &str, secret_key: &str) -> EnvVar { + EnvVar { + name: String::from(var_name), + value_from: Some(EnvVarSource { + secret_key_ref: Some(SecretKeySelector { + name: Some(String::from(secret)), + key: String::from(secret_key), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } + } } #[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -350,21 +378,6 @@ pub struct CommandStatus { pub finished_at: Option