Upgrade operator-rs to 0.24.0 (#219)

nightkr · nightkr · commit b6436eb36fec · 2022-08-11T13:56:37.000Z
# Description

*Please add a description here. This will become the commit message of the merge request later.*
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml
@@ -8,7 +8,7 @@ repository = "https://github.com/stackabletech/hdfs-operator"
 version = "0.5.0-nightly"
 
 [dependencies]
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.22.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.24.0" }
 
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }
diff --git a/rust/crd/src/error.rs b/rust/crd/src/error.rs
@@ -69,6 +69,8 @@ pub enum Error {
     #[error("Cannot find rolegroup [{rolegroup}]")]
     RoleGroupNotFound { rolegroup: String },
 
+    #[error("Object has no name")]
+    ObjectHasNoName,
     #[error("Object has no namespace [{obj_ref}]")]
     ObjectHasNoNamespace { obj_ref: ObjectRef<HdfsCluster> },
 
@@ -81,10 +83,9 @@ pub enum Error {
         role_group: String,
     },
 
-    #[error("Cannot build config discovery config map {name}. Caused by: {source}")]
+    #[error("Cannot build config discovery config map. Caused by: {source}")]
     BuildDiscoveryConfigMap {
         source: stackable_operator::error::Error,
-        name: String,
     },
 
     #[error("Pod has no name")]
diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml
@@ -9,7 +9,7 @@ version = "0.5.0-nightly"
 build = "build.rs"
 
 [dependencies]
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.22.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.24.0" }
 stackable-hdfs-crd = { path = "../crd" }
 stackable-hdfs-operator = { path = "../operator" }
 clap = { version = "3.2", features = ["derive"] }
@@ -19,7 +19,7 @@ serde_yaml = "0.8"
 
 [build-dependencies]
 built = { version =  "0.5", features = ["chrono", "git2"] }
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.22.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.24.0" }
 stackable-hdfs-crd = { path = "../crd" }
 
 [[bin]]
diff --git a/rust/operator/Cargo.toml b/rust/operator/Cargo.toml
@@ -8,7 +8,7 @@ repository = "https://github.com/stackabletech/hdfs-operator"
 version = "0.5.0-nightly"
 
 [dependencies]
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.22.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.24.0" }
 stackable-hdfs-crd = { path = "../crd" }
 
 futures = "0.3"
diff --git a/rust/operator/src/discovery.rs b/rust/operator/src/discovery.rs
@@ -1,4 +1,7 @@
-use crate::config::{CoreSiteConfigBuilder, HdfsNodeDataDirectory, HdfsSiteConfigBuilder};
+use crate::{
+    config::{CoreSiteConfigBuilder, HdfsNodeDataDirectory, HdfsSiteConfigBuilder},
+    OPERATOR_NAME,
+};
 use stackable_hdfs_crd::{
     constants::{APP_NAME, CORE_SITE_XML, HDFS_SITE_XML},
     HdfsCluster, HdfsPodRef, HdfsRole,
@@ -26,16 +29,20 @@ pub fn build_discovery_configmap(
                     APP_NAME,
                     hdfs.hdfs_version()
                         .map_err(|_| Error::MissingObjectKey { key: "version" })?,
+                    OPERATOR_NAME,
                     &HdfsRole::NameNode.to_string(),
                     "discovery",
                 )
                 .build(),
         )
         .add_data(
             HDFS_SITE_XML,
-            build_discovery_hdfs_site_xml(hdfs.name(), namenode_podrefs),
+            build_discovery_hdfs_site_xml(hdfs.name_any(), namenode_podrefs),
+        )
+        .add_data(
+            CORE_SITE_XML,
+            build_discovery_core_site_xml(hdfs.name_any()),
         )
-        .add_data(CORE_SITE_XML, build_discovery_core_site_xml(hdfs.name()))
         .build()
 }
 
diff --git a/rust/operator/src/hdfs_controller.rs b/rust/operator/src/hdfs_controller.rs
@@ -2,6 +2,7 @@ use crate::config::{
     CoreSiteConfigBuilder, HdfsNodeDataDirectory, HdfsSiteConfigBuilder, ROOT_DATA_DIR,
 };
 use crate::discovery::build_discovery_configmap;
+use crate::OPERATOR_NAME;
 use stackable_hdfs_crd::error::{Error, HdfsOperatorResult};
 use stackable_hdfs_crd::{constants::*, ROLE_PORTS};
 use stackable_hdfs_crd::{HdfsCluster, HdfsPodRef, HdfsRole};
@@ -24,7 +25,6 @@ use stackable_operator::kube::api::ObjectMeta;
 use stackable_operator::kube::runtime::controller::Action;
 use stackable_operator::kube::runtime::events::{Event, EventType, Recorder, Reporter};
 use stackable_operator::kube::runtime::reflector::ObjectRef;
-use stackable_operator::kube::ResourceExt;
 use stackable_operator::labels::role_group_selector_labels;
 use stackable_operator::memory::to_java_heap;
 use stackable_operator::product_config::{types::PropertyNameKind, ProductConfigManager};
@@ -59,12 +59,8 @@ pub async fn reconcile_hdfs(hdfs: Arc<HdfsCluster>, ctx: Arc<Ctx>) -> HdfsOperat
     let namenode_podrefs = hdfs.pod_refs(&HdfsRole::NameNode)?;
     let journalnode_podrefs = hdfs.pod_refs(&HdfsRole::JournalNode)?;
 
-    let discovery_cm = build_discovery_configmap(&hdfs, &namenode_podrefs).map_err(|e| {
-        Error::BuildDiscoveryConfigMap {
-            source: e,
-            name: hdfs.name(),
-        }
-    })?;
+    let discovery_cm = build_discovery_configmap(&hdfs, &namenode_podrefs)
+        .map_err(|e| Error::BuildDiscoveryConfigMap { source: e })?;
 
     client
         .apply_patch(FIELD_MANAGER_SCOPE, &discovery_cm, &discovery_cm)
@@ -167,6 +163,7 @@ fn rolegroup_service(
                 hdfs,
                 APP_NAME,
                 hdfs.hdfs_version()?,
+                OPERATOR_NAME,
                 &rolegroup_ref.role,
                 &rolegroup_ref.role_group,
             )
@@ -202,44 +199,52 @@ fn rolegroup_config_map(
 ) -> HdfsOperatorResult<ConfigMap> {
     tracing::info!("Setting up ConfigMap for {:?}", rolegroup_ref);
 
+    let hdfs_name = hdfs
+        .metadata
+        .name
+        .as_deref()
+        .ok_or(Error::ObjectHasNoName)?;
+
     let mut hdfs_site_xml = String::new();
     let mut core_site_xml = String::new();
 
     for (property_name_kind, config) in rolegroup_config {
         match property_name_kind {
             PropertyNameKind::File(file_name) if file_name == HDFS_SITE_XML => {
-                hdfs_site_xml =
-                    HdfsSiteConfigBuilder::new(hdfs.name(), HdfsNodeDataDirectory::default())
-                        // IMPORTANT: these folders must be under the volume mount point, otherwise they will not
-                        // be formatted by the namenode, or used by the other services.
-                        // See also: https://github.com/apache-spark-on-k8s/kubernetes-HDFS/commit/aef9586ecc8551ca0f0a468c3b917d8c38f494a0
-                        .dfs_namenode_name_dir()
-                        .dfs_datanode_data_dir()
-                        .dfs_journalnode_edits_dir()
-                        .dfs_replication(
-                            *hdfs
-                                .spec
-                                .dfs_replication
-                                .as_ref()
-                                .unwrap_or(&DEFAULT_DFS_REPLICATION_FACTOR),
-                        )
-                        .dfs_name_services()
-                        .dfs_ha_namenodes(namenode_podrefs)
-                        .dfs_namenode_shared_edits_dir(journalnode_podrefs)
-                        .dfs_namenode_name_dir_ha(namenode_podrefs)
-                        .dfs_namenode_rpc_address_ha(namenode_podrefs)
-                        .dfs_namenode_http_address_ha(namenode_podrefs)
-                        .dfs_client_failover_proxy_provider()
-                        .add("dfs.ha.fencing.methods", "shell(/bin/true)")
-                        .add("dfs.ha.nn.not-become-active-in-safemode", "true")
-                        .add("dfs.ha.automatic-failover.enabled", "true")
-                        .add("dfs.ha.namenode.id", "${env.POD_NAME}")
-                        // the extend with config must come last in order to have overrides working!!!
-                        .extend(config)
-                        .build_as_xml();
+                hdfs_site_xml = HdfsSiteConfigBuilder::new(
+                    hdfs_name.to_string(),
+                    HdfsNodeDataDirectory::default(),
+                )
+                // IMPORTANT: these folders must be under the volume mount point, otherwise they will not
+                // be formatted by the namenode, or used by the other services.
+                // See also: https://github.com/apache-spark-on-k8s/kubernetes-HDFS/commit/aef9586ecc8551ca0f0a468c3b917d8c38f494a0
+                .dfs_namenode_name_dir()
+                .dfs_datanode_data_dir()
+                .dfs_journalnode_edits_dir()
+                .dfs_replication(
+                    *hdfs
+                        .spec
+                        .dfs_replication
+                        .as_ref()
+                        .unwrap_or(&DEFAULT_DFS_REPLICATION_FACTOR),
+                )
+                .dfs_name_services()
+                .dfs_ha_namenodes(namenode_podrefs)
+                .dfs_namenode_shared_edits_dir(journalnode_podrefs)
+                .dfs_namenode_name_dir_ha(namenode_podrefs)
+                .dfs_namenode_rpc_address_ha(namenode_podrefs)
+                .dfs_namenode_http_address_ha(namenode_podrefs)
+                .dfs_client_failover_proxy_provider()
+                .add("dfs.ha.fencing.methods", "shell(/bin/true)")
+                .add("dfs.ha.nn.not-become-active-in-safemode", "true")
+                .add("dfs.ha.automatic-failover.enabled", "true")
+                .add("dfs.ha.namenode.id", "${env.POD_NAME}")
+                // the extend with config must come last in order to have overrides working!!!
+                .extend(config)
+                .build_as_xml();
             }
             PropertyNameKind::File(file_name) if file_name == CORE_SITE_XML => {
-                core_site_xml = CoreSiteConfigBuilder::new(hdfs.name())
+                core_site_xml = CoreSiteConfigBuilder::new(hdfs_name.to_string())
                     .fs_default_fs()
                     .ha_zookeeper_quorum()
                     // the extend with config must come last in order to have overrides working!!!
@@ -264,6 +269,7 @@ fn rolegroup_config_map(
                     hdfs,
                     APP_NAME,
                     hdfs.hdfs_version()?,
+                    OPERATOR_NAME,
                     &rolegroup_ref.role,
                     &rolegroup_ref.role_group,
                 )
@@ -352,6 +358,7 @@ fn rolegroup_statefulset(
                 hdfs,
                 APP_NAME,
                 hdfs.hdfs_version()?,
+                OPERATOR_NAME,
                 &rolegroup_ref.role,
                 &rolegroup_ref.role_group,
             )
diff --git a/rust/operator/src/lib.rs b/rust/operator/src/lib.rs
@@ -20,6 +20,8 @@ use stackable_operator::product_config::ProductConfigManager;
 use tracing::info_span;
 use tracing_futures::Instrument;
 
+const OPERATOR_NAME: &str = "hdfs-operator";
+
 pub async fn create_controller(
     client: Client,
     product_config: ProductConfigManager,
