Use NiFi without Zookeeper; fix various problems (#221)

* Use NiFi without Zookeeper

* fix: Superset listenerClass CRD change

* fix: Setup Superset Jobs

Author: sbernauer

demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml

@@ -39,7 +39,7 @@ data:
     import logging
     import requests
 
-    base_url = "http://superset-external:8088" # For local testing / developing replace it, afterwards change back to http://superset-external:8088
+    base_url = "http://superset-node-default:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default:8088
     superset_username = open("/superset-credentials/adminUser.username").read()
     superset_password =  open("/superset-credentials/adminUser.password").read()
     trino_username = "admin"

demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml

@@ -55,7 +55,7 @@ data:
     import logging
     import requests
 
-    base_url = "http://superset-external:8088" # For local testing / developing replace it, afterwards change back to http://superset-external:8088
+    base_url = "http://superset-node-default:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default:8088
     superset_username = open("/superset-credentials/adminUser.username").read()
     superset_password =  open("/superset-credentials/adminUser.password").read()
 

demos/nifi-kafka-druid-water-level-data/setup-superset.yaml

@@ -55,7 +55,7 @@ data:
     import logging
     import requests
 
-    base_url = "http://superset-external:8088" # For local testing / developing replace it, afterwards change back to http://superset-external:8088
+    base_url = "http://superset-node-default:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default:8088
     superset_username = open("/superset-credentials/adminUser.username").read()
     superset_password =  open("/superset-credentials/adminUser.password").read()
 

demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml

@@ -39,7 +39,7 @@ data:
     import logging
     import requests
 
-    base_url = "http://superset-external:8088" # For local testing / developing replace it, afterwards change back to http://superset-external:8088
+    base_url = "http://superset-node-default:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default:8088
     superset_username = open("/superset-credentials/adminUser.username").read()
     superset_password =  open("/superset-credentials/adminUser.password").read()
     trino_username = "admin"

demos/trino-taxi-data/setup-superset.yaml

@@ -39,7 +39,7 @@ data:
     import logging
     import requests
 
-    base_url = "http://superset-external:8088" # For local testing / developing replace it, afterwards change back to http://superset-external:8088
+    base_url = "http://superset-node-default:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default:8088
     superset_username = open("/superset-credentials/adminUser.username").read()
     superset_password =  open("/superset-credentials/adminUser.password").read()
     trino_username = "admin"

stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml

@@ -1,12 +1,4 @@
 ---
-apiVersion: zookeeper.stackable.tech/v1alpha1
-kind: ZookeeperZnode
-metadata:
-  name: kafka-znode
-spec:
-  clusterRef:
-    name: zookeeper
----
 apiVersion: kafka.stackable.tech/v1alpha1
 kind: KafkaCluster
 metadata:
@@ -46,3 +38,11 @@ spec:
   provider:
     tls:
       clientCertSecretClass: tls
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperZnode
+metadata:
+  name: kafka-znode
+spec:
+  clusterRef:
+    name: zookeeper

stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml

@@ -13,7 +13,6 @@ spec:
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
-    zookeeperConfigMapName: nifi-znode
   nodes:
     config:
       resources:
@@ -53,11 +52,3 @@ metadata:
   name: nifi-admin-credentials-secret
 stringData:
   admin: {{ nifiAdminPassword }}
----
-apiVersion: zookeeper.stackable.tech/v1alpha1
-kind: ZookeeperZnode
-metadata:
-  name: nifi-znode
-spec:
-  clusterRef:
-    name: zookeeper

stacks/end-to-end-security/superset.yaml

@@ -7,7 +7,6 @@ spec:
   image:
     productVersion: 4.1.2
   clusterConfig:
-    listenerClass: external-unstable
     credentialsSecret: superset-credentials
     mapboxSecret: superset-mapbox-api-key
     authentication:
@@ -16,6 +15,8 @@ spec:
           clientCredentialsSecret: superset-keycloak-client
         userRegistrationRole: Gamma_extended
   nodes:
+    config:
+      listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1

stacks/keycloak-opa-poc/superset.yaml

@@ -22,11 +22,12 @@ spec:
     productVersion: 4.1.2
   clusterConfig:
     credentialsSecret: superset-credentials
-    listenerClass: external-unstable
   nodes:
     roleGroups:
       default:
         replicas: 1
+    config:
+      listenerClass: external-unstable
     configOverrides:
       superset_config.py:
         AUTH_TYPE: AUTH_OAUTH

stacks/nifi-kafka-druid-superset-s3/nifi.yaml

@@ -13,7 +13,6 @@ spec:
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
-    zookeeperConfigMapName: nifi-znode
   nodes:
     config:
       resources:
@@ -53,11 +52,3 @@ metadata:
   name: nifi-admin-credentials-secret
 stringData:
   admin: {{ nifiAdminPassword }}
----
-apiVersion: zookeeper.stackable.tech/v1alpha1
-kind: ZookeeperZnode
-metadata:
-  name: nifi-znode
-spec:
-  clusterRef:
-    name: zookeeper

stacks/nifi-kafka-druid-superset-s3/superset.yaml

@@ -7,10 +7,11 @@ spec:
   image:
     productVersion: 4.1.2
   clusterConfig:
-    listenerClass: external-unstable
     credentialsSecret: superset-credentials
     mapboxSecret: superset-mapbox-api-key
   nodes:
+    config:
+      listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1

stacks/signal-processing/nifi.yaml

@@ -12,7 +12,6 @@ spec:
     # pullPolicy: IfNotPresent
   clusterConfig:
     listenerClass: external-unstable
-    zookeeperConfigMapName: nifi-znode
     authentication:
       - authenticationClass: nifi-users
     sensitiveProperties:
@@ -57,11 +56,3 @@ metadata:
   name: nifi-admin-credentials
 stringData:
   admin: {{ nifiAdminPassword }}
----
-apiVersion: zookeeper.stackable.tech/v1alpha1
-kind: ZookeeperZnode
-metadata:
-  name: nifi-znode
-spec:
-  clusterRef:
-    name: zookeeper

stacks/stacks-v2.yaml

@@ -629,7 +629,6 @@ stacks:
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/grafana-dashboards.yaml
       - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/grafana.yaml
       - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-timescaledb.yaml
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/nifi-kafka-druid-superset-s3/zookeeper.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/nifi.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/serviceaccount.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/spark_driver_service.yaml

stacks/trino-superset-s3/superset.yaml

@@ -7,10 +7,11 @@ spec:
   image:
     productVersion: 4.1.2
   clusterConfig:
-    listenerClass: external-unstable
     credentialsSecret: superset-credentials
     mapboxSecret: superset-mapbox-api-key
   nodes:
+    config:
+      listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1