update demos for 23.7 (#269)

## Description

Fixes stackabletech/issues#402

N.B. DO NOT MERGE UNTIL AFTER RELEASE-23.7!!
Keep this PR open until after the release so that e.g. versions 0.0.0-dev can be replaced with 23.7



Co-authored-by: Andrew Kenworthy <[email protected]>

Author: sbernauer

demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml

@@ -18,15 +18,15 @@ spec:
           volumeMounts:
             - name: script
               mountPath: /tmp/script
-            - name: trino-credentials
-              mountPath: /trino-credentials
+            - name: trino-users
+              mountPath: /trino-users
       volumes:
         - name: script
           configMap:
             name: create-tables-in-trino-script
-        - name: trino-credentials
+        - name: trino-users
           secret:
-            secretName: trino-credentials
+            secretName: trino-users
       restartPolicy: OnFailure
   backoffLimit: 50
 ---
@@ -49,7 +49,7 @@ data:
             port=8443,
             user="admin",
             http_scheme='https',
-            auth=trino.auth.BasicAuthentication("admin", open("/trino-credentials/admin").read()),
+            auth=trino.auth.BasicAuthentication("admin", open("/trino-users/admin").read()),
         )
         connection._http_session.verify = False
         return connection

demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml

@@ -10,7 +10,7 @@ spec:
         - name: load-test-data
           image: "bitnami/minio:2022-debian-10"
           # Please try to order the load jobs from small to large datasets
-          command: ["bash", "-c", "mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && cd /tmp && curl -O https://repo.stackable.tech/repository/misc/datasets/open-postcode-geo/open-postcode-geo.csv && mc cp open-postcode-geo.csv minio/staging/house-sales/postcode-geo-lookup/ && rm open-postcode-geo.csv && for year in {2005..2021}; do curl -O https://repo.stackable.tech/repository/misc/datasets/uk-house-sales/uk-house-sales-$year.csv && mc cp uk-house-sales-$year.csv minio/staging/house-sales/house-sales/ && rm uk-house-sales-$year.csv; done && curl -O https://repo.stackable.tech/repository/misc/earthquake-data/earthquakes_1950_to_2022.csv && mc cp earthquakes_1950_to_2022.csv minio/staging/earthquakes/earthquakes/ && rm earthquakes_1950_to_2022.csv && curl -O https://repo.stackable.tech/repository/misc/datasets/e-charging-stations/e-charging-stations-2022-08.csv && mc cp e-charging-stations-2022-08.csv minio/staging/smart-city/e-charging-stations/ && rm e-charging-stations-2022-08.csv && curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/taxi_zone_lookup.csv && mc cp taxi_zone_lookup.csv minio/staging/taxi/taxi-zone-lookup/ && rm taxi_zone_lookup.csv && curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/rate_code_lookup.csv && mc cp rate_code_lookup.csv minio/staging/taxi/rate-code-lookup/ && rm rate_code_lookup.csv && curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/payment_type_lookup.csv && mc cp payment_type_lookup.csv minio/staging/taxi/payment-type-lookup/ && rm payment_type_lookup.csv && for month in 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/green_tripdata_$month.parquet && mc cp green_tripdata_$month.parquet minio/staging/taxi/green-tripdata/ && rm green_tripdata_$month.parquet; done && for month in 2015-01 2015-02 2015-03 2015-04 2015-05 2015-06 2015-07 2015-08 2015-09 2015-10 2015-11 2015-12 2016-01 2016-02 2016-03 2016-04 2016-05 2016-06 2016-07 2016-08 2016-09 2016-10 2016-11 2016-12 2017-01 2017-02 2017-03 2017-04 2017-05 2017-06 2017-07 2017-08 2017-09 2017-10 2017-11 2017-12 2018-01 2018-02 2018-03 2018-04 2018-05 2018-06 2018-07 2018-08 2018-09 2018-10 2018-11 2018-12 2019-01 2019-02 2019-03 2019-04 2019-05 2019-06 2019-07 2019-08 2019-09 2019-10 2019-11 2019-12 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet && mc cp yellow_tripdata_$month.parquet minio/staging/taxi/yellow-tripdata/ && rm yellow_tripdata_$month.parquet; done && for month in 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet && mc cp fhvhv_tripdata_$month.parquet minio/staging/taxi/fhvhv-tripdata/ && rm fhvhv_tripdata_$month.parquet; done"]
+          command: ["bash", "-c", "mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && cd /tmp && curl -O https://repo.stackable.tech/repository/misc/datasets/open-postcode-geo/open-postcode-geo.csv && mc cp open-postcode-geo.csv minio/staging/house-sales/postcode-geo-lookup/ && rm open-postcode-geo.csv && for year in {2005..2021}; do curl -O https://repo.stackable.tech/repository/misc/datasets/uk-house-sales/uk-house-sales-$year.csv && mc cp uk-house-sales-$year.csv minio/staging/house-sales/house-sales/ && rm uk-house-sales-$year.csv; done && curl -O https://repo.stackable.tech/repository/misc/earthquake-data/earthquakes_1950_to_2022.csv && mc cp earthquakes_1950_to_2022.csv minio/staging/earthquakes/earthquakes/ && rm earthquakes_1950_to_2022.csv && curl -O https://repo.stackable.tech/repository/misc/datasets/e-charging-stations/e-charging-stations-2022-08.csv && mc cp e-charging-stations-2022-08.csv minio/staging/smart-city/e-charging-stations/ && rm e-charging-stations-2022-08.csv && curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/taxi_zone_lookup.csv && mc cp taxi_zone_lookup.csv minio/staging/taxi/taxi-zone-lookup/ && rm taxi_zone_lookup.csv && curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/rate_code_lookup.csv && mc cp rate_code_lookup.csv minio/staging/taxi/rate-code-lookup/ && rm rate_code_lookup.csv && curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/payment_type_lookup.csv && mc cp payment_type_lookup.csv minio/staging/taxi/payment-type-lookup/ && rm payment_type_lookup.csv && for month in 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/green_tripdata_$month.parquet && mc cp green_tripdata_$month.parquet minio/staging/taxi/green-tripdata/ && rm green_tripdata_$month.parquet; done && for month in 2015-01 2015-02 2015-03 2015-04 2015-05 2015-06 2015-07 2015-08 2015-09 2015-10 2015-11 2015-12 2016-01 2016-02 2016-03 2016-04 2016-05 2016-06 2016-07 2016-08 2016-09 2016-10 2016-11 2016-12 2017-01 2017-02 2017-03 2017-04 2017-05 2017-06 2017-07 2017-08 2017-09 2017-10 2017-11 2017-12 2018-01 2018-02 2018-03 2018-04 2018-05 2018-06 2018-07 2018-08 2018-09 2018-10 2018-11 2018-12 2019-01 2019-02 2019-03 2019-04 2019-05 2019-06 2019-07 2019-08 2019-09 2019-10 2019-11 2019-12 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06 2022-07 2022-08 2022-09 2022-10 2022-11 2022-12 2023-01 2023-02 2023-03 2023-04; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet && mc cp yellow_tripdata_$month.parquet minio/staging/taxi/yellow-tripdata/ && rm yellow_tripdata_$month.parquet; done && for month in 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet && mc cp fhvhv_tripdata_$month.parquet minio/staging/taxi/fhvhv-tripdata/ && rm fhvhv_tripdata_$month.parquet; done"]
           volumeMounts:
             - name: minio-s3-credentials
               mountPath: /minio-s3-credentials

demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml

@@ -13,8 +13,8 @@ spec:
           volumeMounts:
             - name: script
               mountPath: /tmp/script
-            - name: trino-credentials
-              mountPath: /trino-credentials
+            - name: trino-users
+              mountPath: /trino-users
             - name: superset-credentials
               mountPath: /superset-credentials
       volumes:
@@ -24,9 +24,9 @@ spec:
         - name: superset-credentials
           secret:
             secretName: superset-credentials
-        - name: trino-credentials
+        - name: trino-users
           secret:
-            secretName: trino-credentials
+            secretName: trino-users
       restartPolicy: OnFailure
   backoffLimit: 50
 ---
@@ -43,7 +43,7 @@ data:
     superset_username = open("/superset-credentials/adminUser.username").read()
     superset_password =  open("/superset-credentials/adminUser.password").read()
     trino_username = "admin"
-    trino_password = open("/trino-credentials/admin").read()
+    trino_password = open("/trino-users/admin").read()
 
     logging.basicConfig(level=logging.INFO)
     logging.info("Starting setup of Superset")

demos/logging/zookeeper.yaml

@@ -6,7 +6,7 @@ metadata:
 spec:
   image:
     productVersion: 3.8.0
-    stackableVersion: "23.4"
+    stackableVersion: "23.7"
   clusterConfig:
     logging:
       vectorAggregatorConfigMapName: vector-aggregator-discovery

demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml

@@ -13,8 +13,8 @@ spec:
           volumeMounts:
             - name: script
               mountPath: /tmp/script
-            - name: trino-credentials
-              mountPath: /trino-credentials
+            - name: trino-users
+              mountPath: /trino-users
             - name: superset-credentials
               mountPath: /superset-credentials
       volumes:
@@ -24,9 +24,9 @@ spec:
         - name: superset-credentials
           secret:
             secretName: superset-credentials
-        - name: trino-credentials
+        - name: trino-users
           secret:
-            secretName: trino-credentials
+            secretName: trino-users
       restartPolicy: OnFailure
   backoffLimit: 50
 ---
@@ -43,7 +43,7 @@ data:
     superset_username = open("/superset-credentials/adminUser.username").read()
     superset_password =  open("/superset-credentials/adminUser.password").read()
     trino_username = "admin"
-    trino_password = open("/trino-credentials/admin").read()
+    trino_password = open("/trino-users/admin").read()
 
     logging.basicConfig(level=logging.INFO)
     logging.info("Starting setup of Superset")

demos/trino-taxi-data/create-table-in-trino.yaml

@@ -13,15 +13,15 @@ spec:
           volumeMounts:
             - name: script
               mountPath: /tmp/script
-            - name: trino-credentials
-              mountPath: /trino-credentials
+            - name: trino-users
+              mountPath: /trino-users
       volumes:
         - name: script
           configMap:
             name: create-ny-taxi-data-table-in-trino-script
-        - name: trino-credentials
+        - name: trino-users
           secret:
-            secretName: trino-credentials
+            secretName: trino-users
       restartPolicy: OnFailure
   backoffLimit: 50
 ---
@@ -44,7 +44,7 @@ data:
             port=8443,
             user="admin",
             http_scheme='https',
-            auth=trino.auth.BasicAuthentication("admin", open("/trino-credentials/admin").read()),
+            auth=trino.auth.BasicAuthentication("admin", open("/trino-users/admin").read()),
         )
         connection._http_session.verify = False
         return connection

demos/trino-taxi-data/setup-superset.yaml

@@ -13,8 +13,8 @@ spec:
           volumeMounts:
             - name: script
               mountPath: /tmp/script
-            - name: trino-credentials
-              mountPath: /trino-credentials
+            - name: trino-users
+              mountPath: /trino-users
             - name: superset-credentials
               mountPath: /superset-credentials
       volumes:
@@ -24,9 +24,9 @@ spec:
         - name: superset-credentials
           secret:
             secretName: superset-credentials
-        - name: trino-credentials
+        - name: trino-users
           secret:
-            secretName: trino-credentials
+            secretName: trino-users
       restartPolicy: OnFailure
   backoffLimit: 50
 ---
@@ -43,7 +43,7 @@ data:
     superset_username = open("/superset-credentials/adminUser.username").read()
     superset_password =  open("/superset-credentials/adminUser.password").read()
     trino_username = "admin"
-    trino_password = open("/trino-credentials/admin").read()
+    trino_password = open("/trino-users/admin").read()
 
     logging.basicConfig(level=logging.INFO)
     logging.info("Starting setup of Superset")

docs/modules/ROOT/pages/demos/trino-iceberg.adoc

@@ -65,6 +65,7 @@ java -jar ~/Downloads/trino-cli-396-executable.jar --user admin --insecure --pas
 ----
 
 Make sure to replace the server endpoint with the endpoint listed in the `stackablectl services list` output.
+When prompted enter the password `adminadmin`.
 
 == Create testdata
 === Create Schema

stacks/_templates/minio-distributed-small.yaml

@@ -3,7 +3,7 @@ name: minio
 repo:
   name: minio
   url: https://charts.min.io/
-version: 5.0.8
+version: 5.0.13
 options:
   rootUser: admin
   rootPassword: {{ minioAdminPassword }}

stacks/_templates/minio-distributed.yaml

@@ -3,7 +3,7 @@ name: minio
 repo:
   name: minio
   url: https://charts.min.io/
-version: 5.0.8
+version: 5.0.13
 options:
   rootUser: admin
   rootPassword: {{ minioAdminPassword }}

stacks/_templates/minio.yaml

@@ -3,7 +3,7 @@ name: minio
 repo:
   name: minio
   url: https://charts.min.io/
-version: 5.0.8
+version: 5.0.13
 options:
   rootUser: admin
   rootPassword: {{ minioAdminPassword }}

stacks/_templates/opensearch-dashboards.yaml

@@ -3,7 +3,7 @@ name: opensearch-dashboards
 repo:
   name: opensearch-dashboards
   url: https://opensearch-project.github.io/helm-charts
-version: 2.9.2
+version: 2.11.1
 options:
   service:
     type: NodePort

stacks/_templates/opensearch.yaml

@@ -3,7 +3,7 @@ name: opensearch
 repo:
   name: opensearch
   url: https://opensearch-project.github.io/helm-charts
-version: 2.11.3
+version: 2.13.3
 options:
   config:
     opensearch.yml: |

stacks/_templates/postgresql-airflow.yaml

@@ -3,12 +3,7 @@ name: postgresql
 repo:
   name: bitnami
   url: https://charts.bitnami.com/bitnami/
-# helm search repo bitnami/postgresql --versions
-# bitnami/postgresql      12.2.8          15.2.0          PostgreSQL (Postgres) is an open source object-
-#
-# https://airflow.apache.org/docs/apache-airflow/2.4.1/howto/set-up-database.html
-# Airflow 2.4.1 supports PostgreSQL: 10, 11, 12, 13, seems to work anyway
-version: 12.2.8
+version: 12.6.6
 options:
   auth:
     username: airflow

stacks/_templates/postgresql-druid.yaml

@@ -3,9 +3,7 @@ name: postgresql
 repo:
   name: bitnami
   url: https://charts.bitnami.com/bitnami/
-# helm search repo bitnami/postgresql --versions
-# bitnami/postgresql      12.2.8          15.2.0          PostgreSQL (Postgres) is an open source object-
-version: 12.2.8
+version: 12.6.6
 options:
   auth:
     username: druid

stacks/_templates/postgresql-hive-iceberg.yaml

@@ -3,9 +3,7 @@ name: postgresql
 repo:
   name: bitnami
   url: https://charts.bitnami.com/bitnami/
-# helm search repo bitnami/postgresql --versions
-# bitnami/postgresql      12.2.8          15.2.0          PostgreSQL (Postgres) is an open source object-
-version: 12.2.8
+version: 12.6.6
 options:
   auth:
     username: hive

stacks/_templates/postgresql-hive.yaml

@@ -3,9 +3,7 @@ name: postgresql
 repo:
   name: bitnami
   url: https://charts.bitnami.com/bitnami/
-# helm search repo bitnami/postgresql --versions
-# bitnami/postgresql      12.2.8          15.2.0          PostgreSQL (Postgres) is an open source object-
-version: 12.2.8
+version: 12.6.6
 options:
   auth:
     username: hive

stacks/_templates/postgresql-hivehdfs.yaml

@@ -3,9 +3,7 @@ name: postgresql
 repo:
   name: bitnami
   url: https://charts.bitnami.com/bitnami/
-# helm search repo bitnami/postgresql --versions
-# bitnami/postgresql      12.2.8          15.2.0          PostgreSQL (Postgres) is an open source object-
-version: 12.2.8
+version: 12.6.6
 options:
   auth:
     username: hive

stacks/_templates/postgresql-hives3.yaml

@@ -3,9 +3,7 @@ name: postgresql
 repo:
   name: bitnami
   url: https://charts.bitnami.com/bitnami/
-# helm search repo bitnami/postgresql --versions
-# bitnami/postgresql      12.2.8          15.2.0          PostgreSQL (Postgres) is an open source object-
-version: 12.2.8
+version: 12.6.6
 options:
   auth:
     username: hive

stacks/_templates/postgresql-superset.yaml

@@ -3,9 +3,7 @@ name: postgresql
 repo:
   name: bitnami
   url: https://charts.bitnami.com/bitnami/
-# helm search repo bitnami/postgresql --versions
-# bitnami/postgresql      12.2.8          15.2.0          PostgreSQL (Postgres) is an open source object-
-version: 12.2.8
+version: 12.6.6
 options:
   auth:
     username: superset

stacks/_templates/prometheus.yaml

@@ -3,7 +3,7 @@ name: kube-prometheus-stack
 repo:
   name: prometheus-community
   url: https://prometheus-community.github.io/helm-charts
-version: 45.10.1
+version: 48.1.1
 options:
   prometheus:
     prometheusSpec:

stacks/_templates/redis-airflow.yaml

@@ -3,10 +3,7 @@ name: redis
 repo:
   name: bitnami
   url: https://charts.bitnami.com/bitnami/
-# helm search repo bitnami/redis --versions
-# NAME                    CHART VERSION   APP VERSION     DESCRIPTION
-# bitnami/redis           17.9.5          7.0.11          Redis(R) is an open source, advanced key-value ...
-version: 17.9.5
+version: 17.13.2
 options:
   auth:
     password: airflow

stacks/_templates/vector-aggregator.yaml

@@ -3,7 +3,7 @@ name: vector
 repo:
   name: vector
   url: https://helm.vector.dev
-version: 0.21.0
+version: 0.23.0
 options:
   role: Aggregator
   customConfig:

stacks/airflow/airflow.yaml

@@ -6,28 +6,28 @@ metadata:
   name: airflow
 spec:
   image:
-    productVersion: 2.4.1
-    stackableVersion: "23.4"
+    productVersion: 2.6.1
+    stackableVersion: "23.7"
   clusterConfig:
     listenerClass: external-unstable
-  executor: CeleryExecutor
-  loadExamples: false
-  exposeConfig: false
-  credentialsSecret: airflow-credentials
-  volumes:
-    - name: airflow-dags
-      configMap:
-        name: airflow-dags
-  volumeMounts:
-    - name: airflow-dags
-      mountPath: /dags/date_demo.py
-      subPath: date_demo.py
-    - name: airflow-dags
-      mountPath: /dags/pyspark_pi.py
-      subPath: pyspark_pi.py
-    - name: airflow-dags
-      mountPath: /dags/pyspark_pi.yaml
-      subPath: pyspark_pi.yaml
+    executor: CeleryExecutor
+    loadExamples: false
+    exposeConfig: false
+    credentialsSecret: airflow-credentials
+    volumes:
+      - name: airflow-dags
+        configMap:
+          name: airflow-dags
+    volumeMounts:
+      - name: airflow-dags
+        mountPath: /dags/date_demo.py
+        subPath: date_demo.py
+      - name: airflow-dags
+        mountPath: /dags/pyspark_pi.py
+        subPath: pyspark_pi.py
+      - name: airflow-dags
+        mountPath: /dags/pyspark_pi.yaml
+        subPath: pyspark_pi.yaml
   webservers:
     roleGroups:
       default: