Skip to content

Commit 0a8bf49

Browse files
siegfriedwebersiegfriedweber
authored
test: HBase connector (#471)
* test: HBase connector * Remove unused Minio Helm chart from hbase-connector test * Remove test images from the test definitons
1 parent e3d877f commit 0a8bf49

15 files changed

+362
-0
lines changed

‎tests/release.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,11 @@ releases:
1212
operatorVersion: 0.0.0-dev
1313
listener:
1414
operatorVersion: 0.0.0-dev
15+
zookeeper:
16+
operatorVersion: 0.0.0-dev
17+
hdfs:
18+
operatorVersion: 0.0.0-dev
19+
hbase:
20+
operatorVersion: 0.0.0-dev
1521
spark-k8s:
1622
operatorVersion: 0.0.0-dev

‎tests/templates/kuttl/hbase-connector/00-assert.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
timeout: 900
5+
---
6+
apiVersion: v1
7+
kind: ServiceAccount
8+
metadata:
9+
name: integration-tests-sa

‎tests/templates/kuttl/hbase-connector/00-patch-ns.yaml.j2

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{% if test_scenario['values']['openshift'] == 'true' %}
2+
# see https://github.com/stackabletech/issues/issues/566
3+
---
4+
apiVersion: kuttl.dev/v1beta1
5+
kind: TestStep
6+
commands:
7+
- script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}'
8+
timeout: 120
9+
{% endif %}

‎tests/templates/kuttl/hbase-connector/00-serviceaccount.yaml.j2

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
---
2+
kind: Role
3+
apiVersion: rbac.authorization.k8s.io/v1
4+
metadata:
5+
name: use-integration-tests-scc
6+
rules:
7+
{% if test_scenario['values']['openshift'] == "true" %}
8+
- apiGroups: ["security.openshift.io"]
9+
resources: ["securitycontextconstraints"]
10+
resourceNames: ["privileged"]
11+
verbs: ["use"]
12+
{% endif %}
13+
---
14+
apiVersion: v1
15+
kind: ServiceAccount
16+
metadata:
17+
name: integration-tests-sa
18+
---
19+
kind: RoleBinding
20+
apiVersion: rbac.authorization.k8s.io/v1
21+
metadata:
22+
name: use-integration-tests-scc
23+
subjects:
24+
- kind: ServiceAccount
25+
name: integration-tests-sa
26+
roleRef:
27+
kind: Role
28+
name: use-integration-tests-scc
29+
apiGroup: rbac.authorization.k8s.io

‎tests/templates/kuttl/hbase-connector/01-assert.yaml.j2

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
5+
---
6+
apiVersion: v1
7+
kind: ConfigMap
8+
metadata:
9+
name: vector-aggregator-discovery
10+
{% endif %}

‎tests/templates/kuttl/hbase-connector/01-install-vector-aggregator-discovery-configmap.yaml.j2

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
2+
---
3+
apiVersion: v1
4+
kind: ConfigMap
5+
metadata:
6+
name: vector-aggregator-discovery
7+
data:
8+
ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
9+
{% endif %}

‎tests/templates/kuttl/hbase-connector/02-assert.yaml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
metadata:
5+
name: install-zk
6+
timeout: 600
7+
---
8+
apiVersion: apps/v1
9+
kind: StatefulSet
10+
metadata:
11+
name: test-zk-server-default
12+
status:
13+
readyReplicas: 1
14+
replicas: 1

‎tests/templates/kuttl/hbase-connector/02-install-zookeeper.yaml.j2

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
---
2+
apiVersion: zookeeper.stackable.tech/v1alpha1
3+
kind: ZookeeperCluster
4+
metadata:
5+
name: test-zk
6+
spec:
7+
image:
8+
productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}"
9+
pullPolicy: IfNotPresent
10+
clusterConfig:
11+
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
12+
vectorAggregatorConfigMapName: vector-aggregator-discovery
13+
{% endif %}
14+
servers:
15+
config:
16+
logging:
17+
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
18+
roleGroups:
19+
default:
20+
replicas: 1
21+
---
22+
apiVersion: zookeeper.stackable.tech/v1alpha1
23+
kind: ZookeeperZnode
24+
metadata:
25+
name: test-znode
26+
spec:
27+
clusterRef:
28+
name: test-zk

‎tests/templates/kuttl/hbase-connector/03-assert.yaml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
metadata:
5+
name: install-hdfs
6+
timeout: 600
7+
---
8+
apiVersion: apps/v1
9+
kind: StatefulSet
10+
metadata:
11+
name: test-hdfs-namenode-default
12+
status:
13+
readyReplicas: 2
14+
replicas: 2
15+
---
16+
apiVersion: apps/v1
17+
kind: StatefulSet
18+
metadata:
19+
name: test-hdfs-journalnode-default
20+
status:
21+
readyReplicas: 1
22+
replicas: 1
23+
---
24+
apiVersion: apps/v1
25+
kind: StatefulSet
26+
metadata:
27+
name: test-hdfs-datanode-default
28+
status:
29+
readyReplicas: 1
30+
replicas: 1

‎tests/templates/kuttl/hbase-connector/03-install-hdfs.yaml.j2

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
---
2+
apiVersion: hdfs.stackable.tech/v1alpha1
3+
kind: HdfsCluster
4+
metadata:
5+
name: test-hdfs
6+
spec:
7+
image:
8+
productVersion: "{{ test_scenario['values']['hdfs-latest'] }}"
9+
pullPolicy: IfNotPresent
10+
clusterConfig:
11+
zookeeperConfigMapName: test-znode
12+
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
13+
vectorAggregatorConfigMapName: vector-aggregator-discovery
14+
{% endif %}
15+
nameNodes:
16+
config:
17+
logging:
18+
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
19+
roleGroups:
20+
default:
21+
replicas: 2
22+
dataNodes:
23+
config:
24+
logging:
25+
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
26+
roleGroups:
27+
default:
28+
replicas: 1
29+
journalNodes:
30+
config:
31+
logging:
32+
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
33+
roleGroups:
34+
default:
35+
replicas: 1

‎tests/templates/kuttl/hbase-connector/04-assert.yaml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
metadata:
5+
name: install-hbase
6+
timeout: 600
7+
---
8+
apiVersion: apps/v1
9+
kind: StatefulSet
10+
metadata:
11+
name: test-hbase-master-default
12+
status:
13+
readyReplicas: 2
14+
replicas: 2
15+
---
16+
apiVersion: apps/v1
17+
kind: StatefulSet
18+
metadata:
19+
name: test-hbase-regionserver-default
20+
status:
21+
readyReplicas: 1
22+
replicas: 1

‎tests/templates/kuttl/hbase-connector/04-install-hbase.yaml.j2

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
---
2+
apiVersion: hbase.stackable.tech/v1alpha1
3+
kind: HbaseCluster
4+
metadata:
5+
name: test-hbase
6+
spec:
7+
image:
8+
productVersion: "{{ test_scenario['values']['hbase'] }}"
9+
pullPolicy: IfNotPresent
10+
clusterConfig:
11+
hdfsConfigMapName: test-hdfs
12+
zookeeperConfigMapName: test-znode
13+
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
14+
vectorAggregatorConfigMapName: vector-aggregator-discovery
15+
{% endif %}
16+
masters:
17+
config:
18+
logging:
19+
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
20+
roleGroups:
21+
default:
22+
replicas: 2
23+
regionServers:
24+
config:
25+
logging:
26+
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
27+
roleGroups:
28+
default:
29+
replicas: 1

‎tests/templates/kuttl/hbase-connector/10-assert.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestAssert
4+
timeout: 900
5+
---
6+
apiVersion: spark.stackable.tech/v1alpha1
7+
kind: SparkApplication
8+
metadata:
9+
name: test-spark-hbase-connector
10+
status:
11+
phase: Succeeded

‎tests/templates/kuttl/hbase-connector/10-deploy-spark-app.yaml.j2

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
---
2+
apiVersion: spark.stackable.tech/v1alpha1
3+
kind: SparkApplication
4+
metadata:
5+
name: test-spark-hbase-connector
6+
spec:
7+
{% if lookup('env', 'VECTOR_AGGREGATOR') %}
8+
vectorAggregatorConfigMapName: vector-aggregator-discovery
9+
{% endif %}
10+
sparkImage:
11+
{% if test_scenario['values']['spark'].find(",") > 0 %}
12+
custom: "{{ test_scenario['values']['spark'].split(',')[1] }}"
13+
productVersion: "{{ test_scenario['values']['spark'].split(',')[0] }}"
14+
{% else %}
15+
productVersion: "{{ test_scenario['values']['spark'] }}"
16+
{% endif %}
17+
# pullPolicy: IfNotPresent
18+
pullPolicy: Always
19+
mode: cluster
20+
mainApplicationFile: local:///stackable/spark/jobs/test-hbase.py
21+
sparkConf:
22+
spark.driver.extraClassPath: /stackable/spark/config
23+
spark.executor.extraClassPath: /stackable/spark/config
24+
driver:
25+
config:
26+
logging:
27+
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
28+
volumeMounts:
29+
- name: script
30+
mountPath: /stackable/spark/jobs
31+
- name: hbase-config
32+
mountPath: /stackable/spark/config/hbase-site.xml
33+
subPath: hbase-site.xml
34+
- name: hdfs-config
35+
mountPath: /stackable/spark/config/hdfs-site.xml
36+
subPath: hdfs-site.xml
37+
- name: hdfs-config
38+
mountPath: /stackable/spark/config/core-site.xml
39+
subPath: core-site.xml
40+
executor:
41+
replicas: 1
42+
config:
43+
logging:
44+
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
45+
volumeMounts:
46+
- name: script
47+
mountPath: /stackable/spark/jobs
48+
- name: hbase-config
49+
mountPath: /stackable/spark/config/hbase-site.xml
50+
subPath: hbase-site.xml
51+
- name: hdfs-config
52+
mountPath: /stackable/spark/config/hdfs-site.xml
53+
subPath: hdfs-site.xml
54+
- name: hdfs-config
55+
mountPath: /stackable/spark/config/core-site.xml
56+
subPath: core-site.xml
57+
volumes:
58+
- name: script
59+
configMap:
60+
name: test-hbase
61+
- name: hbase-config
62+
configMap:
63+
name: test-hbase
64+
- name: hdfs-config
65+
configMap:
66+
name: test-hdfs
67+
---
68+
apiVersion: v1
69+
kind: ConfigMap
70+
metadata:
71+
name: test-hbase
72+
data:
73+
test-hbase.py: |
74+
import os
75+
from pyspark.sql import SparkSession
76+
from pyspark.sql.types import *
77+
78+
spark = SparkSession.builder.appName("test-hbase").getOrCreate()
79+
80+
df = spark.createDataFrame(
81+
[("row1", "Hello, Stackable!")],
82+
"key: string, value: string"
83+
)
84+
85+
spark._jvm.org.apache.hadoop.hbase.spark.HBaseContext(
86+
spark._jsc.sc(),
87+
spark._jvm.org.apache.hadoop.hbase.HBaseConfiguration.create(),
88+
None,
89+
)
90+
91+
catalog = '{\
92+
"table":{"namespace":"default","name":"test-hbase"},\
93+
"rowkey":"key",\
94+
"columns":{\
95+
"key":{"cf":"rowkey","col":"key","type":"string"},\
96+
"value":{"cf":"cf1","col":"value","type":"string"}\
97+
}}'
98+
99+
df\
100+
.write\
101+
.format("org.apache.hadoop.hbase.spark")\
102+
.option('catalog', catalog)\
103+
.option('newtable', '5')\
104+
.save()

‎tests/test-definition.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,16 @@ dimensions:
1414
values:
1515
- 3.5.2
1616
# - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev
17+
- name: hbase
18+
values:
19+
- 2.6.0
20+
- 2.4.18
21+
- name: hdfs-latest
22+
values:
23+
- 3.4.0
24+
- name: zookeeper-latest
25+
values:
26+
- 3.9.2
1727
- name: ny-tlc-report
1828
values:
1929
- 0.2.0
@@ -83,6 +93,13 @@ tests:
8393
- spark-delta-lake
8494
- delta
8595
- openshift
96+
- name: hbase-connector
97+
dimensions:
98+
- spark
99+
- hbase
100+
- hdfs-latest
101+
- zookeeper-latest
102+
- openshift
86103
suites:
87104
- name: nightly
88105
patch:

0 commit comments

Comments
 (0)