Skip to content

Commit 770c3b0

Browse files
adwk67adwk67
committed
hdfs getting-started (#226)
# Description Getting-started docs. Closes #220.
1 parent c17aa71 commit 770c3b0

File tree

15 files changed

+558
-109
lines changed

15 files changed

+558
-109
lines changed

‎docs/antora.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,6 @@ name: hdfs
22
version: "nightly"
33
title: Stackable Operator for Apache HDFS
44
nav:
5+
- modules/getting_started/nav.adoc
56
- modules/ROOT/nav.adoc
67
prerelease: true

‎docs/modules/ROOT/nav.adoc

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
* xref:installation.adoc[]
21
* xref:configuration.adoc[]
32
* xref:usage.adoc[]
43
* xref:implementation.adoc[]

‎docs/modules/ROOT/pages/installation.adoc

Lines changed: 0 additions & 57 deletions
This file was deleted.

‎docs/modules/getting_started/examples/code/getting_started.sh

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# This script contains all the code snippets from the guide, as well as some assert tests
5+
# to test if the instructions in the guide work. The user *could* use it, but it is intended
6+
# for testing only.
7+
# The script will install the operators, create a superset instance and briefly open a port
8+
# forward and connect to the superset instance to make sure it is up and running.
9+
# No running processes are left behind (i.e. the port-forwarding is closed at the end)
10+
11+
if [ $# -eq 0 ]
12+
then
13+
echo "Installation method argument ('helm' or 'stackablectl') required."
14+
exit 1
15+
fi
16+
17+
case "$1" in
18+
"helm")
19+
echo "Adding 'stackable-dev' Helm Chart repository"
20+
# tag::helm-add-repo[]
21+
helm repo add stackable-dev https://repo.stackable.tech/repository/helm-dev/
22+
# end::helm-add-repo[]
23+
echo "Installing Operators with Helm"
24+
# tag::helm-install-operators[]
25+
helm install --wait zookeeper-operator stackable-dev/zookeeper-operator --version 0.11.0-nightly
26+
helm install --wait hdfs-operator stackable-dev/hdfs-operator --version 0.5.0-nightly
27+
helm install --wait commons-operator stackable-dev/commons-operator --version 0.3.0-nightly
28+
helm install --wait secret-operator stackable-dev/secret-operator --version 0.6.0-nightly
29+
# end::helm-install-operators[]
30+
;;
31+
"stackablectl")
32+
echo "installing Operators with stackablectl"
33+
# tag::stackablectl-install-operators[]
34+
stackablectl operator install \
35+
commons=0.3.0-nightly \
36+
secret=0.6.0-nightly \
37+
zookeeper=0.11.0-nightly \
38+
hdfs=0.5.0-nightly
39+
# end::stackablectl-install-operators[]
40+
;;
41+
*)
42+
echo "Need to give 'helm' or 'stackablectl' as an argument for which installation method to use!"
43+
exit 1
44+
;;
45+
esac
46+
47+
echo "Creating Zookeeper cluster"
48+
# tag::install-zk[]
49+
kubectl apply -f zk.yaml
50+
# end::install-zk[]
51+
52+
echo "Creating ZNode"
53+
# tag::install-zk[]
54+
kubectl apply -f znode.yaml
55+
# end::install-zk[]
56+
57+
sleep 5
58+
59+
echo "Awaiting Zookeeper rollout finish"
60+
# tag::watch-zk-rollout[]
61+
kubectl rollout status --watch statefulset/simple-zk-server-default
62+
# end::watch-zk-rollout[]
63+
64+
echo "Creating HDFS cluster"
65+
# tag::install-hdfs[]
66+
kubectl apply -f hdfs.yaml
67+
# end::install-hdfs[]
68+
69+
sleep 5
70+
71+
echo "Awaiting HDFS rollout finish"
72+
# tag::watch-hdfs-rollout[]
73+
kubectl rollout status --watch statefulset/simple-hdfs-datanode-default
74+
kubectl rollout status --watch statefulset/simple-hdfs-namenode-default
75+
kubectl rollout status --watch statefulset/simple-hdfs-journalnode-default
76+
# end::watch-hdfs-rollout[]
77+
78+
echo "Creating Helper"
79+
# tag::install-webhdfs[]
80+
kubectl apply -f webhdfs.yaml
81+
# end::install-webhdfs[]
82+
83+
sleep 5
84+
85+
echo "Awaiting helper rollout finish"
86+
# tag::watch-helper-rollout[]
87+
kubectl rollout status --watch statefulset/webhdfs
88+
# end::watch-helper-rollout[]
89+
90+
file_status() {
91+
# tag::file-status[]
92+
kubectl exec -n default webhdfs-0 -- curl -s -XGET "http://simple-hdfs-namenode-default-0:9870/webhdfs/v1/?op=LISTSTATUS"
93+
# end::file-status[]
94+
}
95+
96+
echo "Confirm that HDFS is empty..."
97+
status=$(file_status | jq -r '.FileStatuses.FileStatus')
98+
99+
if [ "$status" == "[]" ]; then
100+
echo "As expected, HDFS is empty"
101+
else
102+
echo "Detected status: $status"
103+
exit 1
104+
fi
105+
106+
echo "Copy test file"
107+
# tag::copy-file[]
108+
kubectl cp -n default ./testdata.txt webhdfs-0:/tmp
109+
# end::copy-file[]
110+
111+
create_file() {
112+
# tag::create-file[]
113+
kubectl exec -n default webhdfs-0 -- \
114+
curl -s -XPUT -T /tmp/testdata.txt "http://simple-hdfs-namenode-default-0:9870/webhdfs/v1/testdata.txt?user.name=stackable&op=CREATE&noredirect=true"
115+
# end::create-file[]
116+
}
117+
118+
location=$(create_file | jq -r '.Location')
119+
120+
echo "Redirect location: $location"
121+
122+
# tag::create-file-redirected[]
123+
kubectl exec -n default webhdfs-0 -- curl -s -XPUT -T /tmp/testdata.txt "$location"
124+
# end::create-file-redirected[]
125+
126+
echo "Confirm that HDFS is *not* empty..."
127+
found_file=$(file_status | jq -r '.FileStatuses.FileStatus[0].pathSuffix')
128+
echo "Created file: $found_file with status $(file_status)"
129+
130+
echo "Delete file"
131+
delete_file() {
132+
# tag::delete-file[]
133+
kubectl exec -n default webhdfs-0 -- curl -s -XDELETE "http://simple-hdfs-namenode-default-0:9870/webhdfs/v1/testdata.txt?user.name=stackable&op=DELETE"
134+
# end::delete-file[]
135+
}
136+
137+
deleted=$(delete_file | jq -r '.boolean')
138+
139+
if [ "$deleted" == "true" ]; then
140+
echo "File was deleted!"
141+
else
142+
echo "Detected status: $deleted"
143+
exit 1
144+
fi
145+

‎docs/modules/getting_started/examples/code/getting_started.sh.j2

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# This script contains all the code snippets from the guide, as well as some assert tests
5+
# to test if the instructions in the guide work. The user *could* use it, but it is intended
6+
# for testing only.
7+
# The script will install the operators, create a superset instance and briefly open a port
8+
# forward and connect to the superset instance to make sure it is up and running.
9+
# No running processes are left behind (i.e. the port-forwarding is closed at the end)
10+
11+
if [ $# -eq 0 ]
12+
then
13+
echo "Installation method argument ('helm' or 'stackablectl') required."
14+
exit 1
15+
fi
16+
17+
case "$1" in
18+
"helm")
19+
echo "Adding 'stackable-dev' Helm Chart repository"
20+
# tag::helm-add-repo[]
21+
helm repo add stackable-dev https://repo.stackable.tech/repository/helm-dev/
22+
# end::helm-add-repo[]
23+
echo "Installing Operators with Helm"
24+
# tag::helm-install-operators[]
25+
helm install --wait zookeeper-operator stackable-dev/zookeeper-operator --version {{ versions.zookeeper }}
26+
helm install --wait hdfs-operator stackable-dev/hdfs-operator --version {{ versions.hdfs }}
27+
helm install --wait commons-operator stackable-dev/commons-operator --version {{ versions.commons }}
28+
helm install --wait secret-operator stackable-dev/secret-operator --version {{ versions.secret }}
29+
# end::helm-install-operators[]
30+
;;
31+
"stackablectl")
32+
echo "installing Operators with stackablectl"
33+
# tag::stackablectl-install-operators[]
34+
stackablectl operator install \
35+
commons={{ versions.commons }} \
36+
secret={{ versions.secret }} \
37+
zookeeper={{ versions.zookeeper }} \
38+
hdfs={{ versions.hdfs }}
39+
# end::stackablectl-install-operators[]
40+
;;
41+
*)
42+
echo "Need to give 'helm' or 'stackablectl' as an argument for which installation method to use!"
43+
exit 1
44+
;;
45+
esac
46+
47+
echo "Creating Zookeeper cluster"
48+
# tag::install-zk[]
49+
kubectl apply -f zk.yaml
50+
# end::install-zk[]
51+
52+
echo "Creating ZNode"
53+
# tag::install-zk[]
54+
kubectl apply -f znode.yaml
55+
# end::install-zk[]
56+
57+
sleep 5
58+
59+
echo "Awaiting Zookeeper rollout finish"
60+
# tag::watch-zk-rollout[]
61+
kubectl rollout status --watch statefulset/simple-zk-server-default
62+
# end::watch-zk-rollout[]
63+
64+
echo "Creating HDFS cluster"
65+
# tag::install-hdfs[]
66+
kubectl apply -f hdfs.yaml
67+
# end::install-hdfs[]
68+
69+
sleep 5
70+
71+
echo "Awaiting HDFS rollout finish"
72+
# tag::watch-hdfs-rollout[]
73+
kubectl rollout status --watch statefulset/simple-hdfs-datanode-default
74+
kubectl rollout status --watch statefulset/simple-hdfs-namenode-default
75+
kubectl rollout status --watch statefulset/simple-hdfs-journalnode-default
76+
# end::watch-hdfs-rollout[]
77+
78+
echo "Creating Helper"
79+
# tag::install-webhdfs[]
80+
kubectl apply -f webhdfs.yaml
81+
# end::install-webhdfs[]
82+
83+
sleep 5
84+
85+
echo "Awaiting helper rollout finish"
86+
# tag::watch-helper-rollout[]
87+
kubectl rollout status --watch statefulset/webhdfs
88+
# end::watch-helper-rollout[]
89+
90+
file_status() {
91+
# tag::file-status[]
92+
kubectl exec -n default webhdfs-0 -- curl -s -XGET "http://simple-hdfs-namenode-default-0:9870/webhdfs/v1/?op=LISTSTATUS"
93+
# end::file-status[]
94+
}
95+
96+
echo "Confirm that HDFS is empty..."
97+
status=$(file_status | jq -r '.FileStatuses.FileStatus')
98+
99+
if [ "$status" == "[]" ]; then
100+
echo "As expected, HDFS is empty"
101+
else
102+
echo "Detected status: $status"
103+
exit 1
104+
fi
105+
106+
echo "Copy test file"
107+
# tag::copy-file[]
108+
kubectl cp -n default ./testdata.txt webhdfs-0:/tmp
109+
# end::copy-file[]
110+
111+
create_file() {
112+
# tag::create-file[]
113+
kubectl exec -n default webhdfs-0 -- \
114+
curl -s -XPUT -T /tmp/testdata.txt "http://simple-hdfs-namenode-default-0:9870/webhdfs/v1/testdata.txt?user.name=stackable&op=CREATE&noredirect=true"
115+
# end::create-file[]
116+
}
117+
118+
location=$(create_file | jq -r '.Location')
119+
120+
echo "Redirect location: $location"
121+
122+
# tag::create-file-redirected[]
123+
kubectl exec -n default webhdfs-0 -- curl -s -XPUT -T /tmp/testdata.txt "$location"
124+
# end::create-file-redirected[]
125+
126+
echo "Confirm that HDFS is *not* empty..."
127+
found_file=$(file_status | jq -r '.FileStatuses.FileStatus[0].pathSuffix')
128+
echo "Created file: $found_file with status $(file_status)"
129+
130+
echo "Delete file"
131+
delete_file() {
132+
# tag::delete-file[]
133+
kubectl exec -n default webhdfs-0 -- curl -s -XDELETE "http://simple-hdfs-namenode-default-0:9870/webhdfs/v1/testdata.txt?user.name=stackable&op=DELETE"
134+
# end::delete-file[]
135+
}
136+
137+
deleted=$(delete_file | jq -r '.boolean')
138+
139+
if [ "$deleted" == "true" ]; then
140+
echo "File was deleted!"
141+
else
142+
echo "Detected status: $deleted"
143+
exit 1
144+
fi
145+

0 commit comments

Comments
 (0)