Skip to content
This repository was archived by the owner on Feb 16, 2024. It is now read-only.

Commit c7d68c5

Browse files
siegfriedwebersiegfriedweber
committed
Logging (#187)
## Description - Show opensearch-dashboards service with `stackablectl services list`. - Add logging demo.
1 parent 7fcf40f commit c7d68c5

File tree

11 files changed

+471
-3
lines changed

11 files changed

+471
-3
lines changed

‎CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
## [Unreleased]
44

5+
### Added
6+
7+
- Support listing OpenSearch Dashboards services ([#187](https://github.com/stackabletech/stackablectl/pull/187)).
8+
59
### Changed
610

711
- Print CLI tables using [comfy-table](https://crates.io/crates/comfy-table) ([#176](https://github.com/stackabletech/stackablectl/pull/176))

‎demos/demos-v1.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,3 +105,15 @@ demos:
105105
- plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml
106106
- plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml
107107
- plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml
108+
logging:
109+
description: Demo showing the logging stack in action
110+
documentation: https://docs.stackable.tech/stackablectl/stable/demos/logging.html
111+
stackableStack: logging
112+
labels:
113+
- logging
114+
- opensearch
115+
- opensearch-dashboards
116+
- vector
117+
- zookeeper
118+
manifests:
119+
- plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/logging/zookeeper.yaml

‎demos/logging/zookeeper.yaml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
---
2+
apiVersion: zookeeper.stackable.tech/v1alpha1
3+
kind: ZookeeperCluster
4+
metadata:
5+
name: simple-zk
6+
spec:
7+
image:
8+
productVersion: 3.8.0
9+
stackableVersion: 0.9.0
10+
vectorAggregatorConfigMapName: vector-aggregator-discovery
11+
servers:
12+
roleGroups:
13+
default:
14+
replicas: 3
15+
config:
16+
logging:
17+
enableVectorAgent: true
18+
containers:
19+
vector:
20+
file:
21+
level: WARN
22+
zookeeper:
23+
console:
24+
level: INFO
25+
file:
26+
level: INFO
27+
loggers:
28+
ROOT:
29+
level: INFO
30+
org.apache.zookeeper.server.NettyServerCnxn:
31+
level: NONE

‎docs/modules/ROOT/images/logging/login.png

98.8 KB
Loading

‎docs/modules/ROOT/images/logging/logs.png

197 KB
Loading

‎docs/modules/ROOT/images/logging/overview.png

500 KB
Loading

‎docs/modules/ROOT/nav.adoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
** xref:demos/airflow-scheduled-job.adoc[]
1111
** xref:demos/data-lakehouse-iceberg-trino-spark.adoc[]
1212
** xref:demos/hbase-hdfs-load-cycling-data.adoc[]
13+
** xref:demos/logging.adoc[]
1314
** xref:demos/nifi-kafka-druid-earthquake-data.adoc[]
1415
** xref:demos/nifi-kafka-druid-water-level-data.adoc[]
1516
** xref:demos/spark-k8s-anomaly-detection-taxi-data.adoc[]

‎docs/modules/ROOT/pages/demos/logging.adoc

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
= logging
2+
3+
This demo will
4+
5+
* Install the required Stackable operators.
6+
* Spin up the follow data products:
7+
** *Apache ZooKeeper*: A centralized service for maintaining configuration
8+
information, naming, providing distributed synchronization, and providing group
9+
services. This demo makes its log data observable in OpenSearch Dashboards.
10+
** *Vector*: A tool for building observability pipelines. This demo uses Vector
11+
as a log agent to gather and transform the logs and as an aggregator to forward
12+
the collected logs to OpenSearch.
13+
** *OpenSearch*: A data store and search engine. This demo uses it to store and
14+
index the log data.
15+
** *OpenSearch Dashboards*: A visualization and user interface. This demo uses
16+
it to make the log data easily accessible to the user.
17+
* Create a view in OpenSearch Dashboards for convenient browsing the log data.
18+
19+
You can see the deployed products as well as their relationship in the
20+
following diagram:
21+
22+
image::logging/overview.png[]
23+
24+
== OpenSearch prerequisites
25+
26+
=== Mac and Windows
27+
28+
If you are running on Mac or Windows and use Docker to run Kubernetes then set
29+
the RAM to at least 4 GB in _Preferences_ > _Resources_.
30+
31+
=== Linux
32+
33+
OpenSearch uses a mmapfs directory by default to store its indices. The default
34+
operating system limits on mmap counts is likely to be too low – usually 65530,
35+
which may result in out of memory exceptions. So the Linux setting
36+
`vm.max_map_count` on the host machine where kind is running, must be set to at
37+
least 262144.
38+
39+
To check the current value, run this command:
40+
41+
[source,console]
42+
----
43+
sysctl vm.max_map_count
44+
----
45+
46+
The limit can be temporarily increased with:
47+
48+
[source,console]
49+
----
50+
sudo sysctl --write vm.max_map_count=262144
51+
----
52+
53+
To permanently increase the value, add the following line to `/etc/sysctl.conf`:
54+
55+
[source,.properties]
56+
----
57+
vm.max_map_count=262144
58+
----
59+
60+
Then run `sudo sysctl --load` to reload.
61+
62+
== Run the demo
63+
64+
The following command creates a kind cluster and installs this demo:
65+
66+
[source,console]
67+
----
68+
$ stackablectl demo install logging --kind-cluster
69+
----
70+
71+
== List deployed Stackable services
72+
73+
To list the installed Stackable services run the following command:
74+
75+
[source,console]
76+
----
77+
$ stackablectl services list
78+
┌───────────────────────┬───────────────────────┬───────────┬──────────────────────────────┬───────────────────────────────────────────────────────────────────────────────────┐
79+
│ Product ┆ Name ┆ Namespace ┆ Endpoints ┆ Info │
80+
╞═══════════════════════╪═══════════════════════╪═══════════╪══════════════════════════════╪═══════════════════════════════════════════════════════════════════════════════════╡
81+
│ opensearch-dashboards ┆ opensearch-dashboards ┆ default ┆ http http://172.18.0.5:31319 ┆ Third party service │
82+
│ ┆ ┆ ┆ ┆ Logs view: http://172.18.0.5:31319/app/discover?security_tenant=global#/view/logs │
83+
│ ┆ ┆ ┆ ┆ Username: admin, password: adminadmin │
84+
├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
85+
│ zookeeper ┆ simple-zk ┆ default ┆ zk 172.18.0.2:32417 ┆ │
86+
└───────────────────────┴───────────────────────┴───────────┴──────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────┘
87+
----
88+
89+
[NOTE]
90+
====
91+
When a product instance has not finished starting yet, the service will have no
92+
endpoint. Starting all the product instances might take a considerable amount
93+
of time depending on your internet connectivity. In case the product is not
94+
ready yet, a warning might be shown.
95+
====
96+
97+
== Inspect the log data
98+
99+
You can have a look at the log data within the OpenSearch Dashboards web
100+
interface by running `stackablectl services list` and opening the URL shown in
101+
the info column of the `opensearch-dashboards` entry. In this case it is
102+
http://172.18.0.5:31319/app/discover?security_tenant=global#/view/logs.
103+
104+
image::logging/login.png[]
105+
106+
Log in with the username `admin` and password `adminadmin`.
107+
108+
image::logging/logs.png[]
109+
110+
Inspect the logs.

‎src/services.rs

Lines changed: 77 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,10 @@ async fn list_services(
176176
"minio".to_string(),
177177
get_minio_services(!all_namespaces, redact_credentials).await?,
178178
);
179+
output.insert(
180+
"opensearch-dashboards".to_string(),
181+
get_opensearch_dashboards_services(!all_namespaces, redact_credentials).await?,
182+
);
179183

180184
match output_type {
181185
OutputType::Text => {
@@ -188,7 +192,7 @@ async fn list_services(
188192
Cell::new("Name"),
189193
Cell::new("Namespace"),
190194
Cell::new("Endpoints"),
191-
Cell::new("Extra infos"),
195+
Cell::new("Info"),
192196
]);
193197

194198
let max_endpoint_name_length = output
@@ -216,7 +220,7 @@ async fn list_services(
216220
.join("\n");
217221

218222
table.add_row(vec![
219-
Cell::new(&product_name),
223+
Cell::new(product_name.as_str()),
220224
Cell::new(installed_product.name),
221225
Cell::new(installed_product.namespace.unwrap_or_default()),
222226
Cell::new(endpoints),
@@ -551,3 +555,74 @@ pub async fn get_minio_service(
551555
extra_infos,
552556
})
553557
}
558+
559+
async fn get_opensearch_dashboards_services(
560+
namespaced: bool,
561+
redact_credentials: bool,
562+
) -> Result<Vec<InstalledProduct>, Box<dyn Error>> {
563+
let client = get_client().await?;
564+
let list_params = ListParams::default().labels("app.kubernetes.io/name=opensearch-dashboards");
565+
566+
let mut result = Vec::new();
567+
568+
let service_api: Api<Service> = if namespaced {
569+
Api::namespaced(client.to_owned(), NAMESPACE.lock()?.as_str())
570+
} else {
571+
Api::all(client.clone())
572+
};
573+
574+
let services = service_api.list(&list_params).await?;
575+
for service in services {
576+
let namespace = service.namespace().unwrap();
577+
let endpoints =
578+
get_service_endpoint_urls(&service, &service.name_unchecked(), client.to_owned())
579+
.await?;
580+
581+
let annotations = service.annotations();
582+
583+
let mut extra_infos = vec!["Third party service".to_string()];
584+
585+
if let Some(http_endpoint) = endpoints.get("http") {
586+
if let Some(logs_endpoint) = annotations.get("stackable.tech/logging-view-logs") {
587+
extra_infos.push(format!("Logs view: {http_endpoint}{logs_endpoint}"));
588+
}
589+
}
590+
591+
if let Some(credentials_secret) =
592+
annotations.get("stackable.tech/logging-credentials-secret")
593+
{
594+
let secret_api: Api<Secret> = Api::namespaced(client.clone(), &namespace);
595+
if let Ok(credentials) = secret_api.get(credentials_secret).await {
596+
if let Some(username) = credentials
597+
.data
598+
.as_ref()
599+
.and_then(|data| data.get("username"))
600+
.and_then(|value| String::from_utf8(value.0.to_owned()).ok())
601+
{
602+
let mut credentials_info = format!("Username: {username}");
603+
if redact_credentials {
604+
credentials_info.push_str(&format!(", password: {REDACTED_PASSWORD}"));
605+
} else if let Some(password) = credentials
606+
.data
607+
.as_ref()
608+
.and_then(|data| data.get("password"))
609+
.and_then(|value| String::from_utf8(value.0.to_owned()).ok())
610+
{
611+
credentials_info.push_str(&format!(", password: {password}"));
612+
}
613+
extra_infos.push(credentials_info);
614+
}
615+
}
616+
}
617+
618+
let installed_product = InstalledProduct {
619+
name: service.name_unchecked(),
620+
namespace: service.namespace(),
621+
endpoints,
622+
extra_infos,
623+
};
624+
result.push(installed_product);
625+
}
626+
627+
Ok(result)
628+
}

‎stacks/logging/vector-aggregator-discovery.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: vector-aggregator-discovery
5+
data:
6+
ADDRESS: vector-aggregator:6000

0 commit comments

Comments
 (0)