From eefc9dd37843ea93ceb64cf279048696febb2f8e Mon Sep 17 00:00:00 2001 From: Cyanty Date: Sun, 20 Apr 2025 02:48:09 +0800 Subject: [PATCH 1/6] [improve] Add jdbc common collect e2e code --- .../collect/database/JdbcCommonCollect.java | 2 + .../hertzbeat-collector-basic-e2e/pom.xml | 41 +++++++++- .../basic/database/DatabaseImagesEnum.java | 52 ++++++++++++ .../database/JdbcCommonCollectE2eTest.java | 79 +++++++++++++++++++ hertzbeat-e2e/pom.xml | 12 +++ 5 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java create mode 100644 hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java index a607f3a13f2..2ff5cc6ea38 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/database/JdbcCommonCollect.java @@ -383,6 +383,8 @@ private String constructDatabaseUrl(JdbcProtocol jdbcProtocol, String host, Stri case "oracle" -> "jdbc:oracle:thin:@" + host + ":" + port + "/" + (jdbcProtocol.getDatabase() == null ? "" : jdbcProtocol.getDatabase()); case "dm" -> "jdbc:dm://" + host + ":" + port; + case "testcontainers" -> "jdbc:tc:" + host + ":" + port + + ":///" + (jdbcProtocol.getDatabase() == null ? "" : jdbcProtocol.getDatabase()) + "?user=root&password=root"; default -> throw new IllegalArgumentException("Not support database platform: " + jdbcProtocol.getPlatform()); }; } diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml index 79740608210..36711e3d81f 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml @@ -31,6 +31,8 @@ 17 17 UTF-8 + 8.0.30 + 42.5.5 @@ -53,10 +55,47 @@ ${hertzbeat.version} test + + + + org.testcontainers + junit-jupiter + test + + + + org.testcontainers + jdbc + test + + + + org.testcontainers + mysql + test + org.testcontainers - testcontainers + postgresql + test + + + + + mysql + mysql-connector-java + ${mysql.version} test + true + + org.postgresql + postgresql + ${postgresql.version} + test + true + + + \ No newline at end of file diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java new file mode 100644 index 00000000000..f63d9d5329a --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java @@ -0,0 +1,52 @@ +package org.apache.hertzbeat.collector.collect.basic.database; + +public enum DatabaseImagesEnum { + MYSQL("mysql", "8.0.36"), + POSTGRESQL("postgresql", "15"); + + private final String imageName; + private final String defaultTag; + + DatabaseImagesEnum(String imageName, String defaultTag) { + this.imageName = imageName; + this.defaultTag = defaultTag; + } + + public String getImageName() { + return imageName; + } + + public String getDefaultTag() { + return defaultTag; + } + + public String getFullImageName() { + return imageName + ":" + defaultTag; + } + + public static DatabaseImagesEnum fromImageName(String imageName) { + for (DatabaseImagesEnum value : values()) { + if (value.getImageName().equalsIgnoreCase(imageName)) { + return value; + } + } + throw new IllegalArgumentException("Unknown database image name: " + imageName); + } + + public static DatabaseImagesEnum fromFullImageName(String fullImageName) { + for (DatabaseImagesEnum value : values()) { + if (value.getFullImageName().equalsIgnoreCase(fullImageName)) { + return value; + } + } + throw new IllegalArgumentException("Unknown full database image name: " + fullImageName); + } + + @Override + public String toString() { + return "DatabaseImageNameEnum{" + + "imageName='" + imageName + '\'' + + ", defaultTag='" + defaultTag + '\'' + + '}'; + } +} diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java new file mode 100644 index 00000000000..81c2a6f5650 --- /dev/null +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java @@ -0,0 +1,79 @@ +package org.apache.hertzbeat.collector.collect.basic.database; + +import lombok.extern.slf4j.Slf4j; +import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest; +import org.apache.hertzbeat.collector.collect.database.JdbcCommonCollect; +import org.apache.hertzbeat.collector.util.CollectUtil; +import org.apache.hertzbeat.common.entity.job.Configmap; +import org.apache.hertzbeat.common.entity.job.Job; +import org.apache.hertzbeat.common.entity.job.Metrics; +import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol; +import org.apache.hertzbeat.common.entity.job.protocol.Protocol; +import org.apache.hertzbeat.common.entity.message.CollectRep; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +@Slf4j +@ExtendWith(MockitoExtension.class) +public class JdbcCommonCollectE2eTest extends AbstractCollectE2eTest { + private static final DatabaseImagesEnum databaseImage = DatabaseImagesEnum.fromImageName("postgresql"); + + private static final String DATABASE_IMAGE_NAME = databaseImage.getImageName(); + + private static final String DATABASE_IMAGE_TAG = databaseImage.getDefaultTag(); + + @BeforeEach + public void setUp() throws Exception { + super.setUp(); + collect = new JdbcCommonCollect(); + metrics = new Metrics(); + } + + @Override + protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) { + JdbcProtocol jdbcProtocol = (JdbcProtocol) buildProtocol(metricsDef); + metrics.setJdbc(jdbcProtocol); + CollectRep.MetricsData.Builder metricsData = CollectRep.MetricsData.newBuilder(); + metricsData.setApp(DATABASE_IMAGE_NAME); + metrics.setAliasFields(metricsDef.getAliasFields()); + return collectMetricsData(metrics, metricsDef, metricsData); + } + + @Override + protected Protocol buildProtocol(Metrics metricsDef) { + JdbcProtocol jdbcProtocol = metricsDef.getJdbc(); + jdbcProtocol.setHost(DATABASE_IMAGE_NAME); + jdbcProtocol.setPort(DATABASE_IMAGE_TAG); + jdbcProtocol.setDatabase("test"); + jdbcProtocol.setPlatform("testcontainers"); + return jdbcProtocol; + } + + @Test + public void testWithJdbcTcUrl() { + Job dockerJob = appService.getAppDefine(DATABASE_IMAGE_NAME); + List> configmapFromPreCollectData = new LinkedList<>(); + for (Metrics metricsDef : dockerJob.getMetrics()) { + metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef, configmapFromPreCollectData.size() > 0 ? configmapFromPreCollectData.get(0) : new HashMap<>()); + String metricName = metricsDef.getName(); + if ("slow_sql".equals(metricName)) { + Metrics finalMetricsDef = metricsDef; + assertDoesNotThrow(() -> collectMetrics(finalMetricsDef), + String.format("%s failed to collect metrics)", metricName)); + log.info("{} metrics validation passed", metricName); + continue; // skip slow_sql, extra extensions + } + CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricName, true); + CollectUtil.getConfigmapFromPreCollectData(metricsData); + } + } +} diff --git a/hertzbeat-e2e/pom.xml b/hertzbeat-e2e/pom.xml index 45202344190..bdee85a6917 100644 --- a/hertzbeat-e2e/pom.xml +++ b/hertzbeat-e2e/pom.xml @@ -74,4 +74,16 @@ test + + + + + org.testcontainers + testcontainers-bom + ${testcontainers.version} + pom + import + + + \ No newline at end of file From 76f64f6b27ae52c09b034af538f53bdc1279d9bd Mon Sep 17 00:00:00 2001 From: Cyanty <153884653+Cyanty@users.noreply.github.com> Date: Mon, 21 Apr 2025 11:21:24 +0800 Subject: [PATCH 2/6] Update DatabaseImagesEnum.java Signed-off-by: Cyanty <153884653+Cyanty@users.noreply.github.com> --- .../basic/database/DatabaseImagesEnum.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java index f63d9d5329a..f4f098cf8c7 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hertzbeat.collector.collect.basic.database; public enum DatabaseImagesEnum { From 62b991d501bb4c824e07420a23f71624862cba3e Mon Sep 17 00:00:00 2001 From: Cyanty <153884653+Cyanty@users.noreply.github.com> Date: Mon, 21 Apr 2025 11:21:42 +0800 Subject: [PATCH 3/6] Update JdbcCommonCollectE2eTest.java Signed-off-by: Cyanty <153884653+Cyanty@users.noreply.github.com> --- .../database/JdbcCommonCollectE2eTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java index 81c2a6f5650..7855379af00 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hertzbeat.collector.collect.basic.database; import lombok.extern.slf4j.Slf4j; From 42aeb83cb3091ef1414d77fd72e1d542a28f157a Mon Sep 17 00:00:00 2001 From: Cyanty <153884653+Cyanty@users.noreply.github.com> Date: Mon, 21 Apr 2025 12:05:57 +0800 Subject: [PATCH 4/6] Update DatabaseImagesEnum.java Signed-off-by: Cyanty <153884653+Cyanty@users.noreply.github.com> --- .../collector/collect/basic/database/DatabaseImagesEnum.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java index f4f098cf8c7..03427d11f88 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/DatabaseImagesEnum.java @@ -17,6 +17,9 @@ package org.apache.hertzbeat.collector.collect.basic.database; +/** + * Database type (image mame) and image tag enumeration class + */ public enum DatabaseImagesEnum { MYSQL("mysql", "8.0.36"), POSTGRESQL("postgresql", "15"); From d073416a04e7c296f25c57026e740b5fcde157b7 Mon Sep 17 00:00:00 2001 From: Cyanty <153884653+Cyanty@users.noreply.github.com> Date: Mon, 21 Apr 2025 12:09:32 +0800 Subject: [PATCH 5/6] Update JdbcCommonCollectE2eTest.java Signed-off-by: Cyanty <153884653+Cyanty@users.noreply.github.com> --- .../collect/basic/database/JdbcCommonCollectE2eTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java index 7855379af00..69f8492f3cc 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/database/JdbcCommonCollectE2eTest.java @@ -39,6 +39,9 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +/** + * E2e test monitored by Jdbc Common + */ @Slf4j @ExtendWith(MockitoExtension.class) public class JdbcCommonCollectE2eTest extends AbstractCollectE2eTest { From 729e4a3958e2e107b32f1afdbb3b7be3787405f2 Mon Sep 17 00:00:00 2001 From: Cyanty <153884653+Cyanty@users.noreply.github.com> Date: Mon, 21 Apr 2025 13:10:29 +0800 Subject: [PATCH 6/6] [improve] Add jdbc common collect e2e code (#3273) Signed-off-by: Cyanty <153884653+Cyanty@users.noreply.github.com> --- hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml index 36711e3d81f..423ab0c21cb 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/pom.xml @@ -96,6 +96,5 @@ true - - \ No newline at end of file +