From d4cb7b5fa3193c20137f0bde68e18fcf0245c2cd Mon Sep 17 00:00:00 2001 From: vl4deee11 Date: Thu, 10 Jun 2021 09:46:11 +0300 Subject: [PATCH 1/5] added breaking changes feature Signed-off-by: vl4deee11 --- cmd/postgres_exporter/pg_breaking_changes.go | 34 ++++++++++++++++ cmd/postgres_exporter/postgres_exporter.go | 39 +++++++++++++++---- .../postgres_exporter_test.go | 2 +- queries.yaml | 18 ++++++++- 4 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 cmd/postgres_exporter/pg_breaking_changes.go diff --git a/cmd/postgres_exporter/pg_breaking_changes.go b/cmd/postgres_exporter/pg_breaking_changes.go new file mode 100644 index 000000000..ddd4a76aa --- /dev/null +++ b/cmd/postgres_exporter/pg_breaking_changes.go @@ -0,0 +1,34 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/blang/semver" +) + +type BreakingChanges struct { + Version string `yaml:"version"` + ver semver.Version + Columns map[string]string `yaml:"columns"` +} + +func (bc *BreakingChanges) parseVerTolerant() error { + bcVer, err := semver.ParseTolerant(bc.Version) + if err != nil { + return err + } + + bc.ver = bcVer + return nil +} diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 36c524c8e..132e423a7 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -119,11 +119,12 @@ type Mapping map[string]MappingOptions // nolint: golint type UserQuery struct { - Query string `yaml:"query"` - Metrics []Mapping `yaml:"metrics"` - Master bool `yaml:"master"` // Querying only for master database - CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. - RunOnServer string `yaml:"runonserver"` // Querying to run on which server version + Query string `yaml:"query"` + Metrics []Mapping `yaml:"metrics"` + BreakingChanges []BreakingChanges `yaml:"breakingChanges"` + Master bool `yaml:"master"` // Querying only for master database + CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. + RunOnServer string `yaml:"runonserver"` // Querying to run on which server version } // nolint: golint @@ -517,7 +518,7 @@ func makeQueryOverrideMap(pgVersion semver.Version, queryOverrides map[string][] return resultMap } -func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[string]string, error) { +func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]intermediateMetricMap, map[string]string, error) { var userQueries UserQueries err := yaml.Unmarshal(content, &userQueries) @@ -532,6 +533,30 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str for metric, specs := range userQueries { level.Debug(logger).Log("msg", "New user metric namespace from YAML metric", "metric", metric, "cache_seconds", specs.CacheSeconds) newQueryOverrides[metric] = specs.Query + columnT := make(map[string]string) + for i := range specs.BreakingChanges { + if err := specs.BreakingChanges[i].parseVerTolerant(); err != nil { + return nil, nil, err + } + + if pgVersion.GE(specs.BreakingChanges[i].ver) { + for t := range specs.BreakingChanges[i].Columns { + columnT[t] = specs.BreakingChanges[i].Columns[t] + } + } + } + + // nolint: golint + // 2 because old - new + oldnew := make([]string, 0, 2*len(columnT)) + for t := range columnT { + oldnew = append(oldnew, t, columnT[t]) + } + + r := strings.NewReplacer(oldnew...) + + newQueryOverrides[metric] = r.Replace(newQueryOverrides[metric]) + metricMap, ok := metricMaps[metric] if !ok { // Namespace for metric not found - add it. @@ -571,7 +596,7 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str // TODO: test code for all cu. // TODO: the YAML this supports is "non-standard" - we should move away from it. func addQueries(content []byte, pgVersion semver.Version, server *Server) error { - metricMaps, newQueryOverrides, err := parseUserQueries(content) + metricMaps, newQueryOverrides, err := parseUserQueries(content, pgVersion) if err != nil { return err } diff --git a/cmd/postgres_exporter/postgres_exporter_test.go b/cmd/postgres_exporter/postgres_exporter_test.go index 5747f0b07..5645a8ee8 100644 --- a/cmd/postgres_exporter/postgres_exporter_test.go +++ b/cmd/postgres_exporter/postgres_exporter_test.go @@ -410,7 +410,7 @@ func (s *FunctionalSuite) TestBooleanConversionToValueAndString(c *C) { func (s *FunctionalSuite) TestParseUserQueries(c *C) { userQueriesData, err := ioutil.ReadFile("./tests/user_queries_ok.yaml") if err == nil { - metricMaps, newQueryOverrides, err := parseUserQueries(userQueriesData) + metricMaps, newQueryOverrides, err := parseUserQueries(userQueriesData, semver.Version{Major: 13}) c.Assert(err, Equals, nil) c.Assert(metricMaps, NotNil) c.Assert(newQueryOverrides, NotNil) diff --git a/queries.yaml b/queries.yaml index 35b754319..f4f49e609 100644 --- a/queries.yaml +++ b/queries.yaml @@ -159,8 +159,24 @@ pg_database: description: "Disk space used by the database" pg_stat_statements: - query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time / 1000 as total_time_seconds, min_time / 1000 as min_time_seconds, max_time / 1000 as max_time_seconds, mean_time / 1000 as mean_time_seconds, stddev_time / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" + query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time_T / 1000 as total_time_seconds, min_time_T / 1000 as min_time_seconds, max_time_T / 1000 as max_time_seconds, mean_time_T / 1000 as mean_time_seconds, stddev_time_T / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" master: true + breakingChanges: + # should be in asc order by version + - version: '9.4.0' + columns: + total_time_T: total_time + min_time_T: min_time + max_time_T: max_time + mean_time_T: mean_time + stddev_time_T: stddev_time + - version: '13.0.0' + columns: + total_time_T: total_exec_time + min_time_T: min_exec_time + max_time_T: max_exec_time + mean_time_T: mean_exec_time + stddev_time_T: stddev_exec_time metrics: - rolname: usage: "LABEL" From 0af7736d008e47d57fd7fab20d69d9c7ff522861 Mon Sep 17 00:00:00 2001 From: vl4deee11 Date: Thu, 10 Jun 2021 12:10:46 +0300 Subject: [PATCH 2/5] fixed Signed-off-by: vl4deee11 --- .../{pg_breaking_changes.go => pg_version.go} | 12 +++--- cmd/postgres_exporter/postgres_exporter.go | 38 +++++++------------ queries.yaml | 20 ++-------- 3 files changed, 22 insertions(+), 48 deletions(-) rename cmd/postgres_exporter/{pg_breaking_changes.go => pg_version.go} (79%) diff --git a/cmd/postgres_exporter/pg_breaking_changes.go b/cmd/postgres_exporter/pg_version.go similarity index 79% rename from cmd/postgres_exporter/pg_breaking_changes.go rename to cmd/postgres_exporter/pg_version.go index ddd4a76aa..4ab665858 100644 --- a/cmd/postgres_exporter/pg_breaking_changes.go +++ b/cmd/postgres_exporter/pg_version.go @@ -10,25 +10,23 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package main import ( "github.com/blang/semver" ) -type BreakingChanges struct { +type VersionQueries struct { Version string `yaml:"version"` ver semver.Version - Columns map[string]string `yaml:"columns"` + Query string `yaml:"query"` } -func (bc *BreakingChanges) parseVerTolerant() error { - bcVer, err := semver.ParseTolerant(bc.Version) +func (v *VersionQueries) parseVerTolerant() error { + bcVer, err := semver.ParseTolerant(v.Version) if err != nil { return err } - - bc.ver = bcVer + v.ver = bcVer return nil } diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 132e423a7..bcce124d2 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -119,12 +119,12 @@ type Mapping map[string]MappingOptions // nolint: golint type UserQuery struct { - Query string `yaml:"query"` - Metrics []Mapping `yaml:"metrics"` - BreakingChanges []BreakingChanges `yaml:"breakingChanges"` - Master bool `yaml:"master"` // Querying only for master database - CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. - RunOnServer string `yaml:"runonserver"` // Querying to run on which server version + Query string `yaml:"query"` + Metrics []Mapping `yaml:"metrics"` + VersionQueries []VersionQueries `yaml:"versionQueries"` + Master bool `yaml:"master"` // Querying only for master database + CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. + RunOnServer string `yaml:"runonserver"` // Querying to run on which server version } // nolint: golint @@ -533,30 +533,18 @@ func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]inte for metric, specs := range userQueries { level.Debug(logger).Log("msg", "New user metric namespace from YAML metric", "metric", metric, "cache_seconds", specs.CacheSeconds) newQueryOverrides[metric] = specs.Query - columnT := make(map[string]string) - for i := range specs.BreakingChanges { - if err := specs.BreakingChanges[i].parseVerTolerant(); err != nil { + if len(specs.VersionQueries) == 0 { + continue + } + for i := range specs.VersionQueries { + if err := specs.VersionQueries[i].parseVerTolerant(); err != nil { return nil, nil, err } - - if pgVersion.GE(specs.BreakingChanges[i].ver) { - for t := range specs.BreakingChanges[i].Columns { - columnT[t] = specs.BreakingChanges[i].Columns[t] - } + if pgVersion.GE(specs.VersionQueries[i].ver) { + newQueryOverrides[metric] = specs.VersionQueries[i].Query } } - // nolint: golint - // 2 because old - new - oldnew := make([]string, 0, 2*len(columnT)) - for t := range columnT { - oldnew = append(oldnew, t, columnT[t]) - } - - r := strings.NewReplacer(oldnew...) - - newQueryOverrides[metric] = r.Replace(newQueryOverrides[metric]) - metricMap, ok := metricMaps[metric] if !ok { // Namespace for metric not found - add it. diff --git a/queries.yaml b/queries.yaml index f4f49e609..91762f8e8 100644 --- a/queries.yaml +++ b/queries.yaml @@ -159,24 +159,12 @@ pg_database: description: "Disk space used by the database" pg_stat_statements: - query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time_T / 1000 as total_time_seconds, min_time_T / 1000 as min_time_seconds, max_time_T / 1000 as max_time_seconds, mean_time_T / 1000 as mean_time_seconds, stddev_time_T / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" - master: true - breakingChanges: + versionQueries: # should be in asc order by version - - version: '9.4.0' - columns: - total_time_T: total_time - min_time_T: min_time - max_time_T: max_time - mean_time_T: mean_time - stddev_time_T: stddev_time - version: '13.0.0' - columns: - total_time_T: total_exec_time - min_time_T: min_exec_time - max_time_T: max_exec_time - mean_time_T: mean_exec_time - stddev_time_T: stddev_exec_time + query: "SELECT t2.rolname, t3.datname, queryid, calls, total_exec_time / 1000 as total_time_seconds, min_exec_time / 1000 as min_time_seconds, max_exec_time / 1000 as max_time_seconds, mean_exec_time / 1000 as mean_time_seconds, stddev_exec_time / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" + query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time_T / 1000 as total_time_seconds, min_time_T / 1000 as min_time_seconds, max_time_T / 1000 as max_time_seconds, mean_time_T / 1000 as mean_time_seconds, stddev_time_T / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" + master: true metrics: - rolname: usage: "LABEL" From 8bf43dc9674989fd8d8ae1a123c6f3e15161621f Mon Sep 17 00:00:00 2001 From: vl4deee11 Date: Thu, 10 Jun 2021 12:12:07 +0300 Subject: [PATCH 3/5] rename Signed-off-by: vl4deee11 --- cmd/postgres_exporter/pg_version.go | 4 ++-- cmd/postgres_exporter/postgres_exporter.go | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/postgres_exporter/pg_version.go b/cmd/postgres_exporter/pg_version.go index 4ab665858..4cc38c504 100644 --- a/cmd/postgres_exporter/pg_version.go +++ b/cmd/postgres_exporter/pg_version.go @@ -16,13 +16,13 @@ import ( "github.com/blang/semver" ) -type VersionQueries struct { +type VersionQ struct { Version string `yaml:"version"` ver semver.Version Query string `yaml:"query"` } -func (v *VersionQueries) parseVerTolerant() error { +func (v *VersionQ) parseVerTolerant() error { bcVer, err := semver.ParseTolerant(v.Version) if err != nil { return err diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index bcce124d2..aff4b74aa 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -119,12 +119,12 @@ type Mapping map[string]MappingOptions // nolint: golint type UserQuery struct { - Query string `yaml:"query"` - Metrics []Mapping `yaml:"metrics"` - VersionQueries []VersionQueries `yaml:"versionQueries"` - Master bool `yaml:"master"` // Querying only for master database - CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. - RunOnServer string `yaml:"runonserver"` // Querying to run on which server version + Query string `yaml:"query"` + Metrics []Mapping `yaml:"metrics"` + VersionQueries []VersionQ `yaml:"versionQueries"` + Master bool `yaml:"master"` // Querying only for master database + CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. + RunOnServer string `yaml:"runonserver"` // Querying to run on which server version } // nolint: golint From 2c14a46e4c84298cc4acd67177381e052a5f515a Mon Sep 17 00:00:00 2001 From: vl4deee11 Date: Thu, 10 Jun 2021 12:18:41 +0300 Subject: [PATCH 4/5] fix query Signed-off-by: vl4deee11 --- queries.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries.yaml b/queries.yaml index 91762f8e8..b233c64fe 100644 --- a/queries.yaml +++ b/queries.yaml @@ -163,7 +163,7 @@ pg_stat_statements: # should be in asc order by version - version: '13.0.0' query: "SELECT t2.rolname, t3.datname, queryid, calls, total_exec_time / 1000 as total_time_seconds, min_exec_time / 1000 as min_time_seconds, max_exec_time / 1000 as max_time_seconds, mean_exec_time / 1000 as mean_time_seconds, stddev_exec_time / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" - query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time_T / 1000 as total_time_seconds, min_time_T / 1000 as min_time_seconds, max_time_T / 1000 as max_time_seconds, mean_time_T / 1000 as mean_time_seconds, stddev_time_T / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" + query: "SELECT t2.rolname, t3.datname, queryid, calls, total_time / 1000 as total_time_seconds, min_time / 1000 as min_time_seconds, max_time / 1000 as max_time_seconds, mean_time / 1000 as mean_time_seconds, stddev_time / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin'" master: true metrics: - rolname: From 3dafce02b3695afdffb3b4d9342281ca3d363556 Mon Sep 17 00:00:00 2001 From: vl4deee11 Date: Thu, 10 Jun 2021 12:25:17 +0300 Subject: [PATCH 5/5] fix cond Signed-off-by: vl4deee11 --- cmd/postgres_exporter/postgres_exporter.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index aff4b74aa..8ec367cc2 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -533,15 +533,14 @@ func parseUserQueries(content []byte, pgVersion semver.Version) (map[string]inte for metric, specs := range userQueries { level.Debug(logger).Log("msg", "New user metric namespace from YAML metric", "metric", metric, "cache_seconds", specs.CacheSeconds) newQueryOverrides[metric] = specs.Query - if len(specs.VersionQueries) == 0 { - continue - } - for i := range specs.VersionQueries { - if err := specs.VersionQueries[i].parseVerTolerant(); err != nil { - return nil, nil, err - } - if pgVersion.GE(specs.VersionQueries[i].ver) { - newQueryOverrides[metric] = specs.VersionQueries[i].Query + if len(specs.VersionQueries) != 0 { + for i := range specs.VersionQueries { + if err := specs.VersionQueries[i].parseVerTolerant(); err != nil { + return nil, nil, err + } + if pgVersion.GE(specs.VersionQueries[i].ver) { + newQueryOverrides[metric] = specs.VersionQueries[i].Query + } } }