Merge pull request #21 from ncabatoff/feature-queries_from_file

Feature queries from file

Author: wrouesnel

README.md

@@ -65,6 +65,10 @@ for l in StringIO(x):
 Adjust the value of the resultant prometheus value type appropriately. This helps build
 rich self-documenting metrics for the exporter.
 
+### Adding new metrics via a config file
+
+The -extend.query-path command-line argument specifies a YAML file containing additional queries to run.
+Some examples are provided in [queries.yaml](queries.yaml).
 
 ### Running as non-superuser
 

postgres_exporter.go

@@ -4,12 +4,14 @@ import (
 	"database/sql"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"math"
 	"net/http"
 	"os"
+	"strconv"
 	"time"
 
-	"strconv"
+	"gopkg.in/yaml.v2"
 
 	_ "github.com/lib/pq"
 	"github.com/prometheus/client_golang/prometheus"
@@ -27,6 +29,14 @@ var (
 		"web.telemetry-path", "/metrics",
 		"Path under which to expose metrics.",
 	)
+	queriesPath = flag.String(
+		"extend.query-path", "",
+		"Path to custom queries to run.",
+	)
+	onlyDumpMaps = flag.Bool(
+		"dumpmaps", false,
+		"Do not run, simply dump the maps.",
+	)
 )
 
 // Metric name parts.
@@ -102,6 +112,21 @@ var variableMaps = map[string]map[string]ColumnMapping{
 	},
 }
 
+func dumpMaps() {
+	for name, cmap := range metricMaps {
+		query, ok := queryOverrides[name]
+		if ok {
+			fmt.Printf("%s: %s\n", name, query)
+		} else {
+			fmt.Println(name)
+		}
+		for column, details := range cmap {
+			fmt.Printf("  %-40s %v\n", column, details)
+		}
+		fmt.Println()
+	}
+}
+
 var metricMaps = map[string]map[string]ColumnMapping{
 	"pg_stat_bgwriter": map[string]ColumnMapping{
 		"checkpoints_timed":     {COUNTER, "Number of scheduled checkpoints that have been performed", nil},
@@ -223,6 +248,68 @@ var queryOverrides = map[string]string{
       ON tmp.state = tmp2.state AND pg_database.datname = tmp2.datname`,
 }
 
+// Add queries to the metricMaps and queryOverrides maps
+func addQueries(queriesPath string) (err error) {
+	var extra map[string]interface{}
+
+	content, err := ioutil.ReadFile(queriesPath)
+	if err != nil {
+		return err
+	}
+
+	err = yaml.Unmarshal(content, &extra)
+	if err != nil {
+		return err
+	}
+
+	for metric, specs := range extra {
+		for key, value := range specs.(map[interface{}]interface{}) {
+			switch key.(string) {
+			case "query":
+				query := value.(string)
+				queryOverrides[metric] = query
+
+			case "metrics":
+				for _, c := range value.([]interface{}) {
+					column := c.(map[interface{}]interface{})
+
+					for n, a := range column {
+						var cmap ColumnMapping
+
+						metric_map, ok := metricMaps[metric]
+						if !ok {
+							metric_map = make(map[string]ColumnMapping)
+						}
+
+						name := n.(string)
+
+						for attr_key, attr_val := range a.(map[interface{}]interface{}) {
+							switch attr_key.(string) {
+							case "usage":
+								usage, err := stringToColumnUsage(attr_val.(string))
+								if err != nil {
+									return err
+								}
+								cmap.usage = usage
+							case "description":
+								cmap.description = attr_val.(string)
+							}
+						}
+
+						cmap.mapping = nil
+
+						metric_map[name] = cmap
+
+						metricMaps[metric] = metric_map
+					}
+				}
+			}
+		}
+	}
+
+	return
+}
+
 // Turn the MetricMap column mapping into a prometheus descriptor mapping.
 func makeDescMap(metricMaps map[string]map[string]ColumnMapping) map[string]MetricMapNamespace {
 	var metricMap = make(map[string]MetricMapNamespace)
@@ -317,6 +404,34 @@ func makeDescMap(metricMaps map[string]map[string]ColumnMapping) map[string]Metr
 	return metricMap
 }
 
+// convert a string to the corresponding ColumnUsage
+func stringToColumnUsage(s string) (u ColumnUsage, err error) {
+	switch s {
+	case "DISCARD":
+		u = DISCARD
+
+	case "LABEL":
+		u = LABEL
+
+	case "COUNTER":
+		u = COUNTER
+
+	case "GAUGE":
+		u = GAUGE
+
+	case "MAPPEDMETRIC":
+		u = MAPPEDMETRIC
+
+	case "DURATION":
+		u = DURATION
+
+	default:
+		err = fmt.Errorf("wrong ColumnUsage given : %s", s)
+	}
+
+	return
+}
+
 // Convert database.sql types to float64s for Prometheus consumption. Null types are mapped to NaN. string and []byte
 // types are mapped as NaN and !ok
 func dbToFloat64(t interface{}) (float64, bool) {
@@ -590,6 +705,18 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
 func main() {
 	flag.Parse()
 
+	if *queriesPath != "" {
+		err := addQueries(*queriesPath)
+		if err != nil {
+			log.Warnln("Unparseable queries file - discarding merge: ", *queriesPath, err)
+		}
+	}
+
+	if *onlyDumpMaps {
+		dumpMaps()
+		return
+	}
+
 	dsn := os.Getenv("DATA_SOURCE_NAME")
 	if len(dsn) == 0 {
 		log.Fatal("couldn't find environment variable DATA_SOURCE_NAME")

queries.yaml

@@ -0,0 +1,100 @@
+pg_replication:
+  query: "SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::INT as lag"
+  metrics:
+    - lag:
+        usage: "GAUGE"
+        description: "Replication lag behind master in seconds"
+
+pg_postmaster:
+  query: "SELECT pg_postmaster_start_time as start_time_seconds from pg_postmaster_start_time()"
+  metrics:
+    - start_time_seconds:
+        usage: "GAUGE"
+        description: "Time at which postmaster started"
+
+pg_settings_shared_buffers:
+  query: "SELECT 8192*setting::int as bytes from pg_settings where name = 'shared_buffers'"
+  metrics:
+    - bytes:
+        usage: "GAUGE"
+        description: "Size of shared_buffers"
+
+pg_settings_checkpoint:
+  query: "select (select setting::int from pg_settings where name = 'checkpoint_segments') as segments, (select setting::int from pg_settings where name = 'checkpoint_timeout') as timeout_seconds, (select setting::float from pg_settings where name = 'checkpoint_completion_target') as completion_target"
+  metrics:
+    - segments:
+        usage: "GAUGE"
+        description: "Number of checkpoint segments"
+    - timeout_seconds:
+        usage: "GAUGE"
+        description: "Checkpoint timeout in seconds"
+    - completion_target:
+        usage: "GAUGE"
+        description: "Checkpoint completion target, ranging from 0 to 1"
+
+pg_stat_user_tables:
+  query: "SELECT schemaname, relname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze, last_vacuum, last_autovacuum, last_analyze, last_autoanalyze, vacuum_count, autovacuum_count, analyze_count, autoanalyze_count FROM pg_stat_user_tables"
+  metrics:
+    - schemaname:
+        usage: "LABEL"
+        description: "Name of the schema that this table is in"
+    - relname:
+        usage: "LABEL"
+        description: "Name of this table"
+    - seq_scan:
+        usage: "COUNTER"
+        description: "Number of sequential scans initiated on this table"
+    - seq_tup_read:
+        usage: "COUNTER"
+        description: "Number of live rows fetched by sequential scans"
+    - idx_scan:
+        usage: "COUNTER"
+        description: "Number of index scans initiated on this table"
+    - idx_tup_fetch:
+        usage: "COUNTER"
+        description: "Number of live rows fetched by index scans"
+    - n_tup_ins:
+        usage: "COUNTER"
+        description: "Number of rows inserted"
+    - n_tup_upd:
+        usage: "COUNTER"
+        description: "Number of rows updated"
+    - n_tup_del:
+        usage: "COUNTER"
+        description: "Number of rows deleted"
+    - n_tup_hot_upd:
+        usage: "COUNTER"
+        description: "Number of rows HOT updated (i.e., with no separate index update required)"
+    - n_live_tup:
+        usage: "GAUGE"
+        description: "Estimated number of live rows"
+    - n_dead_tup:
+        usage: "GAUGE"
+        description: "Estimated number of dead rows"
+    - n_mod_since_analyze:
+        usage: "GAUGE"
+        description: "Estimated number of rows changed since last analyze"
+    - last_vacuum:
+        usage: "GAUGE"
+        description: "Last time at which this table was manually vacuumed (not counting VACUUM FULL)"
+    - last_autovacuum:
+        usage: "GAUGE"
+        description: "Last time at which this table was vacuumed by the autovacuum daemon"
+    - last_analyze:
+        usage: "GAUGE"
+        description: "Last time at which this table was manually analyzed"
+    - last_autoanalyze:
+        usage: "GAUGE"
+        description: "Last time at which this table was analyzed by the autovacuum daemon"
+    - vacuum_count:
+        usage: "COUNTER"
+        description: "Number of times this table has been manually vacuumed (not counting VACUUM FULL)"
+    - autovacuum_count:
+        usage: "COUNTER"
+        description: "Number of times this table has been vacuumed by the autovacuum daemon"
+    - analyze_count:
+        usage: "COUNTER"
+        description: "Number of times this table has been manually analyzed"
+    - autoanalyze_count:
+        usage: "COUNTER"
+        description: "Number of times this table has been analyzed by the autovacuum daemon"