Skip to content

Commit d49127d

Browse files
wrouesnelwrouesnel
committed
Re-enabled the YAML file format processing.
1 parent 2297eb5 commit d49127d

File tree

2 files changed

+145
-81
lines changed

2 files changed

+145
-81
lines changed

‎.travis.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,13 @@ services:
44
language: go
55
go:
66
- '1.7'
7-
# Make sure we have p2
7+
# Make sure we have p2 and the postgres client.
88
before_install:
99
- sudo wget -O /usr/local/bin/p2 https://github.com/wrouesnel/p2cli/releases/download/r4/p2 &&
1010
sudo chmod +x /usr/local/bin/p2
1111
- sudo wget -O /usr/local/bin/docker-compose https://github.com/docker/compose/releases/download/1.9.0-rc4/docker-compose-Linux-x86_64 &&
1212
sudo chmod +x /usr/local/bin/docker-compose
13+
- sudo apt-get update && apt-get install postgresql-client-common
1314

1415
script:
1516
- make all

‎postgres_exporter.go

+143-80
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"database/sql"
55
"flag"
66
"fmt"
7-
//"io/ioutil"
7+
"io/ioutil"
88
"math"
99
"net/http"
1010
"os"
@@ -13,7 +13,7 @@ import (
1313
"regexp"
1414
"errors"
1515

16-
//"gopkg.in/yaml.v2"
16+
"gopkg.in/yaml.v2"
1717

1818
_ "github.com/lib/pq"
1919
"github.com/prometheus/client_golang/prometheus"
@@ -41,10 +41,6 @@ var (
4141
"dumpmaps", false,
4242
"Do not run, simply dump the maps.",
4343
)
44-
expectReplicationStats = flag.Bool(
45-
"config.expect-replication-stats", false,
46-
"The target database has replication configured, log missing replication stats as an error.",
47-
)
4844
)
4945

5046
// Metric name parts.
@@ -71,6 +67,22 @@ var landingPage = []byte(`<html>
7167

7268
type ColumnUsage int
7369

70+
// Implements the yaml.Unmarshaller interface
71+
func (this *ColumnUsage) UnmarshalYAML(unmarshal func(interface{}) error) error {
72+
var value string
73+
if err := unmarshal(&value); err != nil {
74+
return err
75+
}
76+
77+
columnUsage, err := stringToColumnUsage(value)
78+
if err != nil {
79+
return err
80+
}
81+
82+
*this = columnUsage
83+
return nil
84+
}
85+
7486
const (
7587
DISCARD ColumnUsage = iota // Ignore this column
7688
LABEL ColumnUsage = iota // Use this column as a label
@@ -103,10 +115,18 @@ func parseVersion(versionString string) (semver.Version, error) {
103115

104116
// User-friendly representation of a prometheus descriptor map
105117
type ColumnMapping struct {
106-
usage ColumnUsage
107-
description string
108-
mapping map[string]float64 // Optional column mapping for MAPPEDMETRIC
109-
supportedVersions semver.Range // Semantic version ranges which are supported. Unsupported columns are not queried (internally converted to DISCARD).
118+
usage ColumnUsage `yaml:"usage"`
119+
description string `yaml:"description"`
120+
mapping map[string]float64 `yaml:"metric_mapping"` // Optional column mapping for MAPPEDMETRIC
121+
supportedVersions semver.Range `yaml:"pg_version"` // Semantic version ranges which are supported. Unsupported columns are not queried (internally converted to DISCARD).
122+
}
123+
124+
func (this *ColumnMapping) UnmarshalYAML(unmarshal func(interface{}) error) error {
125+
type plain ColumnMapping
126+
if err := unmarshal((*plain)(this)); err != nil {
127+
return err
128+
}
129+
return nil
110130
}
111131

112132
// Groups metric maps under a shared set of labels
@@ -364,67 +384,110 @@ func makeQueryOverrideMap(pgVersion semver.Version, queryOverrides map[string][]
364384
return resultMap
365385
}
366386

367-
// Add queries to the metricMaps and queryOverrides maps
368-
//func addQueries(queriesPath string) (err error) {
369-
// var extra map[string]interface{}
370-
//
371-
// content, err := ioutil.ReadFile(queriesPath)
372-
// if err != nil {
373-
// return err
374-
// }
375-
//
376-
// err = yaml.Unmarshal(content, &extra)
377-
// if err != nil {
378-
// return err
379-
// }
380-
//
381-
// for metric, specs := range extra {
382-
// for key, value := range specs.(map[interface{}]interface{}) {
383-
// switch key.(string) {
384-
// case "query":
385-
// query := value.(string)
386-
// queryOverrides[metric] = query
387+
// Add queries to the metricMaps and queryOverrides maps. Added queries do not
388+
// respect version requirements, because it is assumed that the user knows
389+
// what they are doing with their version of postgres.
387390
//
388-
// case "metrics":
389-
// for _, c := range value.([]interface{}) {
390-
// column := c.(map[interface{}]interface{})
391-
//
392-
// for n, a := range column {
393-
// var cmap ColumnMapping
394-
//
395-
// metric_map, ok := metricMaps[metric]
396-
// if !ok {
397-
// metric_map = make(map[string]ColumnMapping)
398-
// }
399-
//
400-
// name := n.(string)
401-
//
402-
// for attr_key, attr_val := range a.(map[interface{}]interface{}) {
403-
// switch attr_key.(string) {
404-
// case "usage":
405-
// usage, err := stringToColumnUsage(attr_val.(string))
406-
// if err != nil {
407-
// return err
408-
// }
409-
// cmap.usage = usage
410-
// case "description":
411-
// cmap.description = attr_val.(string)
412-
// }
413-
// }
414-
//
415-
// cmap.mapping = nil
416-
//
417-
// metric_map[name] = cmap
418-
//
419-
// metricMaps[metric] = metric_map
420-
// }
421-
// }
422-
// }
423-
// }
424-
// }
425-
//
426-
// return
427-
//}
391+
// This function modifies metricMap and queryOverrideMap to contain the new
392+
// queries.
393+
// TODO: test code for all this.
394+
// TODO: use proper struct type system
395+
// TODO: the YAML this supports is "non-standard" - we should move away from it.
396+
func addQueries(queriesPath string, pgVersion semver.Version, exporterMap map[string]MetricMapNamespace, queryOverrideMap map[string]string) error {
397+
var extra map[string]interface{}
398+
399+
content, err := ioutil.ReadFile(queriesPath)
400+
if err != nil {
401+
return err
402+
}
403+
404+
err = yaml.Unmarshal(content, &extra)
405+
if err != nil {
406+
return err
407+
}
408+
409+
// Stores the loaded map representation
410+
metricMaps := make(map[string]map[string]ColumnMapping)
411+
newQueryOverrides := make(map[string]string)
412+
413+
for metric, specs := range extra {
414+
log.Debugln("New user metric namespace from YAML:", metric)
415+
for key, value := range specs.(map[interface{}]interface{}) {
416+
switch key.(string) {
417+
case "query":
418+
query := value.(string)
419+
newQueryOverrides[metric] = query
420+
421+
case "metrics":
422+
for _, c := range value.([]interface{}) {
423+
column := c.(map[interface{}]interface{})
424+
425+
for n, a := range column {
426+
var columnMapping ColumnMapping
427+
428+
// Fetch the metric map we want to work on.
429+
metricMap, ok := metricMaps[metric]
430+
if !ok {
431+
// Namespace for metric not found - add it.
432+
metricMap = make(map[string]ColumnMapping)
433+
metricMaps[metric] = metricMap
434+
}
435+
436+
// Get name.
437+
name := n.(string)
438+
439+
for attrKey, attrVal := range a.(map[interface{}]interface{}) {
440+
switch attrKey.(string) {
441+
case "usage":
442+
usage, err := stringToColumnUsage(attrVal.(string))
443+
if err != nil {
444+
return err
445+
}
446+
columnMapping.usage = usage
447+
case "description":
448+
columnMapping.description = attrVal.(string)
449+
}
450+
}
451+
452+
// TODO: we should support this
453+
columnMapping.mapping = nil
454+
// Should we support this for users?
455+
columnMapping.supportedVersions = nil
456+
457+
metricMap[name] = columnMapping
458+
}
459+
}
460+
}
461+
}
462+
}
463+
464+
// Convert the loaded metric map into exporter representation
465+
partialExporterMap := makeDescMap(pgVersion, metricMaps)
466+
467+
// Merge the two maps (which are now quite flatteend)
468+
for k, v := range partialExporterMap {
469+
_, found := exporterMap[k]
470+
if found {
471+
log.Debugln("Overriding metric", k, "from user YAML file.")
472+
} else {
473+
log.Debugln("Adding new metric", k, "from user YAML file.")
474+
}
475+
exporterMap[k] = v
476+
}
477+
478+
// Merge the query override map
479+
for k, v := range newQueryOverrides {
480+
_, found := queryOverrideMap[k]
481+
if found {
482+
log.Debugln("Overriding query override", k, "from user YAML file.")
483+
} else {
484+
log.Debugln("Adding new query override", k, "from user YAML file.")
485+
}
486+
queryOverrideMap[k] = v
487+
}
488+
489+
return nil
490+
}
428491

429492
// Turn the MetricMap column mapping into a prometheus descriptor mapping.
430493
func makeDescMap(pgVersion semver.Version, metricMaps map[string]map[string]ColumnMapping) map[string]MetricMapNamespace {
@@ -619,6 +682,7 @@ func dbToString(t interface{}) (string, bool) {
619682
// Exporter collects Postgres metrics. It implements prometheus.Collector.
620683
type Exporter struct {
621684
dsn string
685+
userQueriesPath string
622686
duration, error prometheus.Gauge
623687
totalScrapes prometheus.Counter
624688

@@ -635,9 +699,10 @@ type Exporter struct {
635699
}
636700

637701
// NewExporter returns a new PostgreSQL exporter for the provided DSN.
638-
func NewExporter(dsn string) *Exporter {
702+
func NewExporter(dsn string, userQueriesPath string) *Exporter {
639703
return &Exporter{
640704
dsn: dsn,
705+
userQueriesPath: userQueriesPath,
641706
duration: prometheus.NewGauge(prometheus.GaugeOpts{
642707
Namespace: namespace,
643708
Subsystem: exporter,
@@ -881,6 +946,12 @@ func (e *Exporter) checkMapVersions(ch chan<- prometheus.Metric, db *sql.DB) err
881946
e.queryOverrides = makeQueryOverrideMap(semanticVersion, queryOverrides)
882947
e.lastMapVersion = semanticVersion
883948

949+
if e.userQueriesPath != "" {
950+
if err := addQueries(e.userQueriesPath, semanticVersion, e.metricMap, e.queryOverrides) ; err != nil {
951+
log.Errorln("Failed to reload user queries:", e.userQueriesPath, err)
952+
}
953+
}
954+
884955
e.mappingMtx.Unlock()
885956
}
886957

@@ -933,14 +1004,6 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
9331004
func main() {
9341005
flag.Parse()
9351006

936-
// TODO: restroe addQueries functionality
937-
//if *queriesPath != "" {
938-
// err := addQueries(*queriesPath)
939-
// if err != nil {
940-
// log.Warnln("Unparseable queries file - discarding merge: ", *queriesPath, err)
941-
// }
942-
//}
943-
9441007
if *onlyDumpMaps {
9451008
dumpMaps()
9461009
return
@@ -951,7 +1014,7 @@ func main() {
9511014
log.Fatal("couldn't find environment variable DATA_SOURCE_NAME")
9521015
}
9531016

954-
exporter := NewExporter(dsn)
1017+
exporter := NewExporter(dsn, *queriesPath)
9551018
prometheus.MustRegister(exporter)
9561019

9571020
http.Handle(*metricPath, prometheus.Handler())

0 commit comments

Comments
 (0)