Skip to content

Commit e3ba4b3

Browse files
authored
Merge pull request #33 from wrouesnel/bugfixes_and_upgrades
Bugfixes and upgrades
2 parents 9143f59 + 98d26cb commit e3ba4b3

File tree

219 files changed

+117577
-288
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

219 files changed

+117577
-288
lines changed

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
.build
22
postgres_exporter
3+
postgres_exporter_integration_test
34
*.tar.gz
45
*.test
56
*-stamp

.travis.yml

+15-2
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,26 @@ services:
44
language: go
55
go:
66
- '1.7'
7+
# Make sure we have p2 and the postgres client.
8+
before_install:
9+
- sudo wget -O /usr/local/bin/p2 https://github.com/wrouesnel/p2cli/releases/download/r4/p2 &&
10+
sudo chmod +x /usr/local/bin/p2
11+
- sudo wget -O /usr/local/bin/docker-compose https://github.com/docker/compose/releases/download/1.9.0-rc4/docker-compose-Linux-x86_64 &&
12+
sudo chmod +x /usr/local/bin/docker-compose
13+
- sudo apt-get update && sudo apt-get install postgresql-client-common
14+
715
script:
816
- make all
917
- make docker
1018
- make test-integration
1119
after_success:
12-
- if [ "$TRAVIS_BRANCH" == "master" ]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER
13-
-p $DOCKER_PASS ; docker push wrouesnel/postgres_exporter ; fi
20+
- docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
21+
# Push a tagged build if a tag is found.
22+
- if [ ! -z "$TRAVIS_TAG" ]; then
23+
docker tag wrouesnel/postgres_exporter:latest wrouesnel/postgres_exporter:$TRAVIS_TAG ;
24+
fi
25+
# Push a latest version
26+
- if [ "$TRAVIS_BRANCH" == "master" ]; then docker push wrouesnel/postgres_exporter ; fi
1427
env:
1528
global:
1629
- secure: RfoWQj5tEB/t3XL2tqJW7u7Qscpz1QBOfF9lMFpB4kAUMTtZU0zBbXfMo1JheGoJQQxD/7NLRHhbUWPT2489o3KKpRTQ7RHn3k8n5U7opH01bWX0+l/EPVmhlsKjSDSLGgmxz80j3I6C8ZV3qDUijSx7r90QUNHGbZtV7g+KtoUTpRV0zir/heK6qq9LHWNHbNsJyHK8qHmd6g1UzWIBaZPJ6a/n/rO2jq4uS1JR0VlIJPRF11HOLH8IjFQvVYpN7YbEslxyNsfQJUSP/7CghSLLVWPSATEjMm8a5GJVLc564+nYghm484psEtiMXkZ3n6ie7AT8aJrKfexWrwh2aCc+cK4PiyXrf4euZehZNYogmFCqWzd1LJKcN2uIkpBSuZQDm3e6c4qkkWGpx+RdFWtAMG8IgZLDbcuryxFNzMwHc2CJ009s9Zsa+g7D57csyR5LCZ8YtNGI3g8FmhwpCKvYkfKa9aijUEWyJMyT4Vhd/w7btMTuwYHgUQ85k4ov4Xjz5SNpAGgemig5G5w7PJj4NhGvIBz9weL154x/BFVjHOZZ6Y/bWgJIPoW1KM15x5K8QylWYEBUHtwiyVyXOxHqt6MOX1vYo1L37jMK88IErrfh/VmlxEhtN9wOghk8IudMfFwQtjIwiWlJf218wxMIzUjoyb5/25tU9f2OJrg=

Makefile

+7-3
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@ all: vet test postgres_exporter
77

88
# Simple go build
99
postgres_exporter: $(GO_SRC)
10-
CGO_ENABLED=0 go build -a -ldflags "-extldflags '-static' -X main.Version=git:$(shell git rev-parse HEAD)" -o postgres_exporter .
10+
CGO_ENABLED=0 go build -a -ldflags "-extldflags '-static' -X main.Version=$(shell git describe --dirty)" -o postgres_exporter .
11+
12+
postgres_exporter_integration_test: $(GO_SRC)
13+
CGO_ENABLED=0 go test -c -tags integration \
14+
-a -ldflags "-extldflags '-static' -X main.Version=git:$(shell git describe --dirty)" -o postgres_exporter_integration_test .
1115

1216
# Take a go build and turn it into a minimal container
1317
docker: postgres_exporter
@@ -19,8 +23,8 @@ vet:
1923
test:
2024
go test -v .
2125

22-
test-integration:
23-
tests/test-smoke
26+
test-integration: postgres_exporter postgres_exporter_integration_test
27+
tests/test-smoke ./postgres_exporter ./postgres_exporter_integration_test
2428

2529
# Do a self-contained docker build - we pull the official upstream container
2630
# and do a self-contained build.

README.md

+9-4
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,15 @@ Package vendoring is handled with [`govendor`](https://github.com/kardianos/gove
3434

3535
### Flags
3636

37-
Name | Description
38-
-------------------|------------
39-
web.listen-address | Address to listen on for web interface and telemetry.
40-
web.telemetry-path | Path under which to expose metrics.
37+
* `web.listen-address`
38+
Address to listen on for web interface and telemetry.
39+
40+
* `web.telemetry-path`
41+
Path under which to expose metrics.
42+
43+
* `config.expect-replication-stats`
44+
The target database has replication turned on - log errors when
45+
replication stats are missing.
4146

4247
### Setting the Postgres server's data source name
4348

postgres_exporter.go

+537-244
Large diffs are not rendered by default.

postgres_exporter_integration_test.go

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// These are specialized integration tests. We only build them when we're doing
2+
// a lot of additional work to keep the external docker environment they require
3+
// working.
4+
// +build integration
5+
6+
package main
7+
8+
import (
9+
"os"
10+
"testing"
11+
12+
. "gopkg.in/check.v1"
13+
14+
"github.com/prometheus/client_golang/prometheus"
15+
"database/sql"
16+
_ "github.com/lib/pq"
17+
"fmt"
18+
)
19+
20+
// Hook up gocheck into the "go test" runner.
21+
func Test(t *testing.T) { TestingT(t) }
22+
23+
type IntegrationSuite struct{
24+
e *Exporter
25+
}
26+
27+
var _ = Suite(&IntegrationSuite{})
28+
29+
func (s *IntegrationSuite) SetUpSuite(c *C) {
30+
dsn := os.Getenv("DATA_SOURCE_NAME")
31+
c.Assert(dsn, Not(Equals), "")
32+
33+
exporter := NewExporter(dsn, "")
34+
c.Assert(exporter, NotNil)
35+
// Assign the exporter to the suite
36+
s.e = exporter
37+
38+
prometheus.MustRegister(exporter)
39+
}
40+
41+
// TODO: it would be nice if this didn't mostly just recreate the scrape function
42+
func (s *IntegrationSuite) TestAllNamespacesReturnResults(c *C) {
43+
// Setup a dummy channel to consume metrics
44+
ch := make(chan prometheus.Metric, 100)
45+
go func() {
46+
for _ = range ch {}
47+
}()
48+
49+
// Open a database connection
50+
db, err := sql.Open("postgres", s.e.dsn)
51+
c.Assert(db, NotNil)
52+
c.Assert(err, IsNil)
53+
defer db.Close()
54+
55+
// Do a version update
56+
err = s.e.checkMapVersions(ch, db)
57+
c.Assert(err, IsNil)
58+
59+
// Check the show variables work
60+
nonFatalErrors := queryShowVariables(ch, db, s.e.variableMap)
61+
if !c.Check(len(nonFatalErrors), Equals, 0) {
62+
fmt.Println("## NONFATAL ERRORS FOUND")
63+
for _, err := range nonFatalErrors {
64+
fmt.Println(err)
65+
}
66+
}
67+
68+
69+
// This should never happen in our test cases.
70+
errMap := queryNamespaceMappings(ch, db, s.e.metricMap, s.e.queryOverrides)
71+
if !c.Check(len(errMap), Equals, 0) {
72+
fmt.Println("## NAMESPACE ERRORS FOUND")
73+
for namespace, err := range errMap {
74+
fmt.Println(namespace, ":", err)
75+
}
76+
}
77+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM postgres:9.6
2+
MAINTAINER Daniel Dent (https://www.danieldent.com)
3+
ENV PG_MAX_WAL_SENDERS 8
4+
ENV PG_WAL_KEEP_SEGMENTS 8
5+
COPY setup-replication.sh /docker-entrypoint-initdb.d/
6+
COPY docker-entrypoint.sh /docker-entrypoint.sh
7+
RUN chmod +x /docker-entrypoint-initdb.d/setup-replication.sh /docker-entrypoint.sh
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM postgres:{{VERSION}}
2+
MAINTAINER Daniel Dent (https://www.danieldent.com)
3+
ENV PG_MAX_WAL_SENDERS 8
4+
ENV PG_WAL_KEEP_SEGMENTS 8
5+
COPY setup-replication.sh /docker-entrypoint-initdb.d/
6+
COPY docker-entrypoint.sh /docker-entrypoint.sh
7+
RUN chmod +x /docker-entrypoint-initdb.d/setup-replication.sh /docker-entrypoint.sh
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Replicated postgres cluster in docker.
2+
3+
Upstream is forked from https://github.com/DanielDent/docker-postgres-replication
4+
5+
My version lives at https://github.com/wrouesnel/docker-postgres-replication
6+
7+
This very simple docker-compose file lets us stand up a replicated postgres
8+
cluster so we can test streaming.
9+
10+
# TODO:
11+
Pull in p2 and template the Dockerfile so we can test multiple versions.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
2+
version: '2'
3+
4+
services:
5+
pg-master:
6+
build: '.'
7+
image: 'danieldent/postgres-replication'
8+
restart: 'always'
9+
environment:
10+
POSTGRES_USER: 'postgres'
11+
POSTGRES_PASSWORD: 'postgres'
12+
PGDATA: '/var/lib/postgresql/data/pgdata'
13+
volumes:
14+
- '/var/lib/postgresql/data'
15+
expose:
16+
- '5432'
17+
18+
pg-slave:
19+
build: '.'
20+
image: 'danieldent/postgres-replication'
21+
restart: 'always'
22+
environment:
23+
POSTGRES_USER: 'postgres'
24+
POSTGRES_PASSWORD: 'postgres'
25+
PGDATA: '/var/lib/postgresql/data/pgdata'
26+
REPLICATE_FROM: 'pg-master'
27+
volumes:
28+
- '/var/lib/postgresql/data'
29+
expose:
30+
- '5432'
31+
links:
32+
- 'pg-master'
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
#!/bin/bash
2+
3+
# Backwards compatibility for old variable names (deprecated)
4+
if [ "x$PGUSER" != "x" ]; then
5+
POSTGRES_USER=$PGUSER
6+
fi
7+
if [ "x$PGPASSWORD" != "x" ]; then
8+
POSTGRES_PASSWORD=$PGPASSWORD
9+
fi
10+
11+
# Forwards-compatibility for old variable names (pg_basebackup uses them)
12+
if [ "x$PGPASSWORD" = "x" ]; then
13+
export PGPASSWORD=$POSTGRES_PASSWORD
14+
fi
15+
16+
# Based on official postgres package's entrypoint script (https://hub.docker.com/_/postgres/)
17+
# Modified to be able to set up a slave. The docker-entrypoint-initdb.d hook provided is inadequate.
18+
19+
set -e
20+
21+
if [ "${1:0:1}" = '-' ]; then
22+
set -- postgres "$@"
23+
fi
24+
25+
if [ "$1" = 'postgres' ]; then
26+
mkdir -p "$PGDATA"
27+
chmod 700 "$PGDATA"
28+
chown -R postgres "$PGDATA"
29+
30+
mkdir -p /run/postgresql
31+
chmod g+s /run/postgresql
32+
chown -R postgres /run/postgresql
33+
34+
# look specifically for PG_VERSION, as it is expected in the DB dir
35+
if [ ! -s "$PGDATA/PG_VERSION" ]; then
36+
if [ "x$REPLICATE_FROM" == "x" ]; then
37+
eval "gosu postgres initdb $POSTGRES_INITDB_ARGS"
38+
else
39+
until ping -c 1 -W 1 ${REPLICATE_FROM}
40+
do
41+
echo "Waiting for master to ping..."
42+
sleep 1s
43+
done
44+
until gosu postgres pg_basebackup -h ${REPLICATE_FROM} -D ${PGDATA} -U ${POSTGRES_USER} -vP -w
45+
do
46+
echo "Waiting for master to connect..."
47+
sleep 1s
48+
done
49+
fi
50+
51+
# check password first so we can output the warning before postgres
52+
# messes it up
53+
if [ ! -z "$POSTGRES_PASSWORD" ]; then
54+
pass="PASSWORD '$POSTGRES_PASSWORD'"
55+
authMethod=md5
56+
else
57+
# The - option suppresses leading tabs but *not* spaces. :)
58+
cat >&2 <<-'EOWARN'
59+
****************************************************
60+
WARNING: No password has been set for the database.
61+
This will allow anyone with access to the
62+
Postgres port to access your database. In
63+
Docker's default configuration, this is
64+
effectively any other container on the same
65+
system.
66+
67+
Use "-e POSTGRES_PASSWORD=password" to set
68+
it in "docker run".
69+
****************************************************
70+
EOWARN
71+
72+
pass=
73+
authMethod=trust
74+
fi
75+
76+
if [ "x$REPLICATE_FROM" == "x" ]; then
77+
78+
{ echo; echo "host replication all 0.0.0.0/0 $authMethod"; } | gosu postgres tee -a "$PGDATA/pg_hba.conf" > /dev/null
79+
{ echo; echo "host all all 0.0.0.0/0 $authMethod"; } | gosu postgres tee -a "$PGDATA/pg_hba.conf" > /dev/null
80+
81+
# internal start of server in order to allow set-up using psql-client
82+
# does not listen on external TCP/IP and waits until start finishes
83+
gosu postgres pg_ctl -D "$PGDATA" \
84+
-o "-c listen_addresses='localhost'" \
85+
-w start
86+
87+
: ${POSTGRES_USER:=postgres}
88+
: ${POSTGRES_DB:=$POSTGRES_USER}
89+
export POSTGRES_USER POSTGRES_DB
90+
91+
psql=( psql -v ON_ERROR_STOP=1 )
92+
93+
if [ "$POSTGRES_DB" != 'postgres' ]; then
94+
"${psql[@]}" --username postgres <<-EOSQL
95+
CREATE DATABASE "$POSTGRES_DB" ;
96+
EOSQL
97+
echo
98+
fi
99+
100+
if [ "$POSTGRES_USER" = 'postgres' ]; then
101+
op='ALTER'
102+
else
103+
op='CREATE'
104+
fi
105+
"${psql[@]}" --username postgres <<-EOSQL
106+
$op USER "$POSTGRES_USER" WITH SUPERUSER $pass ;
107+
EOSQL
108+
echo
109+
110+
fi
111+
112+
psql+=( --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" )
113+
114+
echo
115+
for f in /docker-entrypoint-initdb.d/*; do
116+
case "$f" in
117+
*.sh) echo "$0: running $f"; . "$f" ;;
118+
*.sql) echo "$0: running $f"; "${psql[@]}" < "$f"; echo ;;
119+
*.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;;
120+
*) echo "$0: ignoring $f" ;;
121+
esac
122+
echo
123+
done
124+
125+
if [ "x$REPLICATE_FROM" == "x" ]; then
126+
gosu postgres pg_ctl -D "$PGDATA" -m fast -w stop
127+
fi
128+
129+
echo
130+
echo 'PostgreSQL init process complete; ready for start up.'
131+
echo
132+
fi
133+
134+
# We need this health check so we know when it's started up.
135+
touch /tmp/.postgres_init_complete
136+
137+
exec gosu postgres "$@"
138+
fi
139+
140+
exec "$@"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
3+
if [ "x$REPLICATE_FROM" == "x" ]; then
4+
5+
cat >> ${PGDATA}/postgresql.conf <<EOF
6+
wal_level = hot_standby
7+
max_wal_senders = $PG_MAX_WAL_SENDERS
8+
wal_keep_segments = $PG_WAL_KEEP_SEGMENTS
9+
hot_standby = on
10+
EOF
11+
12+
else
13+
14+
cat > ${PGDATA}/recovery.conf <<EOF
15+
standby_mode = on
16+
primary_conninfo = 'host=${REPLICATE_FROM} port=5432 user=${POSTGRES_USER} password=${POSTGRES_PASSWORD}'
17+
trigger_file = '/tmp/touch_me_to_promote_to_me_master'
18+
EOF
19+
chown postgres ${PGDATA}/recovery.conf
20+
chmod 600 ${PGDATA}/recovery.conf
21+
22+
fi

0 commit comments

Comments
 (0)