Skip to content

Commit 303c47f

Browse files
rnovikovPrnovikovP
authored
Merge pull request #23 from percona/PMM-2019-Innodb-Compression-statistics
PMM-2019 Innodb Compression statistics
2 parents 6833d94 + 947400d commit 303c47f

5 files changed

+340
-0
lines changed

‎collector/info_schema_innodb_cmp.go

+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// Scrape `information_schema.client_statistics`.
2+
3+
package collector
4+
5+
import (
6+
"database/sql"
7+
"fmt"
8+
"strings"
9+
10+
"github.com/prometheus/client_golang/prometheus"
11+
"github.com/prometheus/common/log"
12+
)
13+
14+
const innodbCmpQuery = `
15+
SELECT
16+
page_size, compress_ops, compress_ops_ok, compress_time, uncompress_ops, uncompress_time
17+
FROM information_schema.INNODB_CMP
18+
`
19+
20+
var (
21+
// Map known innodb_cmp values to types. Unknown types will be mapped as
22+
// untyped.
23+
informationSchemaInnodbCmpTypes = map[string]struct {
24+
vtype prometheus.ValueType
25+
desc *prometheus.Desc
26+
}{
27+
"compress_ops": {prometheus.CounterValue,
28+
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmp_compress_ops_total"),
29+
"Number of times a B-tree page of the size PAGE_SIZE has been compressed.",
30+
[]string{"page_size"}, nil)},
31+
"compress_ops_ok": {prometheus.CounterValue,
32+
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmp_compress_ops_ok_total"),
33+
"Number of times a B-tree page of the size PAGE_SIZE has been successfully compressed.",
34+
[]string{"page_size"}, nil)},
35+
"compress_time": {prometheus.CounterValue,
36+
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmp_compress_time_seconds_total"),
37+
"Total time in seconds spent in attempts to compress B-tree pages.",
38+
[]string{"page_size"}, nil)},
39+
"uncompress_ops": {prometheus.CounterValue,
40+
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmp_uncompress_ops_total"),
41+
"Number of times a B-tree page has been uncompressed.",
42+
[]string{"page_size"}, nil)},
43+
"uncompress_time": {prometheus.CounterValue,
44+
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmp_uncompress_time_seconds_total"),
45+
"Total time in seconds spent in uncompressing B-tree pages.",
46+
[]string{"page_size"}, nil)},
47+
}
48+
)
49+
50+
// ScrapeInnodbCmp collects from `information_schema.innodb_cmp`.
51+
type ScrapeInnodbCmp struct{}
52+
53+
// Name of the Scraper.
54+
func (ScrapeInnodbCmp) Name() string {
55+
return "info_schema.innodb_cmp"
56+
}
57+
58+
// Help returns additional information about Scraper.
59+
func (ScrapeInnodbCmp) Help() string {
60+
return "Please set next variables SET GLOBAL innodb_file_per_table=1;SET GLOBAL innodb_file_format=Barracuda;"
61+
}
62+
63+
// Version of MySQL from which scraper is available.
64+
func (ScrapeInnodbCmp) Version() float64 {
65+
return 5.5
66+
}
67+
68+
// Scrape collects data.
69+
func (ScrapeInnodbCmp) Scrape(db *sql.DB, ch chan<- prometheus.Metric) error {
70+
informationSchemaInnodbCmpRows, err := db.Query(innodbCmpQuery)
71+
if err != nil {
72+
log.Debugln("INNODB_CMP stats are not available.")
73+
return err
74+
}
75+
defer informationSchemaInnodbCmpRows.Close()
76+
77+
// The client column is assumed to be column[0], while all other data is assumed to be coerceable to float64.
78+
// Because of the client column, clientStatData[0] maps to columnNames[1] when reading off the metrics
79+
// (because clientStatScanArgs is mapped as [ &client, &clientData[0], &clientData[1] ... &clientdata[n] ]
80+
// To map metrics to names therefore we always range over columnNames[1:]
81+
columnNames, err := informationSchemaInnodbCmpRows.Columns()
82+
if err != nil {
83+
log.Debugln("INNODB_CMP stats are not available.")
84+
return err
85+
}
86+
87+
var (
88+
client string // Holds the client name, which should be in column 0.
89+
clientStatData = make([]float64, len(columnNames)-1) // 1 less because of the client column.
90+
clientStatScanArgs = make([]interface{}, len(columnNames))
91+
)
92+
93+
clientStatScanArgs[0] = &client
94+
for i := range clientStatData {
95+
clientStatScanArgs[i+1] = &clientStatData[i]
96+
}
97+
98+
for informationSchemaInnodbCmpRows.Next() {
99+
if err := informationSchemaInnodbCmpRows.Scan(clientStatScanArgs...); err != nil {
100+
return err
101+
}
102+
103+
// Loop over column names, and match to scan data. Unknown columns
104+
// will be filled with an untyped metric number. We assume other then
105+
// cient, that we'll only get numbers.
106+
for idx, columnName := range columnNames[1:] {
107+
if metricType, ok := informationSchemaInnodbCmpTypes[columnName]; ok {
108+
ch <- prometheus.MustNewConstMetric(metricType.desc, metricType.vtype, float64(clientStatData[idx]), client)
109+
} else {
110+
// Unknown metric. Report as untyped.
111+
desc := prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, fmt.Sprintf("innodb_cmp_%s", strings.ToLower(columnName))), fmt.Sprintf("Unsupported metric from column %s", columnName), []string{"page_size"}, nil)
112+
ch <- prometheus.MustNewConstMetric(desc, prometheus.UntypedValue, float64(clientStatData[idx]), client)
113+
}
114+
}
115+
}
116+
return nil
117+
}

‎collector/info_schema_innodb_cmp_test.go

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package collector
2+
3+
import (
4+
"testing"
5+
6+
"github.com/prometheus/client_golang/prometheus"
7+
dto "github.com/prometheus/client_model/go"
8+
"github.com/smartystreets/goconvey/convey"
9+
"gopkg.in/DATA-DOG/go-sqlmock.v1"
10+
)
11+
12+
func TestScrapeInnodbCmp(t *testing.T) {
13+
db, mock, err := sqlmock.New()
14+
if err != nil {
15+
t.Fatalf("error opening a stub database connection: %s", err)
16+
}
17+
defer db.Close()
18+
19+
columns := []string{"page_size", "compress_ops", "compress_ops_ok", "compress_time", "uncompress_ops", "uncompress_time"}
20+
rows := sqlmock.NewRows(columns).
21+
AddRow("1024", 10, 20, 30, 40, 50)
22+
mock.ExpectQuery(sanitizeQuery(innodbCmpQuery)).WillReturnRows(rows)
23+
24+
ch := make(chan prometheus.Metric)
25+
go func() {
26+
if err = (ScrapeInnodbCmp{}).Scrape(db, ch); err != nil {
27+
t.Errorf("error calling function on test: %s", err)
28+
}
29+
close(ch)
30+
}()
31+
32+
expected := []MetricResult{
33+
{labels: labelMap{"page_size": "1024"}, value: 10, metricType: dto.MetricType_COUNTER},
34+
{labels: labelMap{"page_size": "1024"}, value: 20, metricType: dto.MetricType_COUNTER},
35+
{labels: labelMap{"page_size": "1024"}, value: 30, metricType: dto.MetricType_COUNTER},
36+
{labels: labelMap{"page_size": "1024"}, value: 40, metricType: dto.MetricType_COUNTER},
37+
{labels: labelMap{"page_size": "1024"}, value: 50, metricType: dto.MetricType_COUNTER},
38+
}
39+
convey.Convey("Metrics comparison", t, func() {
40+
for _, expect := range expected {
41+
got := readMetric(<-ch)
42+
convey.So(expect, convey.ShouldResemble, got)
43+
}
44+
})
45+
46+
// Ensure all SQL queries were executed
47+
if err := mock.ExpectationsWereMet(); err != nil {
48+
t.Errorf("there were unfulfilled expections: %s", err)
49+
}
50+
}

‎collector/info_schema_innodb_cmpmem.go

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
// Scrape `information_schema.innodb_cmpmem`.
2+
3+
package collector
4+
5+
import (
6+
"database/sql"
7+
"fmt"
8+
"strings"
9+
10+
"github.com/prometheus/client_golang/prometheus"
11+
"github.com/prometheus/common/log"
12+
)
13+
14+
const innodbCmpMemQuery = `
15+
SELECT
16+
page_size, buffer_pool_instance, pages_used, pages_free, relocation_ops, relocation_time
17+
FROM information_schema.INNODB_CMPMEM
18+
`
19+
20+
//Metric descriptors.
21+
var (
22+
// Map known innodb_cmp values to types. Unknown types will be mapped as
23+
// untyped.
24+
informationSchemaInnodbCmpMemTypes = map[string]struct {
25+
vtype prometheus.ValueType
26+
desc *prometheus.Desc
27+
}{
28+
"pages_used": {prometheus.CounterValue,
29+
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmpmem_pages_used_total"),
30+
"Number of blocks of the size PAGE_SIZE that are currently in use.",
31+
[]string{"page_size", "buffer"}, nil)},
32+
"pages_free": {prometheus.CounterValue,
33+
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmpmem_pages_free_total"),
34+
"Number of blocks of the size PAGE_SIZE that are currently available for allocation.",
35+
[]string{"page_size", "buffer"}, nil)},
36+
"relocation_ops": {prometheus.CounterValue,
37+
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmpmem_relocation_ops_total"),
38+
"Number of times a block of the size PAGE_SIZE has been relocated.",
39+
[]string{"page_size", "buffer"}, nil)},
40+
"relocation_time": {prometheus.CounterValue,
41+
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmpmem_relocation_time_seconds_total"),
42+
"Total time in microseconds spent in relocating blocks of the size PAGE_SIZE.",
43+
[]string{"page_size", "buffer"}, nil)},
44+
}
45+
)
46+
47+
// ScrapeInnodbCmpMem collects from `information_schema.innodb_cmpmem`.
48+
type ScrapeInnodbCmpMem struct{}
49+
50+
// Name of the Scraper.
51+
func (ScrapeInnodbCmpMem) Name() string {
52+
return "info_schema.innodb_cmpmem"
53+
}
54+
55+
// Help returns additional information about Scraper.
56+
func (ScrapeInnodbCmpMem) Help() string {
57+
return "Please set next variables SET GLOBAL innodb_file_per_table=1;SET GLOBAL innodb_file_format=Barracuda;"
58+
}
59+
60+
// Version of MySQL from which scraper is available.
61+
func (ScrapeInnodbCmpMem) Version() float64 {
62+
return 5.5
63+
}
64+
65+
// Scrape collects data.
66+
func (ScrapeInnodbCmpMem) Scrape(db *sql.DB, ch chan<- prometheus.Metric) error {
67+
informationSchemaInnodbCmpMemRows, err := db.Query(innodbCmpMemQuery)
68+
if err != nil {
69+
log.Debugln("INNODB_CMPMEM stats are not available.")
70+
return err
71+
}
72+
defer informationSchemaInnodbCmpMemRows.Close()
73+
74+
// The client column is assumed to be column[0], while all other data is assumed to be coerceable to float64.
75+
// Because of the client column, clientStatData[0] maps to columnNames[1] when reading off the metrics
76+
// (because clientStatScanArgs is mapped as [ &client, &buffer, &clientData[0], &clientData[1] ... &clientdata[n] ]
77+
// To map metrics to names therefore we always range over columnNames[1:]
78+
columnNames, err := informationSchemaInnodbCmpMemRows.Columns()
79+
80+
if err != nil {
81+
log.Debugln("INNODB_CMPMEM stats are not available.")
82+
return err
83+
}
84+
85+
var (
86+
client string // Holds the client name, which should be in column 0.
87+
buffer string // Holds the buffer number, which should be in column 1.
88+
clientStatData = make([]float64, len(columnNames)-2) // 2 less because of the client column.
89+
clientStatScanArgs = make([]interface{}, len(columnNames))
90+
)
91+
92+
clientStatScanArgs[0] = &client
93+
clientStatScanArgs[1] = &buffer
94+
for i := range clientStatData {
95+
clientStatScanArgs[i+2] = &clientStatData[i]
96+
}
97+
98+
for informationSchemaInnodbCmpMemRows.Next() {
99+
if err := informationSchemaInnodbCmpMemRows.Scan(clientStatScanArgs...); err != nil {
100+
return err
101+
}
102+
// Loop over column names, and match to scan data. Unknown columns
103+
// will be filled with an untyped metric number. We assume other then
104+
// cient, that we'll only get numbers.
105+
for idx, columnName := range columnNames[2:] {
106+
if metricType, ok := informationSchemaInnodbCmpMemTypes[columnName]; ok {
107+
if columnName == "relocation_time" {
108+
ch <- prometheus.MustNewConstMetric(metricType.desc, metricType.vtype, float64(clientStatData[idx]/1000), client, buffer)
109+
} else {
110+
ch <- prometheus.MustNewConstMetric(metricType.desc, metricType.vtype, float64(clientStatData[idx]), client, buffer)
111+
}
112+
} else {
113+
// Unknown metric. Report as untyped.
114+
desc := prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, fmt.Sprintf("innodb_cmpmem_%s", strings.ToLower(columnName))), fmt.Sprintf("Unsupported metric from column %s", columnName), []string{"page_size", "buffer"}, nil)
115+
ch <- prometheus.MustNewConstMetric(desc, prometheus.UntypedValue, float64(clientStatData[idx]), client, buffer)
116+
}
117+
}
118+
}
119+
return nil
120+
}

‎collector/info_schema_innodb_cmpmem_test.go

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package collector
2+
3+
import (
4+
"testing"
5+
6+
"github.com/prometheus/client_golang/prometheus"
7+
dto "github.com/prometheus/client_model/go"
8+
"github.com/smartystreets/goconvey/convey"
9+
"gopkg.in/DATA-DOG/go-sqlmock.v1"
10+
)
11+
12+
func TestScrapeInnodbCmpMem(t *testing.T) {
13+
db, mock, err := sqlmock.New()
14+
if err != nil {
15+
t.Fatalf("error opening a stub database connection: %s", err)
16+
}
17+
defer db.Close()
18+
19+
columns := []string{"page_size", "buffer", "pages_used", "pages_free", "relocation_ops", "relocation_time"}
20+
rows := sqlmock.NewRows(columns).
21+
AddRow("1024", "0", 30, 40, 50, 60)
22+
mock.ExpectQuery(sanitizeQuery(innodbCmpMemQuery)).WillReturnRows(rows)
23+
24+
ch := make(chan prometheus.Metric)
25+
go func() {
26+
if err = (ScrapeInnodbCmpMem{}).Scrape(db, ch); err != nil {
27+
t.Errorf("error calling function on test: %s", err)
28+
}
29+
close(ch)
30+
}()
31+
32+
expected := []MetricResult{
33+
{labels: labelMap{"page_size": "1024", "buffer": "0"}, value: 30, metricType: dto.MetricType_COUNTER},
34+
{labels: labelMap{"page_size": "1024", "buffer": "0"}, value: 40, metricType: dto.MetricType_COUNTER},
35+
{labels: labelMap{"page_size": "1024", "buffer": "0"}, value: 50, metricType: dto.MetricType_COUNTER},
36+
{labels: labelMap{"page_size": "1024", "buffer": "0"}, value: 0.06, metricType: dto.MetricType_COUNTER},
37+
}
38+
convey.Convey("Metrics comparison", t, func() {
39+
for _, expect := range expected {
40+
got := readMetric(<-ch)
41+
convey.So(expect, convey.ShouldResemble, got)
42+
}
43+
})
44+
45+
// Ensure all SQL queries were executed
46+
if err := mock.ExpectationsWereMet(); err != nil {
47+
t.Errorf("there were unfulfilled expections: %s", err)
48+
}
49+
}

‎mysqld_exporter.go

+4
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,8 @@ var scrapers = map[collector.Scraper]bool{
137137
collector.ScrapeEngineTokudbStatus{}: false,
138138
collector.ScrapeEngineInnodbStatus{}: false,
139139
collector.ScrapeHeartbeat{}: false,
140+
collector.ScrapeInnodbCmp{}: false,
141+
collector.ScrapeInnodbCmpMem{}: false,
140142
}
141143

142144
var scrapersHr = map[collector.Scraper]struct{}{
@@ -152,6 +154,8 @@ var scrapersMr = map[collector.Scraper]struct{}{
152154
collector.ScrapePerfTableLockWaits{}: {},
153155
collector.ScrapeQueryResponseTime{}: {},
154156
collector.ScrapeEngineInnodbStatus{}: {},
157+
collector.ScrapeInnodbCmp{}: {},
158+
collector.ScrapeInnodbCmpMem{}: {},
155159
}
156160

157161
var scrapersLr = map[collector.Scraper]struct{}{

0 commit comments

Comments
 (0)