Skip to content

Commit 248c3f7

Browse files
bwplotkasaswatamcode
bwplotka
and
saswatamcode
authored
exp/api: Add experimental exp module; Add remote API with write client and handler. (#1658)
* api: Add remote API with write client; add remote handler. Signed-off-by: bwplotka <[email protected]> * Make Write message type more flexble, address some feedback (#1710) * Address remaining feedback Signed-off-by: Saswata Mukherjee <[email protected]> * Make Write message type more flexible Signed-off-by: Saswata Mukherjee <[email protected]> --------- Signed-off-by: Saswata Mukherjee <[email protected]> * Move remote write API to client_golang/exp (#1711) * Move remote write API to client_golang/exp Signed-off-by: Saswata Mukherjee <[email protected]> * Don't use api.Client structs, add options for middleware Signed-off-by: Saswata Mukherjee <[email protected]> * Fix reqBuf usage Signed-off-by: Saswata Mukherjee <[email protected]> * Fix url path Signed-off-by: Saswata Mukherjee <[email protected]> * Add separate mod file (and workspace file) Signed-off-by: Saswata Mukherjee <[email protected]> * Hook exp tests fmt; Test handler error case; Configure backoff Signed-off-by: Saswata Mukherjee <[email protected]> --------- Signed-off-by: Saswata Mukherjee <[email protected]> * exp: Add README, address feedback, use sync.Pool (#1747) * Implement suggestion for Store interface and contentType Signed-off-by: Saswata Mukherjee <[email protected]> * Add README Signed-off-by: Saswata Mukherjee <[email protected]> * Use sync.Pool Signed-off-by: Saswata Mukherjee <[email protected]> * Implement review suggestions Signed-off-by: Saswata Mukherjee <[email protected]> * Release bufs right after compressPayload Signed-off-by: Saswata Mukherjee <[email protected]> --------- Signed-off-by: Saswata Mukherjee <[email protected]> * Bump exp to go 1.22 Signed-off-by: Saswata Mukherjee <[email protected]> --------- Signed-off-by: bwplotka <[email protected]> Signed-off-by: Saswata Mukherjee <[email protected]> Signed-off-by: Bartlomiej Plotka <[email protected]> Co-authored-by: Saswata Mukherjee <[email protected]>
1 parent 6b820eb commit 248c3f7

24 files changed

+5623
-10
lines changed

‎.bingo/Variables.mk

+11-5
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.8. DO NOT EDIT.
1+
# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.9. DO NOT EDIT.
22
# All tools are designed to be build inside $GOBIN.
33
BINGO_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
44
GOPATH ?= $(shell go env GOPATH)
@@ -7,16 +7,22 @@ GO ?= $(shell which go)
77

88
# Below generated variables ensure that every time a tool under each variable is invoked, the correct version
99
# will be used; reinstalling only if needed.
10-
# For example for goimports variable:
10+
# For example for buf variable:
1111
#
1212
# In your main Makefile (for non array binaries):
1313
#
1414
#include .bingo/Variables.mk # Assuming -dir was set to .bingo .
1515
#
16-
#command: $(GOIMPORTS)
17-
# @echo "Running goimports"
18-
# @$(GOIMPORTS) <flags/args..>
16+
#command: $(BUF)
17+
# @echo "Running buf"
18+
# @$(BUF) <flags/args..>
1919
#
20+
BUF := $(GOBIN)/buf-v1.39.0
21+
$(BUF): $(BINGO_DIR)/buf.mod
22+
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
23+
@echo "(re)installing $(GOBIN)/buf-v1.39.0"
24+
@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=buf.mod -o=$(GOBIN)/buf-v1.39.0 "github.com/bufbuild/buf/cmd/buf"
25+
2026
GOIMPORTS := $(GOBIN)/goimports-v0.9.3
2127
$(GOIMPORTS): $(BINGO_DIR)/goimports.mod
2228
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.

‎.bingo/buf.mod

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
2+
3+
go 1.22.6
4+
5+
require github.com/bufbuild/buf v1.39.0 // cmd/buf

‎.bingo/buf.sum

+336
Large diffs are not rendered by default.

‎.bingo/variables.env

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.8. DO NOT EDIT.
1+
# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.9. DO NOT EDIT.
22
# All tools are designed to be build inside $GOBIN.
33
# Those variables will work only until 'bingo get' was invoked, or if tools were installed via Makefile's Variables.mk.
44
GOBIN=${GOBIN:=$(go env GOBIN)}
@@ -8,5 +8,7 @@ if [ -z "$GOBIN" ]; then
88
fi
99

1010

11+
BUF="${GOBIN}/buf-v1.39.0"
12+
1113
GOIMPORTS="${GOBIN}/goimports-v0.9.3"
1214

‎Makefile

+27-3
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,16 @@
1414
include .bingo/Variables.mk
1515
include Makefile.common
1616

17+
.PHONY: deps
18+
deps:
19+
$(GO) work sync
20+
$(MAKE) common-deps
21+
1722
.PHONY: test
18-
test: deps common-test
23+
test: deps common-test test-exp
1924

2025
.PHONY: test-short
21-
test-short: deps common-test-short
26+
test-short: deps common-test-short test-exp-short
2227

2328
.PHONY: generate-go-collector-test-files
2429
file := supported_go_versions.txt
@@ -35,5 +40,24 @@ generate-go-collector-test-files:
3540
go mod tidy
3641

3742
.PHONY: fmt
38-
fmt: common-format
43+
fmt: common-format $(GOIMPORTS)
3944
$(GOIMPORTS) -local github.com/prometheus/client_golang -w .
45+
46+
.PHONY: proto
47+
proto: ## Regenerate Go from remote write proto.
48+
proto: $(BUF)
49+
@echo ">> regenerating Prometheus Remote Write proto"
50+
@cd exp/api/remote/genproto && $(BUF) generate
51+
@cd exp/api/remote && find genproto/ -type f -exec sed -i '' 's/protohelpers "github.com\/planetscale\/vtprotobuf\/protohelpers"/protohelpers "github.com\/prometheus\/client_golang\/exp\/internal\/github.com\/planetscale\/vtprotobuf\/protohelpers"/g' {} \;
52+
# For some reasons buf generates this unused import, kill it manually for now and reformat.
53+
@cd exp/api/remote && find genproto/ -type f -exec sed -i '' 's/_ "github.com\/gogo\/protobuf\/gogoproto"//g' {} \;
54+
@cd exp/api/remote && go fmt ./genproto/...
55+
$(MAKE) fmt
56+
57+
.PHONY: test-exp
58+
test-exp:
59+
cd exp && $(GOTEST) $(test-flags) $(GOOPTS) $(pkgs)
60+
61+
.PHONY: test-exp-short
62+
test-exp-short:
63+
cd exp && $(GOTEST) -short $(GOOPTS) $(pkgs)

‎api/client.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"bytes"
1919
"context"
2020
"errors"
21+
"io"
2122
"net"
2223
"net/http"
2324
"net/url"
@@ -133,7 +134,8 @@ func (c *httpClient) Do(ctx context.Context, req *http.Request) (*http.Response,
133134
resp, err := c.client.Do(req)
134135
defer func() {
135136
if resp != nil {
136-
resp.Body.Close()
137+
_, _ = io.Copy(io.Discard, resp.Body)
138+
_ = resp.Body.Close()
137139
}
138140
}()
139141

@@ -145,6 +147,7 @@ func (c *httpClient) Do(ctx context.Context, req *http.Request) (*http.Response,
145147
done := make(chan struct{})
146148
go func() {
147149
var buf bytes.Buffer
150+
// TODO(bwplotka): Add LimitReader for too long err messages (e.g. limit by 1KB)
148151
_, err = buf.ReadFrom(resp.Body)
149152
body = buf.Bytes()
150153
close(done)

‎exp/README.md

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# client_golang experimental module
2+
3+
Contains experimental utilities and APIs for Prometheus.
4+
The module may be contain breaking changes or be removed in the future.
5+
6+
Packages within this module are listed below.
7+
8+
## Remote
9+
10+
Implements bindings from Prometheus remote APIs (remote write v1 and v2 for now).
11+
12+
Contains flexible method for building API clients, that can send remote write protocol messages.
13+
14+
```go
15+
import (
16+
"github.com/prometheus/client_golang/exp/api/remote"
17+
)
18+
...
19+
20+
remoteAPI, err := remote.NewAPI(
21+
"https://your-remote-endpoint",
22+
remote.WithAPIHTTPClient(httpClient),
23+
remote.WithAPILogger(logger.With("component", "remote_write_api")),
24+
)
25+
...
26+
27+
stats, err := remoteAPI.Write(ctx, remote.WriteV2MessageType, protoWriteReq)
28+
```
29+
30+
Also contains handler methods for applications that would like to handle and store remote write requests.
31+
32+
```go
33+
import (
34+
"net/http"
35+
"log"
36+
37+
"github.com/prometheus/client_golang/exp/api/remote"
38+
)
39+
...
40+
41+
type db {}
42+
43+
func NewStorage() *db {}
44+
45+
func (d *db) Store(ctx context.Context, msgType remote.WriteMessageType, req *http.Request) (*remote.WriteResponse, error) {}
46+
...
47+
48+
mux := http.NewServeMux()
49+
50+
remoteWriteHandler := remote.NewHandler(storage, remote.WithHandlerLogger(logger.With("component", "remote_write_handler")))
51+
mux.Handle("/api/v1/write", remoteWriteHandler)
52+
53+
server := &http.Server{
54+
Addr: ":8080",
55+
Handler: mux,
56+
}
57+
if err := server.ListenAndServe(); err != nil {
58+
log.Fatal(err)
59+
}
60+
```
61+
62+
For more details, see [go doc](https://pkg.go.dev/github.com/prometheus/client_golang/exp/api/remote).

‎exp/api/remote/genproto/buf.gen.yaml

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# buf.gen.yaml
2+
version: v2
3+
4+
plugins:
5+
- remote: buf.build/protocolbuffers/go:v1.31.0
6+
out: .
7+
opt:
8+
- Mio/prometheus/write/v2/types.proto=./v2
9+
10+
# vtproto for efficiency utilities like pooling etc.
11+
# https://buf.build/community/planetscale-vtprotobuf?version=v0.6.0
12+
- remote: buf.build/community/planetscale-vtprotobuf:v0.6.0
13+
out: .
14+
opt:
15+
- Mio/prometheus/write/v2/types.proto=./v2
16+
- features=marshal+unmarshal+size
17+
18+
inputs:
19+
- module: buf.build/prometheus/prometheus:5b212ab78fb7460e831cf7ff2d83e385
20+
types:
21+
- "io.prometheus.write.v2.Request"

‎exp/api/remote/genproto/v2/symbols.go

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Copyright (c) Bartłomiej Płotka @bwplotka
2+
// Licensed under the Apache License 2.0.
3+
4+
// Copyright 2024 Google LLC
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// https://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
18+
// Copyright 2024 Prometheus Team
19+
// Licensed under the Apache License, Version 2.0 (the "License");
20+
// you may not use this file except in compliance with the License.
21+
// You may obtain a copy of the License at
22+
//
23+
// http://www.apache.org/licenses/LICENSE-2.0
24+
//
25+
// Unless required by applicable law or agreed to in writing, software
26+
// distributed under the License is distributed on an "AS IS" BASIS,
27+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
28+
// See the License for the specific language governing permissions and
29+
// limitations under the License.
30+
31+
package writev2
32+
33+
// SymbolsTable implements table for easy symbol use.
34+
type SymbolsTable struct {
35+
strings []string
36+
symbolsMap map[string]uint32
37+
}
38+
39+
// NewSymbolTable returns a symbol table.
40+
func NewSymbolTable() SymbolsTable {
41+
return SymbolsTable{
42+
// Empty string is required as a first element.
43+
symbolsMap: map[string]uint32{"": 0},
44+
strings: []string{""},
45+
}
46+
}
47+
48+
// Symbolize adds (if not added before) a string to the symbols table,
49+
// while returning its reference number.
50+
func (t *SymbolsTable) Symbolize(str string) uint32 {
51+
if ref, ok := t.symbolsMap[str]; ok {
52+
return ref
53+
}
54+
ref := uint32(len(t.strings))
55+
t.strings = append(t.strings, str)
56+
t.symbolsMap[str] = ref
57+
return ref
58+
}
59+
60+
// SymbolizeLabels symbolize Prometheus labels.
61+
func (t *SymbolsTable) SymbolizeLabels(lbls []string, buf []uint32) []uint32 {
62+
result := buf[:0]
63+
for i := 0; i < len(lbls); i += 2 {
64+
off := t.Symbolize(lbls[i])
65+
result = append(result, off)
66+
off = t.Symbolize(lbls[i+1])
67+
result = append(result, off)
68+
}
69+
return result
70+
}
71+
72+
// Symbols returns computes symbols table to put in e.g. Request.Symbols.
73+
// As per spec, order does not matter.
74+
func (t *SymbolsTable) Symbols() []string {
75+
return t.strings
76+
}
77+
78+
// Reset clears symbols table.
79+
func (t *SymbolsTable) Reset() {
80+
// NOTE: Make sure to keep empty symbol.
81+
t.strings = t.strings[:1]
82+
for k := range t.symbolsMap {
83+
if k == "" {
84+
continue
85+
}
86+
delete(t.symbolsMap, k)
87+
}
88+
}
89+
90+
// DesymbolizeLabels decodes label references, with given symbols to labels.
91+
func DesymbolizeLabels(labelRefs []uint32, symbols, buf []string) []string {
92+
result := buf[:0]
93+
for i := 0; i < len(labelRefs); i += 2 {
94+
result = append(result, symbols[labelRefs[i]], symbols[labelRefs[i+1]])
95+
}
96+
return result
97+
}

‎exp/api/remote/genproto/v2/symbols_test.go

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Copyright (c) Bartłomiej Płotka @bwplotka
2+
// Licensed under the Apache License 2.0.
3+
4+
// Copyright 2024 Google LLC
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// https://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
18+
// Copyright 2024 Prometheus Team
19+
// Licensed under the Apache License, Version 2.0 (the "License");
20+
// you may not use this file except in compliance with the License.
21+
// You may obtain a copy of the License at
22+
//
23+
// http://www.apache.org/licenses/LICENSE-2.0
24+
//
25+
// Unless required by applicable law or agreed to in writing, software
26+
// distributed under the License is distributed on an "AS IS" BASIS,
27+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
28+
// See the License for the specific language governing permissions and
29+
// limitations under the License.
30+
31+
package writev2
32+
33+
import (
34+
"testing"
35+
36+
"github.com/google/go-cmp/cmp"
37+
)
38+
39+
func requireEqual(t testing.TB, expected, got any) {
40+
if diff := cmp.Diff(expected, got); diff != "" {
41+
t.Fatal(diff)
42+
}
43+
}
44+
45+
func TestSymbolsTable(t *testing.T) {
46+
s := NewSymbolTable()
47+
requireEqual(t, []string{""}, s.Symbols())
48+
requireEqual(t, uint32(0), s.Symbolize(""))
49+
requireEqual(t, []string{""}, s.Symbols())
50+
51+
requireEqual(t, uint32(1), s.Symbolize("abc"))
52+
requireEqual(t, []string{"", "abc"}, s.Symbols())
53+
54+
requireEqual(t, uint32(2), s.Symbolize("__name__"))
55+
requireEqual(t, []string{"", "abc", "__name__"}, s.Symbols())
56+
57+
requireEqual(t, uint32(3), s.Symbolize("foo"))
58+
requireEqual(t, []string{"", "abc", "__name__", "foo"}, s.Symbols())
59+
60+
s.Reset()
61+
requireEqual(t, []string{""}, s.Symbols())
62+
requireEqual(t, uint32(0), s.Symbolize(""))
63+
64+
requireEqual(t, uint32(1), s.Symbolize("__name__"))
65+
requireEqual(t, []string{"", "__name__"}, s.Symbols())
66+
67+
requireEqual(t, uint32(2), s.Symbolize("abc"))
68+
requireEqual(t, []string{"", "__name__", "abc"}, s.Symbols())
69+
70+
ls := []string{"__name__", "qwer", "zxcv", "1234"}
71+
encoded := s.SymbolizeLabels(ls, nil)
72+
requireEqual(t, []uint32{1, 3, 4, 5}, encoded)
73+
decoded := DesymbolizeLabels(encoded, s.Symbols(), nil)
74+
requireEqual(t, ls, decoded)
75+
76+
// Different buf.
77+
ls = []string{"__name__", "qwer", "zxcv2222", "1234"}
78+
encoded = s.SymbolizeLabels(ls, []uint32{1, 3, 4, 5})
79+
requireEqual(t, []uint32{1, 3, 6, 5}, encoded)
80+
}

0 commit comments

Comments
 (0)