Skip to content

Commit 72f1672

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #675 from mmiranda96/feat/net-monitor-groupings
Add ExcludeInterfaceRegexp to Net Dev monitor
2 parents 56122ce + 1471f74 commit 72f1672

File tree

5 files changed

+254
-14
lines changed

5 files changed

+254
-14
lines changed

‎config/net-cgroup-system-stats-monitor.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"net": {
3+
"excludeInterfaceRegexp": "^(cali|tunl|veth)",
34
"metricsConfigs": {
45
"net/rx_bytes": {
56
"displayName": "net/rx_bytes"

‎pkg/systemstatsmonitor/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ Below metrics are collected from `net` component:
118118

119119
All of the above have `interface_name` label for the net interface.
120120

121+
Interfaces can be skipped if they don't add any value. See field `ExcludeInterfaceRegexp`.
122+
121123
## Windows Support
122124

123125
NPD has preliminary support for system stats monitor. The following modules are supported:

‎pkg/systemstatsmonitor/net_collector.go

Lines changed: 32 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -26,19 +26,20 @@ import (
2626
"github.com/prometheus/procfs"
2727
)
2828

29+
type newInt64MetricFn func(metricID metrics.MetricID, viewName string, description string, unit string, aggregation metrics.Aggregation, tagNames []string) (metrics.Int64MetricInterface, error)
30+
31+
// newInt64Metric is a wrapper of metrics.NewInt64Metric that returns an interface instead of the specific type
32+
func newInt64Metric(metricID metrics.MetricID, viewName string, description string, unit string, aggregation metrics.Aggregation, tagNames []string) (metrics.Int64MetricInterface, error) {
33+
return metrics.NewInt64Metric(metricID, viewName, description, unit, aggregation, tagNames)
34+
}
35+
2936
type netCollector struct {
3037
config *ssmtypes.NetStatsConfig
3138
procPath string
3239
recorder *ifaceStatRecorder
3340
}
3441

35-
func NewNetCollectorOrDie(netConfig *ssmtypes.NetStatsConfig, procPath string) *netCollector {
36-
nc := netCollector{
37-
config: netConfig,
38-
procPath: procPath,
39-
recorder: newIfaceStatRecorder(),
40-
}
41-
42+
func (nc *netCollector) initOrDie() {
4243
nc.mustRegisterMetric(
4344
metrics.NetDevRxBytes,
4445
"Cumulative count of bytes received.",
@@ -191,8 +192,16 @@ func NewNetCollectorOrDie(netConfig *ssmtypes.NetStatsConfig, procPath string) *
191192
return int64(stat.TxCompressed)
192193
},
193194
)
195+
}
194196

195-
return &nc
197+
func NewNetCollectorOrDie(netConfig *ssmtypes.NetStatsConfig, procPath string) *netCollector {
198+
nc := &netCollector{
199+
config: netConfig,
200+
procPath: procPath,
201+
recorder: newIfaceStatRecorder(newInt64Metric),
202+
}
203+
nc.initOrDie()
204+
return nc
196205
}
197206

198207
func (nc *netCollector) mustRegisterMetric(metricID metrics.MetricID, description, unit string,
@@ -216,7 +225,12 @@ func (nc *netCollector) recordNetDev() {
216225
return
217226
}
218227

228+
excludeInterfaceRegexp := nc.config.ExcludeInterfaceRegexp.R
219229
for iface, ifaceStats := range stats {
230+
if excludeInterfaceRegexp != nil && excludeInterfaceRegexp.MatchString(iface) {
231+
glog.V(6).Infof("Network interface %s matched exclude regexp %q, skipping recording", iface, excludeInterfaceRegexp)
232+
continue
233+
}
220234
tags := map[string]string{}
221235
tags[interfaceNameLabel] = iface
222236

@@ -234,11 +248,16 @@ func (nc *netCollector) collect() {
234248

235249
// TODO(@oif): Maybe implements a generic recorder
236250
type ifaceStatRecorder struct {
237-
collectors map[metrics.MetricID]ifaceStatCollector
251+
// We use a function to allow injecting a mock for testing
252+
newInt64Metric newInt64MetricFn
253+
collectors map[metrics.MetricID]ifaceStatCollector
238254
}
239255

240-
func newIfaceStatRecorder() *ifaceStatRecorder {
241-
return &ifaceStatRecorder{collectors: make(map[metrics.MetricID]ifaceStatCollector)}
256+
func newIfaceStatRecorder(newInt64Metric newInt64MetricFn) *ifaceStatRecorder {
257+
return &ifaceStatRecorder{
258+
newInt64Metric: newInt64Metric,
259+
collectors: make(map[metrics.MetricID]ifaceStatCollector),
260+
}
242261
}
243262

244263
func (r *ifaceStatRecorder) Register(metricID metrics.MetricID, viewName string, description string,
@@ -247,7 +266,7 @@ func (r *ifaceStatRecorder) Register(metricID metrics.MetricID, viewName string,
247266
// Check duplication
248267
return fmt.Errorf("metric %q already registered", metricID)
249268
}
250-
metric, err := metrics.NewInt64Metric(metricID, viewName, description, unit, aggregation, tagNames)
269+
metric, err := r.newInt64Metric(metricID, viewName, description, unit, aggregation, tagNames)
251270
if err != nil {
252271
return err
253272
}
@@ -268,6 +287,6 @@ func (r ifaceStatRecorder) RecordWithSameTags(stat procfs.NetDevLine, tags map[s
268287
}
269288

270289
type ifaceStatCollector struct {
271-
metric *metrics.Int64Metric
290+
metric metrics.Int64MetricInterface
272291
exporter func(procfs.NetDevLine) int64
273292
}

‎pkg/systemstatsmonitor/net_collector_test.go

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
package systemstatsmonitor
2+
3+
import (
4+
"io/ioutil"
5+
"os"
6+
"path"
7+
"regexp"
8+
"testing"
9+
10+
ssmtypes "k8s.io/node-problem-detector/pkg/systemstatsmonitor/types"
11+
"k8s.io/node-problem-detector/pkg/util/metrics"
12+
)
13+
14+
var defaultMetricsConfig = map[string]ssmtypes.MetricConfig{
15+
string(metrics.NetDevRxBytes): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevRxBytes)},
16+
string(metrics.NetDevRxPackets): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevRxPackets)},
17+
string(metrics.NetDevRxErrors): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevRxErrors)},
18+
string(metrics.NetDevRxDropped): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevRxDropped)},
19+
string(metrics.NetDevRxFifo): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevRxFifo)},
20+
string(metrics.NetDevRxFrame): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevRxFrame)},
21+
string(metrics.NetDevRxCompressed): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevRxCompressed)},
22+
string(metrics.NetDevRxMulticast): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevRxMulticast)},
23+
string(metrics.NetDevTxBytes): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevTxBytes)},
24+
string(metrics.NetDevTxPackets): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevTxPackets)},
25+
string(metrics.NetDevTxErrors): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevTxErrors)},
26+
string(metrics.NetDevTxDropped): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevTxDropped)},
27+
string(metrics.NetDevTxFifo): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevTxFifo)},
28+
string(metrics.NetDevTxCollisions): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevTxCollisions)},
29+
string(metrics.NetDevTxCarrier): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevTxCarrier)},
30+
string(metrics.NetDevTxCompressed): ssmtypes.MetricConfig{DisplayName: string(metrics.NetDevTxCompressed)},
31+
}
32+
33+
// To get a similar output, run `cat /proc/net/dev` on a Linux machine
34+
// docker: 1500 100 8 7 0 0 0 0 9000 450 565 200 20 30 0 0
35+
const fakeNetProcContent = `Inter-| Receive | Transmit
36+
face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed
37+
eth0: 5000 100 0 0 0 0 0 0 2500 30 0 0 0 0 0 0
38+
docker0: 1000 90 8 7 0 0 0 0 0 0 0 0 0 0 0 0
39+
docker1: 500 10 0 0 0 0 0 0 3000 150 15 0 20 30 0 0
40+
docker2: 0 0 0 0 0 0 0 0 6000 300 550 200 0 0 0 0
41+
`
42+
43+
// newFakeInt64Metric is a wrapper around metrics.NewFakeInt64Metric
44+
func newFakeInt64Metric(metricID metrics.MetricID, viewName string, description string, unit string, aggregation metrics.Aggregation, tagNames []string) (metrics.Int64MetricInterface, error) {
45+
return metrics.NewFakeInt64Metric(viewName, aggregation, tagNames), nil
46+
}
47+
48+
// testCollectAux is a test auxiliary function used for testing netCollector.Collect
49+
func testCollectAux(t *testing.T, name string, excludeInterfaceRegexp ssmtypes.NetStatsInterfaceRegexp, validate func(*testing.T, *netCollector)) {
50+
// mkdir /tmp/proc-X
51+
procDir, err := ioutil.TempDir(os.TempDir(), "proc-")
52+
if err != nil {
53+
t.Fatalf("Failed to create temp proc directory: %v", err)
54+
}
55+
// rm -r /tmp/proc-X
56+
defer os.RemoveAll(procDir)
57+
// mkdir -C /tmp/proc-X/net
58+
procNetDir := path.Join(procDir, "net")
59+
if err := os.Mkdir(procNetDir, 0777); err != nil {
60+
t.Fatalf("Failed to create directory %q: %v", procNetDir, err)
61+
}
62+
63+
// touch /tmp/proc-X/net/dev
64+
filename := path.Join(procNetDir, "dev")
65+
f, err := os.Create(filename)
66+
if err != nil {
67+
t.Fatalf("Failed to create file %q: %v", filename, err)
68+
}
69+
// echo $FILE_CONTENT > /tmp/proc-X/net/dev
70+
if _, err = f.WriteString(fakeNetProcContent); err != nil {
71+
t.Fatalf("Failed to write to file %q: %v", filename, err)
72+
}
73+
if err = f.Close(); err != nil {
74+
t.Fatalf("Failed to close file %q: %v", filename, err)
75+
}
76+
77+
// Build the netCollector
78+
config := &ssmtypes.NetStatsConfig{
79+
ExcludeInterfaceRegexp: excludeInterfaceRegexp,
80+
MetricsConfigs: defaultMetricsConfig,
81+
}
82+
netCollector := &netCollector{
83+
config: config,
84+
procPath: procDir,
85+
recorder: newIfaceStatRecorder(newFakeInt64Metric),
86+
}
87+
netCollector.initOrDie()
88+
netCollector.collect()
89+
validate(t, netCollector)
90+
}
91+
92+
func TestCollect(t *testing.T) {
93+
tcs := []struct {
94+
Name string
95+
ExcludeInterfaceRegexp ssmtypes.NetStatsInterfaceRegexp
96+
Validate func(t *testing.T, nc *netCollector)
97+
}{
98+
{
99+
Name: "NoFilterMatch",
100+
ExcludeInterfaceRegexp: ssmtypes.NetStatsInterfaceRegexp{R: regexp.MustCompile(`^fake$`)},
101+
Validate: func(t *testing.T, nc *netCollector) {
102+
// We just validate two metrics, no need to check all of them
103+
expectedValues := map[metrics.MetricID]map[string]int64{
104+
metrics.NetDevRxBytes: map[string]int64{
105+
"eth0": 5000,
106+
"docker0": 1000,
107+
"docker1": 500,
108+
"docker2": 0,
109+
},
110+
metrics.NetDevTxBytes: map[string]int64{
111+
"eth0": 2500,
112+
"docker0": 0,
113+
"docker1": 3000,
114+
"docker2": 6000,
115+
},
116+
}
117+
for metricID, interfaceValues := range expectedValues {
118+
collector, ok := nc.recorder.collectors[metricID]
119+
if !ok {
120+
t.Errorf("Failed to get collector of metric %s", metricID)
121+
continue
122+
}
123+
fakeInt64Metric, ok := collector.metric.(*metrics.FakeInt64Metric)
124+
if !ok {
125+
t.Fatalf("Failed to convert metric %s to fakeMetric", string(metricID))
126+
}
127+
for _, repr := range fakeInt64Metric.ListMetrics() {
128+
interfaceName, ok := repr.Labels[interfaceNameLabel]
129+
if !ok {
130+
t.Fatalf("Failed to get label %q for ", interfaceNameLabel)
131+
}
132+
expectedValue, ok := interfaceValues[interfaceName]
133+
if !ok {
134+
t.Fatalf("Failed to get metric value for interface %q", interfaceName)
135+
}
136+
if repr.Value != expectedValue {
137+
t.Errorf("Mismatch in metric %q for interface %q: expected %d, got %d", metricID, interfaceName, expectedValue, repr.Value)
138+
}
139+
}
140+
}
141+
},
142+
},
143+
{
144+
Name: "FilterMatch",
145+
ExcludeInterfaceRegexp: ssmtypes.NetStatsInterfaceRegexp{R: regexp.MustCompile(`docker\d+`)},
146+
Validate: func(t *testing.T, nc *netCollector) {
147+
// We just validate two metrics, no need to check all of them
148+
expectedValues := map[metrics.MetricID]map[string]int64{
149+
metrics.NetDevRxBytes: map[string]int64{
150+
"eth0": 5000,
151+
},
152+
metrics.NetDevTxBytes: map[string]int64{
153+
"eth0": 2500,
154+
},
155+
}
156+
for metricID, interfaceValues := range expectedValues {
157+
collector, ok := nc.recorder.collectors[metricID]
158+
if !ok {
159+
t.Errorf("Failed to get collector of metric %s", metricID)
160+
continue
161+
}
162+
fakeInt64Metric, ok := collector.metric.(*metrics.FakeInt64Metric)
163+
if !ok {
164+
t.Fatalf("Failed to convert metric %s to fakeMetric", string(metricID))
165+
}
166+
for _, repr := range fakeInt64Metric.ListMetrics() {
167+
interfaceName, ok := repr.Labels[interfaceNameLabel]
168+
if !ok {
169+
t.Fatalf("Failed to get label %q for ", interfaceNameLabel)
170+
}
171+
expectedValue, ok := interfaceValues[interfaceName]
172+
if !ok {
173+
t.Fatalf("Failed to get metric value for interface %q", interfaceName)
174+
}
175+
if repr.Value != expectedValue {
176+
t.Errorf("Mismatch in metric %q for interface %q: expected %d, got %d", metricID, interfaceName, expectedValue, repr.Value)
177+
}
178+
}
179+
}
180+
},
181+
},
182+
}
183+
for _, tc := range tcs {
184+
tc := tc
185+
t.Run(tc.Name, func(t *testing.T) {
186+
t.Parallel()
187+
testCollectAux(t, tc.Name, tc.ExcludeInterfaceRegexp, tc.Validate)
188+
})
189+
}
190+
}

‎pkg/systemstatsmonitor/types/config.go

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package types
1919
import (
2020
"fmt"
2121
"os"
22+
"regexp"
2223
"time"
2324
)
2425

@@ -58,8 +59,35 @@ type OSFeatureStatsConfig struct {
5859
KnownModulesConfigPath string `json:"knownModulesConfigPath"`
5960
}
6061

62+
// In order to marshal/unmarshal regexp, we need to implement
63+
// MarshalText/UnmarshalText methods in a wrapper struct
64+
type NetStatsInterfaceRegexp struct {
65+
R *regexp.Regexp
66+
}
67+
68+
func (r *NetStatsInterfaceRegexp) UnmarshalText(data []byte) error {
69+
// We don't build Regexp if data is empty
70+
if len(data) == 0 {
71+
return nil
72+
}
73+
regex, err := regexp.Compile(string(data))
74+
if err != nil {
75+
return err
76+
}
77+
r.R = regex
78+
return nil
79+
}
80+
81+
func (r NetStatsInterfaceRegexp) MarshalText() ([]byte, error) {
82+
if r.R == nil {
83+
return nil, nil
84+
}
85+
return []byte(r.R.String()), nil
86+
}
87+
6188
type NetStatsConfig struct {
62-
MetricsConfigs map[string]MetricConfig `json:"metricsConfigs"`
89+
MetricsConfigs map[string]MetricConfig `json:"metricsConfigs"`
90+
ExcludeInterfaceRegexp NetStatsInterfaceRegexp `json:"excludeInterfaceRegexp"`
6391
}
6492

6593
type SystemStatsConfig struct {

0 commit comments

Comments
 (0)