Skip to content

Commit f255071

Browse files
zqztenzqzten
committed
introduce pprof server to manager
1 parent ffd9ec8 commit f255071

File tree

5 files changed

+191
-0
lines changed

5 files changed

+191
-0
lines changed

‎pkg/config/v1alpha1/types.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ type ControllerManagerConfigurationSpec struct {
6868
// +optional
6969
Health ControllerHealth `json:"health,omitempty"`
7070

71+
// Pprof contains the controller pprof configuration
72+
// +optional
73+
Pprof ControllerPprof `json:"pprof,omitempty"`
74+
7175
// Webhook contains the controllers webhook configuration
7276
// +optional
7377
Webhook ControllerWebhook `json:"webhook,omitempty"`
@@ -121,6 +125,17 @@ type ControllerHealth struct {
121125
LivenessEndpointName string `json:"livenessEndpointName,omitempty"`
122126
}
123127

128+
// ControllerPprof defines the pprof configs.
129+
type ControllerPprof struct {
130+
// BindAddress is the TCP address that the controller should bind to
131+
// for serving pprof.
132+
// It can be set to "" or "0" to disable the pprof serving.
133+
// Since pprof may contain sensitive information, make sure to protect it
134+
// before exposing it to public.
135+
// +optional
136+
BindAddress string `json:"bindAddress,omitempty"`
137+
}
138+
124139
// ControllerWebhook defines the webhook server for the controller.
125140
type ControllerWebhook struct {
126141
// Port is the port that the webhook server serves at.

‎pkg/config/v1alpha1/zz_generated.deepcopy.go

Lines changed: 16 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/manager/internal.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"fmt"
2323
"net"
2424
"net/http"
25+
"net/http/pprof"
2526
"sync"
2627
"sync/atomic"
2728
"time"
@@ -106,6 +107,9 @@ type controllerManager struct {
106107
// Healthz probe handler
107108
healthzHandler *healthz.Handler
108109

110+
// pprofListener is used to serve pprof
111+
pprofListener net.Listener
112+
109113
// controllerOptions are the global controller options.
110114
controllerOptions v1alpha1.ControllerConfigurationSpec
111115

@@ -355,6 +359,19 @@ func (cm *controllerManager) serveHealthProbes() {
355359
go cm.httpServe("health probe", cm.logger, server, cm.healthProbeListener)
356360
}
357361

362+
func (cm *controllerManager) servePprof() {
363+
mux := http.NewServeMux()
364+
server := httpserver.New(mux)
365+
366+
mux.HandleFunc("/debug/pprof/", pprof.Index)
367+
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
368+
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
369+
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
370+
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
371+
372+
go cm.httpServe("pprof", cm.logger, server, cm.pprofListener)
373+
}
374+
358375
func (cm *controllerManager) httpServe(kind string, log logr.Logger, server *http.Server, ln net.Listener) {
359376
log = log.WithValues("kind", kind, "addr", ln.Addr())
360377

@@ -450,6 +467,11 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) {
450467
cm.serveHealthProbes()
451468
}
452469

470+
// Serve pprof.
471+
if cm.pprofListener != nil {
472+
cm.servePprof()
473+
}
474+
453475
// First start any webhook servers, which includes conversion, validation, and defaulting
454476
// webhooks that are registered.
455477
//

‎pkg/manager/manager.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,13 @@ type Options struct {
227227
// Liveness probe endpoint name, defaults to "healthz"
228228
LivenessEndpointName string
229229

230+
// PprofBindAddress is the TCP address that the controller should bind to
231+
// for serving pprof.
232+
// It can be set to "" or "0" to disable the pprof serving.
233+
// Since pprof may contain sensitive information, make sure to protect it
234+
// before exposing it to public.
235+
PprofBindAddress string
236+
230237
// Port is the port that the webhook server serves at.
231238
// It is used to set webhook.Server.Port if WebhookServer is not set.
232239
Port int
@@ -299,6 +306,7 @@ type Options struct {
299306
newResourceLock func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error)
300307
newMetricsListener func(addr string) (net.Listener, error)
301308
newHealthProbeListener func(addr string) (net.Listener, error)
309+
newPprofListener func(addr string) (net.Listener, error)
302310
}
303311

304312
// BaseContextFunc is a function used to provide a base Context to Runnables
@@ -403,6 +411,13 @@ func New(config *rest.Config, options Options) (Manager, error) {
403411
return nil, err
404412
}
405413

414+
// Create pprof listener. This will throw an error if the bind
415+
// address is invalid or already in use.
416+
pprofListener, err := options.newPprofListener(options.PprofBindAddress)
417+
if err != nil {
418+
return nil, err
419+
}
420+
406421
errChan := make(chan error)
407422
runnables := newRunnables(options.BaseContext, errChan)
408423

@@ -428,6 +443,7 @@ func New(config *rest.Config, options Options) (Manager, error) {
428443
healthProbeListener: healthProbeListener,
429444
readinessEndpointName: options.ReadinessEndpointName,
430445
livenessEndpointName: options.LivenessEndpointName,
446+
pprofListener: pprofListener,
431447
gracefulShutdownTimeout: *options.GracefulShutdownTimeout,
432448
internalProceduresStop: make(chan struct{}),
433449
leaderElectionStopped: make(chan struct{}),
@@ -477,6 +493,10 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
477493
o.LivenessEndpointName = newObj.Health.LivenessEndpointName
478494
}
479495

496+
if o.PprofBindAddress == "" && newObj.Pprof.BindAddress != "" {
497+
o.PprofBindAddress = newObj.Pprof.BindAddress
498+
}
499+
480500
if o.Port == 0 && newObj.Webhook.Port != nil {
481501
o.Port = *newObj.Webhook.Port
482502
}
@@ -561,6 +581,19 @@ func defaultHealthProbeListener(addr string) (net.Listener, error) {
561581
return ln, nil
562582
}
563583

584+
// defaultPprofListener creates the default pprof listener bound to the given address.
585+
func defaultPprofListener(addr string) (net.Listener, error) {
586+
if addr == "" || addr == "0" {
587+
return nil, nil
588+
}
589+
590+
ln, err := net.Listen("tcp", addr)
591+
if err != nil {
592+
return nil, fmt.Errorf("error listening on %s: %w", addr, err)
593+
}
594+
return ln, nil
595+
}
596+
564597
// defaultBaseContext is used as the BaseContext value in Options if one
565598
// has not already been set.
566599
func defaultBaseContext() context.Context {
@@ -621,6 +654,10 @@ func setOptionsDefaults(options Options) Options {
621654
options.newHealthProbeListener = defaultHealthProbeListener
622655
}
623656

657+
if options.newPprofListener == nil {
658+
options.newPprofListener = defaultPprofListener
659+
}
660+
624661
if options.GracefulShutdownTimeout == nil {
625662
gracefulShutdownTimeout := defaultGracefulShutdownPeriod
626663
options.GracefulShutdownTimeout = &gracefulShutdownTimeout

‎pkg/manager/manager_test.go

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ var _ = Describe("manger.Manager", func() {
147147
ReadinessEndpointName: "/readyz",
148148
LivenessEndpointName: "/livez",
149149
},
150+
Pprof: v1alpha1.ControllerPprof{
151+
BindAddress: ":6080",
152+
},
150153
Webhook: v1alpha1.ControllerWebhook{
151154
Port: &port,
152155
Host: "localhost",
@@ -171,6 +174,7 @@ var _ = Describe("manger.Manager", func() {
171174
Expect(m.HealthProbeBindAddress).To(Equal("6060"))
172175
Expect(m.ReadinessEndpointName).To(Equal("/readyz"))
173176
Expect(m.LivenessEndpointName).To(Equal("/livez"))
177+
Expect(m.PprofBindAddress).To(Equal(":6080"))
174178
Expect(m.Port).To(Equal(port))
175179
Expect(m.Host).To(Equal("localhost"))
176180
Expect(m.CertDir).To(Equal("/certs"))
@@ -203,6 +207,9 @@ var _ = Describe("manger.Manager", func() {
203207
ReadinessEndpointName: "/readyz",
204208
LivenessEndpointName: "/livez",
205209
},
210+
Pprof: v1alpha1.ControllerPprof{
211+
BindAddress: ":6080",
212+
},
206213
Webhook: v1alpha1.ControllerWebhook{
207214
Port: &port,
208215
Host: "localhost",
@@ -225,6 +232,7 @@ var _ = Describe("manger.Manager", func() {
225232
HealthProbeBindAddress: "5000",
226233
ReadinessEndpointName: "/readiness",
227234
LivenessEndpointName: "/liveness",
235+
PprofBindAddress: ":6000",
228236
Port: 8080,
229237
Host: "example.com",
230238
CertDir: "/pki",
@@ -244,6 +252,7 @@ var _ = Describe("manger.Manager", func() {
244252
Expect(m.HealthProbeBindAddress).To(Equal("5000"))
245253
Expect(m.ReadinessEndpointName).To(Equal("/readiness"))
246254
Expect(m.LivenessEndpointName).To(Equal("/liveness"))
255+
Expect(m.PprofBindAddress).To(Equal(":6000"))
247256
Expect(m.Port).To(Equal(8080))
248257
Expect(m.Host).To(Equal("example.com"))
249258
Expect(m.CertDir).To(Equal("/pki"))
@@ -282,6 +291,7 @@ var _ = Describe("manger.Manager", func() {
282291
LeaderElectionID: "test-leader-election-id-2",
283292
HealthProbeBindAddress: "0",
284293
MetricsBindAddress: "0",
294+
PprofBindAddress: "0",
285295
})
286296
Expect(err).To(BeNil())
287297

@@ -327,6 +337,7 @@ var _ = Describe("manger.Manager", func() {
327337
LeaderElectionID: "test-leader-election-id-3",
328338
HealthProbeBindAddress: "0",
329339
MetricsBindAddress: "0",
340+
PprofBindAddress: "0",
330341
})
331342
Expect(err).To(BeNil())
332343

@@ -361,6 +372,7 @@ var _ = Describe("manger.Manager", func() {
361372
},
362373
HealthProbeBindAddress: "0",
363374
MetricsBindAddress: "0",
375+
PprofBindAddress: "0",
364376
})
365377
Expect(err).ToNot(HaveOccurred())
366378
Expect(m1).ToNot(BeNil())
@@ -381,6 +393,7 @@ var _ = Describe("manger.Manager", func() {
381393
},
382394
HealthProbeBindAddress: "0",
383395
MetricsBindAddress: "0",
396+
PprofBindAddress: "0",
384397
})
385398
Expect(err).ToNot(HaveOccurred())
386399
Expect(m2).ToNot(BeNil())
@@ -1395,6 +1408,94 @@ var _ = Describe("manger.Manager", func() {
13951408
})
13961409
})
13971410

1411+
Context("should start serving pprof", func() {
1412+
var listener net.Listener
1413+
var opts Options
1414+
1415+
BeforeEach(func() {
1416+
listener = nil
1417+
opts = Options{
1418+
newPprofListener: func(addr string) (net.Listener, error) {
1419+
var err error
1420+
listener, err = defaultPprofListener(addr)
1421+
return listener, err
1422+
},
1423+
}
1424+
})
1425+
1426+
AfterEach(func() {
1427+
if listener != nil {
1428+
listener.Close()
1429+
}
1430+
})
1431+
1432+
It("should stop serving pprof when stop is called", func() {
1433+
opts.PprofBindAddress = ":0"
1434+
m, err := New(cfg, opts)
1435+
Expect(err).NotTo(HaveOccurred())
1436+
1437+
ctx, cancel := context.WithCancel(context.Background())
1438+
go func() {
1439+
defer GinkgoRecover()
1440+
Expect(m.Start(ctx)).NotTo(HaveOccurred())
1441+
}()
1442+
<-m.Elected()
1443+
1444+
// Check the pprof started
1445+
endpoint := fmt.Sprintf("http://%s", listener.Addr().String())
1446+
_, err = http.Get(endpoint)
1447+
Expect(err).NotTo(HaveOccurred())
1448+
1449+
// Shutdown the server
1450+
cancel()
1451+
1452+
// Expect the pprof server to shutdown
1453+
Eventually(func() error {
1454+
_, err = http.Get(endpoint)
1455+
return err
1456+
}).ShouldNot(Succeed())
1457+
})
1458+
1459+
It("should serve pprof endpoints", func() {
1460+
opts.PprofBindAddress = ":0"
1461+
m, err := New(cfg, opts)
1462+
Expect(err).NotTo(HaveOccurred())
1463+
1464+
ctx, cancel := context.WithCancel(context.Background())
1465+
defer cancel()
1466+
go func() {
1467+
defer GinkgoRecover()
1468+
Expect(m.Start(ctx)).NotTo(HaveOccurred())
1469+
}()
1470+
<-m.Elected()
1471+
1472+
pprofIndexEndpoint := fmt.Sprintf("http://%s/debug/pprof/", listener.Addr().String())
1473+
resp, err := http.Get(pprofIndexEndpoint)
1474+
Expect(err).NotTo(HaveOccurred())
1475+
Expect(resp.StatusCode).To(Equal(200))
1476+
1477+
pprofCmdlineEndpoint := fmt.Sprintf("http://%s/debug/pprof/cmdline", listener.Addr().String())
1478+
resp, err = http.Get(pprofCmdlineEndpoint)
1479+
Expect(err).NotTo(HaveOccurred())
1480+
Expect(resp.StatusCode).To(Equal(200))
1481+
1482+
pprofProfileEndpoint := fmt.Sprintf("http://%s/debug/pprof/profile", listener.Addr().String())
1483+
resp, err = http.Get(pprofProfileEndpoint)
1484+
Expect(err).NotTo(HaveOccurred())
1485+
Expect(resp.StatusCode).To(Equal(200))
1486+
1487+
pprofSymbolEndpoint := fmt.Sprintf("http://%s/debug/pprof/symbol", listener.Addr().String())
1488+
resp, err = http.Get(pprofSymbolEndpoint)
1489+
Expect(err).NotTo(HaveOccurred())
1490+
Expect(resp.StatusCode).To(Equal(200))
1491+
1492+
pprofTraceEndpoint := fmt.Sprintf("http://%s/debug/pprof/trace", listener.Addr().String())
1493+
resp, err = http.Get(pprofTraceEndpoint)
1494+
Expect(err).NotTo(HaveOccurred())
1495+
Expect(resp.StatusCode).To(Equal(200))
1496+
})
1497+
})
1498+
13981499
Describe("Add", func() {
13991500
It("should immediately start the Component if the Manager has already Started another Component",
14001501
func() {

0 commit comments

Comments
 (0)