Skip to content

Commit 833a6e6

Browse files
zqztenzqzten
committed
introduce pprof server to manager
1 parent 3da2de0 commit 833a6e6

File tree

6 files changed

+259
-0
lines changed

6 files changed

+259
-0
lines changed

‎pkg/config/v1alpha1/types.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ type ControllerManagerConfigurationSpec struct {
6868
// +optional
6969
Health ControllerHealth `json:"health,omitempty"`
7070

71+
// Profiling contains the controller profiling configuration
72+
// +optional
73+
Profiling ControllerProfiling `json:"profiling,omitempty"`
74+
7175
// Webhook contains the controllers webhook configuration
7276
// +optional
7377
Webhook ControllerWebhook `json:"webhook,omitempty"`
@@ -122,6 +126,17 @@ type ControllerHealth struct {
122126
LivenessEndpointName string `json:"livenessEndpointName,omitempty"`
123127
}
124128

129+
// ControllerProfiling defines profiling configuration for controllers.
130+
type ControllerProfiling struct {
131+
// PprofBindAddress is the TCP address that the controller should bind to
132+
// for serving pprof.
133+
// It can be set to "" or "0" to disable the pprof serving.
134+
// Since pprof may contain sensitive information, make sure to protect it
135+
// before exposing it to public.
136+
// +optional
137+
PprofBindAddress string `json:"pprofBindAddress,omitempty"`
138+
}
139+
125140
// ControllerWebhook defines the webhook server for the controller.
126141
type ControllerWebhook struct {
127142
// Port is the port that the webhook server serves at.

‎pkg/config/v1alpha1/zz_generated.deepcopy.go

Lines changed: 16 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/manager/internal.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"fmt"
2424
"net"
2525
"net/http"
26+
"net/http/pprof"
2627
"sync"
2728
"sync/atomic"
2829
"time"
@@ -107,6 +108,9 @@ type controllerManager struct {
107108
// Healthz probe handler
108109
healthzHandler *healthz.Handler
109110

111+
// pprofListener is used to serve pprof
112+
pprofListener net.Listener
113+
110114
// controllerOptions are the global controller options.
111115
controllerOptions v1alpha1.ControllerConfigurationSpec
112116

@@ -362,6 +366,24 @@ func (cm *controllerManager) serveHealthProbes() {
362366
go cm.httpServe("health probe", cm.logger, server, cm.healthProbeListener)
363367
}
364368

369+
func (cm *controllerManager) addPprofServer() error {
370+
mux := http.NewServeMux()
371+
srv := httpserver.New(mux)
372+
373+
mux.HandleFunc("/debug/pprof/", pprof.Index)
374+
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
375+
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
376+
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
377+
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
378+
379+
return cm.Add(&server{
380+
Kind: "pprof",
381+
Log: cm.logger,
382+
Server: srv,
383+
Listener: cm.pprofListener,
384+
})
385+
}
386+
365387
func (cm *controllerManager) httpServe(kind string, log logr.Logger, server *http.Server, ln net.Listener) {
366388
log = log.WithValues("kind", kind, "addr", ln.Addr())
367389

@@ -457,6 +479,13 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) {
457479
cm.serveHealthProbes()
458480
}
459481

482+
// Add pprof server
483+
if cm.pprofListener != nil {
484+
if err := cm.addPprofServer(); err != nil {
485+
return fmt.Errorf("failed to add pprof server: %w", err)
486+
}
487+
}
488+
460489
// First start any webhook servers, which includes conversion, validation, and defaulting
461490
// webhooks that are registered.
462491
//

‎pkg/manager/manager.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,13 @@ type Options struct {
235235
// Liveness probe endpoint name, defaults to "healthz"
236236
LivenessEndpointName string
237237

238+
// PprofBindAddress is the TCP address that the controller should bind to
239+
// for serving pprof.
240+
// It can be set to "" or "0" to disable the pprof serving.
241+
// Since pprof may contain sensitive information, make sure to protect it
242+
// before exposing it to public.
243+
PprofBindAddress string
244+
238245
// Port is the port that the webhook server serves at.
239246
// It is used to set webhook.Server.Port if WebhookServer is not set.
240247
Port int
@@ -310,6 +317,7 @@ type Options struct {
310317
newResourceLock func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error)
311318
newMetricsListener func(addr string) (net.Listener, error)
312319
newHealthProbeListener func(addr string) (net.Listener, error)
320+
newPprofListener func(addr string) (net.Listener, error)
313321
}
314322

315323
// BaseContextFunc is a function used to provide a base Context to Runnables
@@ -419,6 +427,13 @@ func New(config *rest.Config, options Options) (Manager, error) {
419427
return nil, err
420428
}
421429

430+
// Create pprof listener. This will throw an error if the bind
431+
// address is invalid or already in use.
432+
pprofListener, err := options.newPprofListener(options.PprofBindAddress)
433+
if err != nil {
434+
return nil, fmt.Errorf("failed to new pprof listener: %w", err)
435+
}
436+
422437
errChan := make(chan error)
423438
runnables := newRunnables(options.BaseContext, errChan)
424439

@@ -446,6 +461,7 @@ func New(config *rest.Config, options Options) (Manager, error) {
446461
healthProbeListener: healthProbeListener,
447462
readinessEndpointName: options.ReadinessEndpointName,
448463
livenessEndpointName: options.LivenessEndpointName,
464+
pprofListener: pprofListener,
449465
gracefulShutdownTimeout: *options.GracefulShutdownTimeout,
450466
internalProceduresStop: make(chan struct{}),
451467
leaderElectionStopped: make(chan struct{}),
@@ -495,6 +511,10 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
495511
o.LivenessEndpointName = newObj.Health.LivenessEndpointName
496512
}
497513

514+
if o.PprofBindAddress == "" && newObj.Profiling.PprofBindAddress != "" {
515+
o.PprofBindAddress = newObj.Profiling.PprofBindAddress
516+
}
517+
498518
if o.Port == 0 && newObj.Webhook.Port != nil {
499519
o.Port = *newObj.Webhook.Port
500520
}
@@ -579,6 +599,19 @@ func defaultHealthProbeListener(addr string) (net.Listener, error) {
579599
return ln, nil
580600
}
581601

602+
// defaultPprofListener creates the default pprof listener bound to the given address.
603+
func defaultPprofListener(addr string) (net.Listener, error) {
604+
if addr == "" || addr == "0" {
605+
return nil, nil
606+
}
607+
608+
ln, err := net.Listen("tcp", addr)
609+
if err != nil {
610+
return nil, fmt.Errorf("error listening on %s: %w", addr, err)
611+
}
612+
return ln, nil
613+
}
614+
582615
// defaultBaseContext is used as the BaseContext value in Options if one
583616
// has not already been set.
584617
func defaultBaseContext() context.Context {
@@ -639,6 +672,10 @@ func setOptionsDefaults(options Options) Options {
639672
options.newHealthProbeListener = defaultHealthProbeListener
640673
}
641674

675+
if options.newPprofListener == nil {
676+
options.newPprofListener = defaultPprofListener
677+
}
678+
642679
if options.GracefulShutdownTimeout == nil {
643680
gracefulShutdownTimeout := defaultGracefulShutdownPeriod
644681
options.GracefulShutdownTimeout = &gracefulShutdownTimeout

‎pkg/manager/manager_test.go

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,9 @@ var _ = Describe("manger.Manager", func() {
148148
ReadinessEndpointName: "/readyz",
149149
LivenessEndpointName: "/livez",
150150
},
151+
Profiling: v1alpha1.ControllerProfiling{
152+
PprofBindAddress: ":6080",
153+
},
151154
Webhook: v1alpha1.ControllerWebhook{
152155
Port: &port,
153156
Host: "localhost",
@@ -172,6 +175,7 @@ var _ = Describe("manger.Manager", func() {
172175
Expect(m.HealthProbeBindAddress).To(Equal("6060"))
173176
Expect(m.ReadinessEndpointName).To(Equal("/readyz"))
174177
Expect(m.LivenessEndpointName).To(Equal("/livez"))
178+
Expect(m.PprofBindAddress).To(Equal(":6080"))
175179
Expect(m.Port).To(Equal(port))
176180
Expect(m.Host).To(Equal("localhost"))
177181
Expect(m.CertDir).To(Equal("/certs"))
@@ -204,6 +208,9 @@ var _ = Describe("manger.Manager", func() {
204208
ReadinessEndpointName: "/readyz",
205209
LivenessEndpointName: "/livez",
206210
},
211+
Profiling: v1alpha1.ControllerProfiling{
212+
PprofBindAddress: ":6080",
213+
},
207214
Webhook: v1alpha1.ControllerWebhook{
208215
Port: &port,
209216
Host: "localhost",
@@ -229,6 +236,7 @@ var _ = Describe("manger.Manager", func() {
229236
HealthProbeBindAddress: "5000",
230237
ReadinessEndpointName: "/readiness",
231238
LivenessEndpointName: "/liveness",
239+
PprofBindAddress: ":6000",
232240
Port: 8080,
233241
Host: "example.com",
234242
CertDir: "/pki",
@@ -249,6 +257,7 @@ var _ = Describe("manger.Manager", func() {
249257
Expect(m.HealthProbeBindAddress).To(Equal("5000"))
250258
Expect(m.ReadinessEndpointName).To(Equal("/readiness"))
251259
Expect(m.LivenessEndpointName).To(Equal("/liveness"))
260+
Expect(m.PprofBindAddress).To(Equal(":6000"))
252261
Expect(m.Port).To(Equal(8080))
253262
Expect(m.Host).To(Equal("example.com"))
254263
Expect(m.CertDir).To(Equal("/pki"))
@@ -288,6 +297,7 @@ var _ = Describe("manger.Manager", func() {
288297
LeaderElectionID: "test-leader-election-id-2",
289298
HealthProbeBindAddress: "0",
290299
MetricsBindAddress: "0",
300+
PprofBindAddress: "0",
291301
})
292302
Expect(err).To(BeNil())
293303

@@ -333,6 +343,7 @@ var _ = Describe("manger.Manager", func() {
333343
LeaderElectionID: "test-leader-election-id-3",
334344
HealthProbeBindAddress: "0",
335345
MetricsBindAddress: "0",
346+
PprofBindAddress: "0",
336347
})
337348
Expect(err).To(BeNil())
338349

@@ -367,6 +378,7 @@ var _ = Describe("manger.Manager", func() {
367378
},
368379
HealthProbeBindAddress: "0",
369380
MetricsBindAddress: "0",
381+
PprofBindAddress: "0",
370382
})
371383
Expect(err).ToNot(HaveOccurred())
372384
Expect(m1).ToNot(BeNil())
@@ -387,6 +399,7 @@ var _ = Describe("manger.Manager", func() {
387399
},
388400
HealthProbeBindAddress: "0",
389401
MetricsBindAddress: "0",
402+
PprofBindAddress: "0",
390403
})
391404
Expect(err).ToNot(HaveOccurred())
392405
Expect(m2).ToNot(BeNil())
@@ -1420,6 +1433,94 @@ var _ = Describe("manger.Manager", func() {
14201433
})
14211434
})
14221435

1436+
Context("should start serving pprof", func() {
1437+
var listener net.Listener
1438+
var opts Options
1439+
1440+
BeforeEach(func() {
1441+
listener = nil
1442+
opts = Options{
1443+
newPprofListener: func(addr string) (net.Listener, error) {
1444+
var err error
1445+
listener, err = defaultPprofListener(addr)
1446+
return listener, err
1447+
},
1448+
}
1449+
})
1450+
1451+
AfterEach(func() {
1452+
if listener != nil {
1453+
listener.Close()
1454+
}
1455+
})
1456+
1457+
It("should stop serving pprof when stop is called", func() {
1458+
opts.PprofBindAddress = ":0"
1459+
m, err := New(cfg, opts)
1460+
Expect(err).NotTo(HaveOccurred())
1461+
1462+
ctx, cancel := context.WithCancel(context.Background())
1463+
go func() {
1464+
defer GinkgoRecover()
1465+
Expect(m.Start(ctx)).NotTo(HaveOccurred())
1466+
}()
1467+
<-m.Elected()
1468+
1469+
// Check the pprof started
1470+
endpoint := fmt.Sprintf("http://%s", listener.Addr().String())
1471+
_, err = http.Get(endpoint)
1472+
Expect(err).NotTo(HaveOccurred())
1473+
1474+
// Shutdown the server
1475+
cancel()
1476+
1477+
// Expect the pprof server to shutdown
1478+
Eventually(func() error {
1479+
_, err = http.Get(endpoint)
1480+
return err
1481+
}).ShouldNot(Succeed())
1482+
})
1483+
1484+
It("should serve pprof endpoints", func() {
1485+
opts.PprofBindAddress = ":0"
1486+
m, err := New(cfg, opts)
1487+
Expect(err).NotTo(HaveOccurred())
1488+
1489+
ctx, cancel := context.WithCancel(context.Background())
1490+
defer cancel()
1491+
go func() {
1492+
defer GinkgoRecover()
1493+
Expect(m.Start(ctx)).NotTo(HaveOccurred())
1494+
}()
1495+
<-m.Elected()
1496+
1497+
pprofIndexEndpoint := fmt.Sprintf("http://%s/debug/pprof/", listener.Addr().String())
1498+
resp, err := http.Get(pprofIndexEndpoint)
1499+
Expect(err).NotTo(HaveOccurred())
1500+
Expect(resp.StatusCode).To(Equal(200))
1501+
1502+
pprofCmdlineEndpoint := fmt.Sprintf("http://%s/debug/pprof/cmdline", listener.Addr().String())
1503+
resp, err = http.Get(pprofCmdlineEndpoint)
1504+
Expect(err).NotTo(HaveOccurred())
1505+
Expect(resp.StatusCode).To(Equal(200))
1506+
1507+
pprofProfileEndpoint := fmt.Sprintf("http://%s/debug/pprof/profile", listener.Addr().String())
1508+
resp, err = http.Get(pprofProfileEndpoint)
1509+
Expect(err).NotTo(HaveOccurred())
1510+
Expect(resp.StatusCode).To(Equal(200))
1511+
1512+
pprofSymbolEndpoint := fmt.Sprintf("http://%s/debug/pprof/symbol", listener.Addr().String())
1513+
resp, err = http.Get(pprofSymbolEndpoint)
1514+
Expect(err).NotTo(HaveOccurred())
1515+
Expect(resp.StatusCode).To(Equal(200))
1516+
1517+
pprofTraceEndpoint := fmt.Sprintf("http://%s/debug/pprof/trace", listener.Addr().String())
1518+
resp, err = http.Get(pprofTraceEndpoint)
1519+
Expect(err).NotTo(HaveOccurred())
1520+
Expect(resp.StatusCode).To(Equal(200))
1521+
})
1522+
})
1523+
14231524
Describe("Add", func() {
14241525
It("should immediately start the Component if the Manager has already Started another Component",
14251526
func() {

0 commit comments

Comments
 (0)