add loopbacktime to reduce time of journalctl call

Author: smileusd

cmd/healthchecker/options/options.go

@@ -39,6 +39,7 @@ type HealthCheckerOptions struct {
 	CriCtlPath         string
 	CriSocketPath      string
 	CoolDownTime       time.Duration
+	LoopBackTime       time.Duration
 	HealthCheckTimeout time.Duration
 	LogPatterns        types.LogPatternFlag
 }
@@ -56,6 +57,8 @@ func (hco *HealthCheckerOptions) AddFlags(fs *pflag.FlagSet) {
 		"The path to the cri socket. Used with crictl to specify the socket path.")
 	fs.DurationVar(&hco.CoolDownTime, "cooldown-time", types.DefaultCoolDownTime,
 		"The duration to wait for the service to be up before attempting repair.")
+	fs.DurationVar(&hco.LoopBackTime, "loopback-time", types.DefaultLoopBackTime,
+		"The duration to loop back, if it is 0, health-check will check from start time.")
 	fs.DurationVar(&hco.HealthCheckTimeout, "health-check-timeout", types.DefaultHealthCheckTimeout,
 		"The time to wait before marking the component as unhealthy.")
 	fs.Var(&hco.LogPatterns, "log-pattern",

config/health-checker-kubelet.json

@@ -25,6 +25,7 @@
         "--component=kubelet",
         "--enable-repair=true",
         "--cooldown-time=1m",
+        "--loopback-time=5m",
         "--health-check-timeout=10s"
       ],
       "timeout": "3m"

pkg/healthchecker/health_checker.go

@@ -43,6 +43,7 @@ type healthChecker struct {
 	crictlPath         string
 	healthCheckTimeout time.Duration
 	coolDownTime       time.Duration
+	loopBackTime       time.Duration
 	logPatternsToCheck map[string]int
 }
 
@@ -54,6 +55,7 @@ func NewHealthChecker(hco *options.HealthCheckerOptions) (types.HealthChecker, e
 		crictlPath:         hco.CriCtlPath,
 		healthCheckTimeout: hco.HealthCheckTimeout,
 		coolDownTime:       hco.CoolDownTime,
+		loopBackTime:       hco.LoopBackTime,
 		systemdService:     hco.SystemdService,
 		logPatternsToCheck: hco.LogPatterns.GetLogPatternCountMap(),
 	}
@@ -139,11 +141,21 @@ func getHealthCheckFunc(hco *options.HealthCheckerOptions) func() (bool, error)
 // CheckHealth checks for the health of the component and tries to repair if enabled.
 // Returns true if healthy, false otherwise.
 func (hc *healthChecker) CheckHealth() (bool, error) {
+	var logStartTime string
 	healthy, err := hc.healthCheckFunc()
 	if err != nil {
 		return healthy, err
 	}
-	logPatternHealthy, err := logPatternHealthCheck(hc.systemdService, hc.logPatternsToCheck)
+	uptime, err := hc.uptimeFunc()
+	if err != nil {
+		return false, err
+	}
+	if hc.loopBackTime > 0 && uptime > hc.loopBackTime {
+		logStartTime = time.Now().Add(-hc.loopBackTime).Format(types.LogParsingTimeLayout)
+	} else {
+		logStartTime = time.Now().Add(-uptime).Format(types.LogParsingTimeLayout)
+	}
+	logPatternHealthy, err := logPatternHealthCheck(hc.systemdService, logStartTime, hc.logPatternsToCheck)
 	if err != nil {
 		return logPatternHealthy, err
 	}
@@ -154,10 +166,6 @@ func (hc *healthChecker) CheckHealth() (bool, error) {
 	// Attempt repair based on flag.
 	if hc.enableRepair {
 		// repair if the service has been up for the cool down period.
-		uptime, err := hc.uptimeFunc()
-		if err != nil {
-			glog.Infof("error in getting uptime for %v: %v\n", hc.component, err)
-		}
 		glog.Infof("%v is unhealthy, component uptime: %v\n", hc.component, uptime)
 		if uptime > hc.coolDownTime {
 			glog.Infof("%v cooldown period of %v exceeded, repairing", hc.component, hc.coolDownTime)
@@ -182,19 +190,10 @@ func execCommand(timeout time.Duration, command string, args ...string) (string,
 
 // logPatternHealthCheck checks for the provided logPattern occurrences in the service logs.
 // Returns true if the pattern is empty or does not exist logThresholdCount times since start of service, false otherwise.
-func logPatternHealthCheck(service string, logPatternsToCheck map[string]int) (bool, error) {
+func logPatternHealthCheck(service, logStartTime string, logPatternsToCheck map[string]int) (bool, error) {
 	if len(logPatternsToCheck) == 0 {
 		return true, nil
 	}
-	uptimeFunc := getUptimeFunc(service)
-	uptime, err := uptimeFunc()
-	if err != nil {
-		return true, err
-	}
-	logStartTime := time.Now().Add(-uptime).Format(types.LogParsingTimeLayout)
-	if err != nil {
-		return true, err
-	}
 	for pattern, count := range logPatternsToCheck {
 		healthy, err := checkForPattern(service, logStartTime, pattern, count)
 		if err != nil || !healthy {

pkg/healthchecker/types/types.go

@@ -25,6 +25,7 @@ import (
 )
 
 const (
+	DefaultLoopBackTime       = 0 * time.Minute
 	DefaultCoolDownTime       = 2 * time.Minute
 	DefaultHealthCheckTimeout = 10 * time.Second
 	CmdTimeout                = 10 * time.Second