runc-facade: squashes rare runc errors

Author: utam0k

components/docker-up/docker-up/main.go

@@ -59,6 +59,7 @@ var aptUpdated = false
 const (
 	dockerSocketFN = "/var/run/docker.sock"
 	gitpodUserId   = 33333
+	containerIf    = "ceth0"
 )
 
 func main() {
@@ -135,7 +136,6 @@ func runWithinNetns() (err error) {
 	}
 	args = append(args, userArgs...)
 
-	containerIf := "ceth0"
 	netIface, err := netlink.LinkByName(containerIf)
 	if err != nil {
 		return xerrors.Errorf("cannot get container network device %s: %w", containerIf, err)

components/docker-up/runc-facade/main.go

@@ -15,6 +15,8 @@ import (
 	"golang.org/x/xerrors"
 )
 
+const RETRY = 3
+
 var (
 	defaultOOMScoreAdj = 1000
 )
@@ -86,9 +88,15 @@ func createAndRunc(runcPath string, log *logrus.Logger) error {
 		}
 	}
 
-	err = syscall.Exec(runcPath, os.Args, os.Environ())
-	if err != nil {
-		return xerrors.Errorf("exec %s: %w", runcPath, err)
+	// See here for more details on why retries are necessary.
+	// https://github.com/gitpod-io/gitpod/issues/12365
+	for i := 0; i <= RETRY; i++ {
+		err = syscall.Exec(runcPath, os.Args, os.Environ())
+		if err == nil {
+			return err
+		} else {
+			log.WithError(err).Warn("runc failed")
+		}
 	}
-	return nil
+	return xerrors.Errorf("exec %s: %w", runcPath, err)
 }

components/image-builder-bob/cmd/runc-facade/main.go

@@ -11,11 +11,14 @@ import (
 	"path/filepath"
 	"syscall"
 
+	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/xerrors"
 )
 
+const RETRY = 2
+
 func main() {
 	log := logrus.New()
 	log.SetLevel(logrus.DebugLevel)
@@ -95,9 +98,16 @@ func createAndRunc(runcPath, bundle string) error {
 		}
 	}
 
-	err = syscall.Exec(runcPath, os.Args, os.Environ())
-	if err != nil {
-		return xerrors.Errorf("exec %s: %w", runcPath, err)
+	// See here for more details on why retries are necessary.
+	// https://github.com/gitpod-io/gitpod/issues/12365
+	for i := 0; i <= RETRY; i++ {
+		err = syscall.Exec(runcPath, os.Args, os.Environ())
+		if err == nil {
+			return err
+		} else {
+			log.WithError(err).Warn("runc failed")
+		}
 	}
-	return nil
+
+	return xerrors.Errorf("exec %s: %w", runcPath, err)
 }