Skip to content

Commit 3660ed5

Browse files
utam0kroboquat
utam0k
authored andcommitted
runc-facade: squashes rare runc errors
1 parent 257a1f3 commit 3660ed5

File tree

3 files changed

+27
-9
lines changed

3 files changed

+27
-9
lines changed

components/docker-up/docker-up/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ var aptUpdated = false
5959
const (
6060
dockerSocketFN = "/var/run/docker.sock"
6161
gitpodUserId = 33333
62+
containerIf = "ceth0"
6263
)
6364

6465
func main() {
@@ -135,7 +136,6 @@ func runWithinNetns() (err error) {
135136
}
136137
args = append(args, userArgs...)
137138

138-
containerIf := "ceth0"
139139
netIface, err := netlink.LinkByName(containerIf)
140140
if err != nil {
141141
return xerrors.Errorf("cannot get container network device %s: %w", containerIf, err)

components/docker-up/runc-facade/main.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import (
1515
"golang.org/x/xerrors"
1616
)
1717

18+
const RETRY = 3
19+
1820
var (
1921
defaultOOMScoreAdj = 1000
2022
)
@@ -86,9 +88,15 @@ func createAndRunc(runcPath string, log *logrus.Logger) error {
8688
}
8789
}
8890

89-
err = syscall.Exec(runcPath, os.Args, os.Environ())
90-
if err != nil {
91-
return xerrors.Errorf("exec %s: %w", runcPath, err)
91+
// See here for more details on why retries are necessary.
92+
// https://github.com/gitpod-io/gitpod/issues/12365
93+
for i := 0; i <= RETRY; i++ {
94+
err = syscall.Exec(runcPath, os.Args, os.Environ())
95+
if err == nil {
96+
return err
97+
} else {
98+
log.WithError(err).Warn("runc failed")
99+
}
92100
}
93-
return nil
101+
return xerrors.Errorf("exec %s: %w", runcPath, err)
94102
}

components/image-builder-bob/cmd/runc-facade/main.go

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,14 @@ import (
1111
"path/filepath"
1212
"syscall"
1313

14+
"github.com/gitpod-io/gitpod/common-go/log"
1415
"github.com/opencontainers/runtime-spec/specs-go"
1516
"github.com/sirupsen/logrus"
1617
"golang.org/x/xerrors"
1718
)
1819

20+
const RETRY = 2
21+
1922
func main() {
2023
log := logrus.New()
2124
log.SetLevel(logrus.DebugLevel)
@@ -95,9 +98,16 @@ func createAndRunc(runcPath, bundle string) error {
9598
}
9699
}
97100

98-
err = syscall.Exec(runcPath, os.Args, os.Environ())
99-
if err != nil {
100-
return xerrors.Errorf("exec %s: %w", runcPath, err)
101+
// See here for more details on why retries are necessary.
102+
// https://github.com/gitpod-io/gitpod/issues/12365
103+
for i := 0; i <= RETRY; i++ {
104+
err = syscall.Exec(runcPath, os.Args, os.Environ())
105+
if err == nil {
106+
return err
107+
} else {
108+
log.WithError(err).Warn("runc failed")
109+
}
101110
}
102-
return nil
111+
112+
return xerrors.Errorf("exec %s: %w", runcPath, err)
103113
}

0 commit comments

Comments
 (0)