Merge pull request #1308 from giuseppe/fix-systemd-notify

fix systemd-notify when using a different PID namespace

Author: Mrunal Patel

notify_socket.go

@@ -0,0 +1,108 @@
+// +build linux
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"path/filepath"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/urfave/cli"
+)
+
+type notifySocket struct {
+	socket     *net.UnixConn
+	host       string
+	socketPath string
+}
+
+func newNotifySocket(context *cli.Context, notifySocketHost string, id string) *notifySocket {
+	if notifySocketHost == "" {
+		return nil
+	}
+
+	root := filepath.Join(context.GlobalString("root"), id)
+	path := filepath.Join(root, "notify.sock")
+
+	notifySocket := &notifySocket{
+		socket:     nil,
+		host:       notifySocketHost,
+		socketPath: path,
+	}
+
+	return notifySocket
+}
+
+func (ns *notifySocket) Close() error {
+	return ns.socket.Close()
+}
+
+// If systemd is supporting sd_notify protocol, this function will add support
+// for sd_notify protocol from within the container.
+func (s *notifySocket) setupSpec(context *cli.Context, spec *specs.Spec) {
+	mount := specs.Mount{Destination: s.host, Type: "bind", Source: s.socketPath, Options: []string{"bind"}}
+	spec.Mounts = append(spec.Mounts, mount)
+	spec.Process.Env = append(spec.Process.Env, fmt.Sprintf("NOTIFY_SOCKET=%s", s.host))
+}
+
+func (s *notifySocket) setupSocket() error {
+	addr := net.UnixAddr{
+		Name: s.socketPath,
+		Net:  "unixgram",
+	}
+
+	socket, err := net.ListenUnixgram("unixgram", &addr)
+	if err != nil {
+		return err
+	}
+
+	s.socket = socket
+	return nil
+}
+
+// pid1 must be set only with -d, as it is used to set the new process as the main process
+// for the service in systemd
+func (notifySocket *notifySocket) run(pid1 int) {
+	buf := make([]byte, 512)
+	notifySocketHostAddr := net.UnixAddr{Name: notifySocket.host, Net: "unixgram"}
+	client, err := net.DialUnix("unixgram", nil, &notifySocketHostAddr)
+	if err != nil {
+		logrus.Error(err)
+		return
+	}
+	for {
+		r, err := notifySocket.socket.Read(buf)
+		if err != nil {
+			break
+		}
+		var out bytes.Buffer
+		for _, line := range bytes.Split(buf[0:r], []byte{'\n'}) {
+			if bytes.HasPrefix(line, []byte("READY=")) {
+				_, err = out.Write(line)
+				if err != nil {
+					return
+				}
+
+				_, err = out.Write([]byte{'\n'})
+				if err != nil {
+					return
+				}
+
+				_, err = client.Write(out.Bytes())
+				if err != nil {
+					return
+				}
+
+				// now we can inform systemd to use pid1 as the pid to monitor
+				if pid1 > 0 {
+					newPid := fmt.Sprintf("MAINPID=%d\n", pid1)
+					client.Write([]byte(newPid))
+				}
+				return
+			}
+		}
+	}
+}

restore.go

@@ -165,7 +165,14 @@ func restoreContainer(context *cli.Context, spec *specs.Spec, config *configs.Co
 	if err != nil {
 		return -1, err
 	}
-	handler := newSignalHandler(!context.Bool("no-subreaper"))
+
+	notifySocket := newNotifySocket(context, os.Getenv("NOTIFY_SOCKET"), id)
+	if notifySocket != nil {
+		notifySocket.setupSpec(context, spec)
+		notifySocket.setupSocket()
+	}
+
+	handler := newSignalHandler(!context.Bool("no-subreaper"), notifySocket)
 	if err := container.Restore(process, options); err != nil {
 		return -1, err
 	}
@@ -181,10 +188,7 @@ func restoreContainer(context *cli.Context, spec *specs.Spec, config *configs.Co
 			return -1, err
 		}
 	}
-	if detach {
-		return 0, nil
-	}
-	return handler.forward(process, tty)
+	return handler.forward(process, tty, detach)
 }
 
 func criuOptions(context *cli.Context) *libcontainer.CriuOpts {

signals.go

@@ -17,7 +17,9 @@ const signalBufferSize = 2048
 
 // newSignalHandler returns a signal handler for processing SIGCHLD and SIGWINCH signals
 // while still forwarding all other signals to the process.
-func newSignalHandler(enableSubreaper bool) *signalHandler {
+// If notifySocket is present, use it to read systemd notifications from the container and
+// forward them to notifySocketHost.
+func newSignalHandler(enableSubreaper bool, notifySocket *notifySocket) *signalHandler {
 	if enableSubreaper {
 		// set us as the subreaper before registering the signal handler for the container
 		if err := system.SetSubreaper(1); err != nil {
@@ -30,7 +32,8 @@ func newSignalHandler(enableSubreaper bool) *signalHandler {
 	// handle all signals for the process.
 	signal.Notify(s)
 	return &signalHandler{
-		signals: s,
+		signals:      s,
+		notifySocket: notifySocket,
 	}
 }
 
@@ -42,18 +45,33 @@ type exit struct {
 }
 
 type signalHandler struct {
-	signals chan os.Signal
+	signals      chan os.Signal
+	notifySocket *notifySocket
 }
 
 // forward handles the main signal event loop forwarding, resizing, or reaping depending
 // on the signal received.
-func (h *signalHandler) forward(process *libcontainer.Process, tty *tty) (int, error) {
+func (h *signalHandler) forward(process *libcontainer.Process, tty *tty, detach bool) (int, error) {
 	// make sure we know the pid of our main process so that we can return
 	// after it dies.
+	if detach && h.notifySocket == nil {
+		return 0, nil
+	}
+
 	pid1, err := process.Pid()
 	if err != nil {
 		return -1, err
 	}
+
+	if h.notifySocket != nil {
+		if detach {
+			h.notifySocket.run(pid1)
+			return 0, nil
+		} else {
+			go h.notifySocket.run(0)
+		}
+	}
+
 	// perform the initial tty resize.
 	tty.resize()
 	for s := range h.signals {
@@ -75,6 +93,9 @@ func (h *signalHandler) forward(process *libcontainer.Process, tty *tty) (int, e
 					// status because we must ensure that any of the go specific process
 					// fun such as flushing pipes are complete before we return.
 					process.Wait()
+					if h.notifySocket != nil {
+						h.notifySocket.Close()
+					}
 					return e.status, nil
 				}
 			}

utils.go

@@ -63,10 +63,6 @@ func setupSpec(context *cli.Context) (*specs.Spec, error) {
 	if err != nil {
 		return nil, err
 	}
-	notifySocket := os.Getenv("NOTIFY_SOCKET")
-	if notifySocket != "" {
-		setupSdNotify(spec, notifySocket)
-	}
 	if os.Geteuid() != 0 {
 		return nil, fmt.Errorf("runc should be run as root")
 	}

utils_linux.go

@@ -95,13 +95,6 @@ func newProcess(p specs.Process) (*libcontainer.Process, error) {
 	return lp, nil
 }
 
-// If systemd is supporting sd_notify protocol, this function will add support
-// for sd_notify protocol from within the container.
-func setupSdNotify(spec *specs.Spec, notifySocket string) {
-	spec.Mounts = append(spec.Mounts, specs.Mount{Destination: notifySocket, Type: "bind", Source: notifySocket, Options: []string{"bind"}})
-	spec.Process.Env = append(spec.Process.Env, fmt.Sprintf("NOTIFY_SOCKET=%s", notifySocket))
-}
-
 func destroy(container libcontainer.Container) {
 	if err := container.Destroy(); err != nil {
 		logrus.Error(err)
@@ -185,6 +178,7 @@ type runner struct {
 	consoleSocket   string
 	container       libcontainer.Container
 	create          bool
+	notifySocket    *notifySocket
 }
 
 func (r *runner) terminalinfo() *libcontainer.TerminalInfo {
@@ -240,7 +234,7 @@ func (r *runner) run(config *specs.Process) (int, error) {
 	// Setting up IO is a two stage process. We need to modify process to deal
 	// with detaching containers, and then we get a tty after the container has
 	// started.
-	handler := newSignalHandler(r.enableSubreaper)
+	handler := newSignalHandler(r.enableSubreaper, r.notifySocket)
 	tty, err := setupIO(process, rootuid, rootgid, config.Terminal, detach)
 	if err != nil {
 		r.destroy()
@@ -303,13 +297,13 @@ func (r *runner) run(config *specs.Process) (int, error) {
 			return -1, err
 		}
 	}
-	if detach {
-		return 0, nil
-	}
-	status, err := handler.forward(process, tty)
+	status, err := handler.forward(process, tty, detach)
 	if err != nil {
 		r.terminate(process)
 	}
+	if detach {
+		return 0, nil
+	}
 	r.destroy()
 	return status, err
 }
@@ -343,10 +337,21 @@ func startContainer(context *cli.Context, spec *specs.Spec, create bool) (int, e
 	if id == "" {
 		return -1, errEmptyID
 	}
+
+	notifySocket := newNotifySocket(context, os.Getenv("NOTIFY_SOCKET"), id)
+	if notifySocket != nil {
+		notifySocket.setupSpec(context, spec)
+	}
+
 	container, err := createContainer(context, id, spec)
 	if err != nil {
 		return -1, err
 	}
+
+	if notifySocket != nil {
+		notifySocket.setupSocket()
+	}
+
 	// Support on-demand socket activation by passing file descriptors into the container init process.
 	listenFDs := []*os.File{}
 	if os.Getenv("LISTEN_FDS") != "" {
@@ -357,6 +362,7 @@ func startContainer(context *cli.Context, spec *specs.Spec, create bool) (int, e
 		shouldDestroy:   true,
 		container:       container,
 		listenFDs:       listenFDs,
+		notifySocket:    notifySocket,
 		consoleSocket:   context.String("console-socket"),
 		detach:          context.Bool("detach"),
 		pidFile:         context.String("pid-file"),