validation: Use prove(1) as a TAP harness

Capture stdout and stderr from create invocations, because we don't
want to pollute the TAP output with things like runc's:

  Incorrect Usage.
  ...

(which is for some reason printed to stdout) or:

  runc: "create" requires exactly 1 argument(s)

which is printed to stderr.  Instead, show the captured stderr as a
diagnostic, and hide the stdout completely.  Unless stderr is empty,
in which case show stdout in case it contains something useful.

Most of these tests are broken because we aren't collecting the
container exit code or post-start stdout.  But the tests haven't been
doing that since the create/start split in 15577bd (add runtime
struct; add create test, 2017-08-24, #447) anyway [1].  This commit
just makes that more obvious.

The patsubst and wildcard Makefile syntax is documented in [2].  The
$(VALIDATION_TESTS) rule uses the static pattern rule syntax [3].

And use 'console' instead of 'sh' in the README, because these are
shell sessions, not scripts.  See [4,5].  I don't have a working runc
locally, so the mock results based on a dummy runtime script.

[1]: #447 (comment)
[2]: https://www.gnu.org/software/make/manual/html_node/Wildcard-Function.html
[3]: https://www.gnu.org/software/make/manual/html_node/Static-Usage.html#Static-Usage
[4]: https://help.github.com/articles/creating-and-highlighting-code-blocks/#syntax-highlighting
[5]: https://github.com/github/linguist/blob/v5.3.3/lib/linguist/languages.yml#L4249-L4260

Signed-off-by: W. Trevor King <[email protected]>

Author: wking

Makefile

@@ -5,8 +5,9 @@ BUILDTAGS=
 RUNTIME ?= runc
 COMMIT=$(shell git rev-parse HEAD 2> /dev/null || true)
 VERSION := ${shell cat ./VERSION}
+VALIDATION_TESTS ?= $(patsubst %.go,%.t,$(wildcard validation/*.go))
 
-all: tool runtimetest
+all: tool runtimetest validation-executables
 
 tool:
 	go build -tags "$(BUILDTAGS)" -ldflags "-X main.gitCommit=${COMMIT} -X main.version=${VERSION}" -o oci-runtime-tool ./cmd/oci-runtime-tool
@@ -35,10 +36,18 @@ uninstall:
 	rm -f $(PREFIX)/share/bash-completion/completions/oci-runtime-tool
 
 clean:
-	rm -f oci-runtime-tool runtimetest *.1
+	rm -f oci-runtime-tool runtimetest *.1 $(VALIDATION_TESTS)
 
-localvalidation: runtimetest
-	RUNTIME=$(RUNTIME) go test -tags "$(BUILDTAGS)" ${TESTFLAGS} -v github.com/opencontainers/runtime-tools/validation
+localvalidation:
+	RUNTIME=$(RUNTIME) prove $(VALIDATION_TESTS)
+
+.PHONY: validation-executables
+validation-executables: $(VALIDATION_TESTS)
+
+.PRECIOUS: $(VALIDATION_TESTS)
+.PHONY: $(VALIDATION_TESTS)
+$(VALIDATION_TESTS): %.t: %.go
+	go build -tags "$(BUILDTAGS)" ${TESTFLAGS} -o $@ $<
 
 .PHONY: test .gofmt .govet .golint
 

README.md

@@ -8,7 +8,7 @@ To build from source code, runtime-tools requires Go 1.7.x or above.
 [`oci-runtime-tool generate`][generate.1] generates [configuration JSON][config.json] for an [OCI bundle][bundle].
 [OCI-compatible runtimes][runtime-spec] like [runC][] expect to read the configuration from `config.json`.
 
-```sh
+```console
 $ oci-runtime-tool generate --output config.json
 $ cat config.json
 {
@@ -22,63 +22,114 @@ $ cat config.json
 [`oci-runtime-tool validate`][validate.1] validates an OCI bundle.
 The error message will be printed if the OCI bundle failed the validation procedure.
 
-```sh
+```console
 $ oci-runtime-tool generate
 $ oci-runtime-tool validate
 INFO[0000] Bundle validation succeeded.
 ```
 
 ## Testing OCI runtimes
 
-```sh
+The runtime validation suite uses [`prove`][prove], which is packaged for most distributions (for example, it is in [Debian's `perl` package][debian-perl] and [Gentoo's `dev-lang/perl` package][gentoo-perl]).
+If you cannot install `prove`, you can probably run the test suite with another [TAP consumer][tap-consumers], although you'll have to edit the [`Makefile`](Makefile) to replace `prove`.
+
+```console
+$ make runtimetest validation-executables
 $ sudo make RUNTIME=runc localvalidation
-RUNTIME=runc go test -tags ""  -v github.com/opencontainers/runtime-tools/validation
-=== RUN   TestValidateBasic
-TAP version 13
-ok 1 - root filesystem
-ok 2 - hostname
-ok 3 - mounts
-ok 4 - capabilities
-ok 5 - default symlinks
-ok 6 - default devices
-ok 7 - linux devices
-ok 8 - linux process
-ok 9 - masked paths
-ok 10 - oom score adj
-ok 11 - read only paths
-ok 12 - rlimits
-ok 13 - sysctls
-ok 14 - uid mappings
-ok 15 - gid mappings
-1..15
---- PASS: TestValidateBasic (0.08s)
-=== RUN   TestValidateSysctls
+RUNTIME=runc prove validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t
+validation/linux_rootfs_propagation_shared.t ...... Failed 1/19 subtests
+validation/create.t ............................... ok
+validation/default.t .............................. ok
+validation/linux_readonly_paths.t ................. ok
+validation/linux_masked_paths.t ................... Failed 1/19 subtests
+validation/mounts.t ............................... ok
+validation/process.t .............................. ok
+validation/root_readonly_false.t .................. ok
+validation/linux_sysctl.t ......................... ok
+validation/linux_devices.t ........................ ok
+validation/linux_gid_mappings.t ................... Failed 1/19 subtests
+validation/process_oom_score_adj.t ................ ok
+validation/process_capabilities.t ................. ok
+validation/process_rlimits.t ...................... ok
+validation/root_readonly_true.t ................... ok
+validation/linux_rootfs_propagation_unbindable.t .. failed to create the container
+rootfsPropagation=unbindable is not supported
+exit status 1
+validation/linux_rootfs_propagation_unbindable.t .. Dubious, test returned 1 (wstat 256, 0x100)
+No subtests run
+validation/hostname.t ............................. ok
+validation/linux_uid_mappings.t ................... failed to create the container
+User namespace mappings specified, but USER namespace isn't enabled in the config
+exit status 1
+validation/linux_uid_mappings.t ................... Dubious, test returned 1 (wstat 256, 0x100)
+No subtests run
+
+Test Summary Report
+-------------------
+validation/linux_rootfs_propagation_shared.t    (Wstat: 0 Tests: 19 Failed: 1)
+  Failed test:  16
+validation/linux_masked_paths.t                 (Wstat: 0 Tests: 19 Failed: 1)
+  Failed test:  13
+validation/linux_gid_mappings.t                 (Wstat: 0 Tests: 19 Failed: 1)
+  Failed test:  19
+validation/linux_rootfs_propagation_unbindable.t (Wstat: 256 Tests: 0 Failed: 0)
+  Non-zero exit status: 1
+  Parse errors: No plan found in TAP output
+validation/linux_uid_mappings.t                 (Wstat: 256 Tests: 0 Failed: 0)
+  Non-zero exit status: 1
+  Parse errors: No plan found in TAP output
+Files=18, Tests=271, 31 wallclock secs ( 0.06 usr  0.01 sys +  0.52 cusr  0.21 csys =  0.80 CPU)
+Result: FAIL
+make: *** [Makefile:42: localvalidation] Error 1
+```
+
+If you are confident that the `validation/*.t` are current, you can run `prove` (or your preferred TAP consumer) directly to avoid unnecessary rebuilds:
+
+```console
+$ sudo RUNTIME=runc prove -Q validation/*.t
+
+Test Summary Report
+-------------------
+…
+Files=18, Tests=271, 31 wallclock secs ( 0.07 usr  0.01 sys +  0.51 cusr  0.22 csys =  0.81 CPU)
+Result: FAIL
+```
+
+And you can run an individual test executable directly:
+
+```console
+$ RUNTIME=runc validation/default.t
 TAP version 13
 ok 1 - root filesystem
 ok 2 - hostname
-ok 3 - mounts
-ok 4 - capabilities
-ok 5 - default symlinks
-ok 6 - default devices
-ok 7 - linux devices
-ok 8 - linux process
-ok 9 - masked paths
-ok 10 - oom score adj
-ok 11 - read only paths
-ok 12 - rlimits
-ok 13 - sysctls
-ok 14 - uid mappings
-ok 15 - gid mappings
-1..15
---- PASS: TestValidateSysctls (0.20s)
-PASS
-ok      github.com/opencontainers/runtime-tools/validation      0.281s
+ok 3 - process
+ok 4 - mounts
+ok 5 - user
+ok 6 - rlimits
+ok 7 - capabilities
+ok 8 - default symlinks
+ok 9 - default file system
+ok 10 - default devices
+ok 11 - linux devices
+ok 12 - linux process
+ok 13 - masked paths
+ok 14 - oom score adj
+ok 15 - read only paths
+ok 16 - rootfs propagation
+ok 17 - sysctls
+ok 18 - uid mappings
+ok 19 - gid mappings
+1..19
 ```
 
 [bundle]: https://github.com/opencontainers/runtime-spec/blob/master/bundle.md
 [config.json]: https://github.com/opencontainers/runtime-spec/blob/master/config.md
+[debian-perl]: https://packages.debian.org/stretch/perl
+[gentoo-perl]: https://packages.gentoo.org/packages/dev-lang/perl
+[prove]: http://search.cpan.org/~leont/Test-Harness-3.39/bin/prove
 [runC]: https://github.com/opencontainers/runc
 [runtime-spec]: https://github.com/opencontainers/runtime-spec
+[tap-consumers]: https://testanything.org/consumers.html
 
 [generate.1]: man/oci-runtime-tool-generate.1.md
 [validate.1]: man/oci-runtime-tool-validate.1.md

validation/.gitignore

@@ -0,0 +1 @@
+/*.t

validation/create.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/mndrix/tap-go"
+	rspecs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/generate"
+	"github.com/opencontainers/runtime-tools/specerror"
+	"github.com/opencontainers/runtime-tools/validation/util"
+	"github.com/satori/go.uuid"
+)
+
+func main() {
+	t := tap.New()
+
+	g := generate.New()
+	g.SetRootPath(".")
+	g.SetProcessArgs([]string{"ls"})
+
+	bundleDir, err := util.PrepareBundle()
+	if err != nil {
+		util.Fatal(err)
+	}
+
+	r, err := util.NewRuntime(util.RuntimeCommand, bundleDir)
+	if err != nil {
+		util.Fatal(err)
+	}
+	defer r.Clean(true)
+
+	err = r.SetConfig(&g)
+	if err != nil {
+		util.Fatal(err)
+	}
+
+	containerID := uuid.NewV4().String()
+	cases := []struct {
+		id          string
+		errExpected bool
+		err         error
+	}{
+		{"", false, specerror.NewError(specerror.CreateWithBundlePathAndID, fmt.Errorf("create MUST generate an error if the ID is not provided"), rspecs.Version)},
+		{containerID, true, specerror.NewError(specerror.CreateNewContainer, fmt.Errorf("create MUST create a new container"), rspecs.Version)},
+		{containerID, false, specerror.NewError(specerror.CreateWithUniqueID, fmt.Errorf("create MUST generate an error if the ID provided is not unique"), rspecs.Version)},
+	}
+
+	for _, c := range cases {
+		r.SetID(c.id)
+		stderr, err := r.Create()
+		t.Ok((err == nil) == c.errExpected, c.err.(*specerror.Error).Err.Err.Error())
+		t.Diagnostic(c.err.(*specerror.Error).Err.Reference)
+		if err != nil {
+			t.Diagnostic(err.Error())
+		}
+		if len(stderr) > 0 {
+			t.Diagnostic(string(stderr))
+		}
+
+		if err == nil {
+			state, _ := r.State()
+			t.Ok(state.ID == c.id, "")
+			t.Diagnosticf("container PID: %d, state ID: %d", c.id, state.ID)
+		}
+	}
+
+	t.AutoPlan()
+}

validation/default.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/hostname.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.SetHostname("hostname-specific")
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_devices.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"os"
+
+	rspecs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+
+	// add char device
+	cdev := rspecs.LinuxDevice{}
+	cdev.Path = "/dev/test1"
+	cdev.Type = "c"
+	cdev.Major = 10
+	cdev.Minor = 666
+	cmode := os.FileMode(int32(432))
+	cdev.FileMode = &cmode
+	cuid := uint32(0)
+	cdev.UID = &cuid
+	cgid := uint32(0)
+	cdev.GID = &cgid
+	g.AddDevice(cdev)
+
+	// add block device
+	bdev := rspecs.LinuxDevice{}
+	bdev.Path = "/dev/test2"
+	bdev.Type = "b"
+	bdev.Major = 8
+	bdev.Minor = 666
+	bmode := os.FileMode(int32(432))
+	bdev.FileMode = &bmode
+	uid := uint32(0)
+	bdev.UID = &uid
+	gid := uint32(0)
+	bdev.GID = &gid
+	g.AddDevice(bdev)
+
+	// add fifo device
+	pdev := rspecs.LinuxDevice{}
+	pdev.Path = "/dev/test3"
+	pdev.Type = "p"
+	pdev.Major = 8
+	pdev.Minor = 666
+	pmode := os.FileMode(int32(432))
+	pdev.FileMode = &pmode
+	g.AddDevice(pdev)
+
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_gid_mappings.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.AddLinuxGIDMapping(uint32(1000), uint32(0), uint32(3200))
+	err := util.RuntimeInsideValidate(g, nil)
+	if err != nil {
+		util.Fatal(err)
+	}
+}

validation/linux_masked_paths.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/opencontainers/runtime-tools/validation/util"
+)
+
+func main() {
+	g := util.GetDefaultGenerator()
+	g.AddLinuxMaskedPaths("/masktest")
+	err := util.RuntimeInsideValidate(g, func(path string) error {
+		pathName := filepath.Join(path, "masktest")
+		return os.MkdirAll(pathName, 0700)
+	})
+	if err != nil {
+		util.Fatal(err)
+	}
+}