From 20a71e4540b01200a69f4cf55047175e4086bfd4 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 11 Jan 2017 13:06:24 -0800 Subject: [PATCH 1/4] runtimetest: Make TAP output more granular Granular TAP output makes it easier to see what's being tested (and what's going wrong). You'll want a TAP harness looking for test failures, because runtimetest now returns a lot of information, most of which is (hopefully ;) about passing tests. In order to accomplish this change, I've setup a complianceTester structure to make it easy to pass the TAP harness down into the helper functions. complianceTester also holds the target compliance level, so it can decide whether a spec violation is sufficiently serious to be a failed test violoation or is minor enough to be a warning. I've used skips for the warnings, since they seemed like the closest fit from the TAP spec. And I've made a number of other improvements while I was working through this, but haven't written them all up in this commit message. One change is the new support for testing file write access; previously we only tested directory write access. Its possible that some WriteFile errors should kill the test suite, but for now I'm counting all of them as successful "cannot write" determinations. Signed-off-by: W. Trevor King --- cmd/runtimetest/main.go | 989 ++++++++++++++++++++++++++-------------- error/error.go | 30 ++ 2 files changed, 666 insertions(+), 353 deletions(-) diff --git a/cmd/runtimetest/main.go b/cmd/runtimetest/main.go index 765f35394..a8809ed07 100644 --- a/cmd/runtimetest/main.go +++ b/cmd/runtimetest/main.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -14,7 +15,6 @@ import ( "strings" "syscall" - "github.com/hashicorp/go-multierror" "github.com/mndrix/tap-go" rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" @@ -57,22 +57,76 @@ var ( "/dev/stderr": "/proc/self/fd/2", } - defaultDevices = []string{ - "/dev/null", - "/dev/zero", - "/dev/full", - "/dev/random", - "/dev/urandom", - "/dev/tty", - "/dev/ptmx", + defaultDevices = []rspec.LinuxDevice{ + { + Path: "/dev/null", + Type: "c", + Major: 1, + Minor: 3, + }, + { + Path: "/dev/zero", + Type: "c", + Major: 1, + Minor: 5, + }, + { + Path: "/dev/full", + Type: "c", + Major: 1, + Minor: 7, + }, + { + Path: "/dev/random", + Type: "c", + Major: 1, + Minor: 8, + }, + { + Path: "/dev/urandom", + Type: "c", + Major: 1, + Minor: 9, + }, + { + Path: "/dev/tty", + Type: "c", + Major: 5, + Minor: 0, + }, + { + Path: "/dev/ptmx", + Type: "c", + Major: 5, + Minor: 2, + }, } ) -type validation struct { - test func(*rspec.Spec, *tap.T) error - description string +type complianceTester struct { + harness *tap.T + complianceLevel rfc2119.Level } +func (c *complianceTester) Ok(test bool, condition specerror.Code, version string, description string) (rfcError *rfc2119.Error, err error) { + err = specerror.NewError(condition, errors.New(description), version) + runtimeError, ok := err.(*specerror.Error) + if !ok { + return nil, fmt.Errorf("cannot convert %v to a runtime-spec error", err) + } + rfcError = &runtimeError.Err + if test { + c.harness.Pass(description) + } else if runtimeError.Err.Level < c.complianceLevel { + c.harness.Skip(1, description) + } else { + c.harness.Fail(description) + } + return rfcError, nil +} + +type validator func(config *rspec.Spec) (err error) + func loadSpecConfig(path string) (spec *rspec.Spec, err error) { configPath := filepath.Join(path, specConfig) cf, err := os.Open(configPath) @@ -91,19 +145,24 @@ func loadSpecConfig(path string) (spec *rspec.Spec, err error) { return spec, nil } -func validatePosixUser(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validatePosixUser(spec *rspec.Spec) error { if spec.Process == nil { return nil } - uid := os.Getuid() - if uint32(uid) != spec.Process.User.UID { - return fmt.Errorf("UID expected: %v, actual: %v", spec.Process.User.UID, uid) - } - gid := os.Getgid() - if uint32(gid) != spec.Process.User.GID { - return fmt.Errorf("GID expected: %v, actual: %v", spec.Process.User.GID, gid) - } + uid := uint32(os.Getuid()) + c.harness.Ok(uid == spec.Process.User.UID, "has expected user ID") + c.harness.YAML(map[string]uint32{ + "expected": spec.Process.User.UID, + "actual": uid, + }) + + gid := uint32(os.Getgid()) + c.harness.Ok(gid == spec.Process.User.GID, "has expected group ID") + c.harness.YAML(map[string]uint32{ + "expected": spec.Process.User.GID, + "actual": gid, + }) groups, err := os.Getgroups() if err != nil { @@ -116,27 +175,30 @@ func validatePosixUser(spec *rspec.Spec, t *tap.T) error { } for _, g := range spec.Process.User.AdditionalGids { - if !groupsMap[int(g)] { - return fmt.Errorf("Groups expected: %v, actual (should be superset): %v", spec.Process.User.AdditionalGids, groups) - } + c.harness.Ok(groupsMap[int(g)], fmt.Sprintf("has expected additional group ID %v", g)) } return nil } -func validateProcess(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validateProcess(spec *rspec.Spec) error { if spec.Process == nil { + c.harness.Skip(1, "process not set") return nil } - if spec.Process.Cwd != "" { + if spec.Process.Cwd == "" { + c.harness.Skip(1, "process.cwd not set") + } else { cwd, err := os.Getwd() if err != nil { return err } - if cwd != spec.Process.Cwd { - return fmt.Errorf("Cwd expected: %v, actual: %v", spec.Process.Cwd, cwd) - } + c.harness.Ok(cwd == spec.Process.Cwd, "has expected working directory") + c.harness.YAML(map[string]string{ + "expected": spec.Process.Cwd, + "actual": cwd, + }) } for _, env := range spec.Process.Env { @@ -144,16 +206,20 @@ func validateProcess(spec *rspec.Spec, t *tap.T) error { key := parts[0] expectedValue := parts[1] actualValue := os.Getenv(key) - if actualValue != expectedValue { - return fmt.Errorf("Env %v expected: %v, actual: %v", key, expectedValue, actualValue) - } + c.harness.Ok(expectedValue == actualValue, fmt.Sprintf("has expected environment variable %v", key)) + c.harness.YAML(map[string]string{ + "variable": key, + "expected": expectedValue, + "actual": actualValue, + }) } return nil } -func validateLinuxProcess(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validateLinuxProcess(spec *rspec.Spec) error { if spec.Process == nil { + c.harness.Skip(1, "process not set") return nil } @@ -163,31 +229,33 @@ func validateLinuxProcess(spec *rspec.Spec, t *tap.T) error { } args := bytes.Split(bytes.Trim(cmdlineBytes, "\x00"), []byte("\x00")) - if len(args) != len(spec.Process.Args) { - return fmt.Errorf("Process arguments expected: %v, actual: %v", len(spec.Process.Args), len(args)) - } + c.harness.Ok(len(args) == len(spec.Process.Args), "has expected number of process arguments") + c.harness.YAML(map[string]interface{}{ + "expected": spec.Process.Args, + "actual": args, + }) for i, a := range args { - if string(a) != spec.Process.Args[i] { - return fmt.Errorf("Process arguments expected: %v, actual: %v", string(a), spec.Process.Args[i]) - } + c.harness.Ok(string(a) == spec.Process.Args[i], fmt.Sprintf("has expected process argument %d", i)) + c.harness.YAML(map[string]interface{}{ + "index": i, + "expected": spec.Process.Args[i], + "actual": string(a), + }) } ret, _, errno := syscall.Syscall6(syscall.SYS_PRCTL, PrGetNoNewPrivs, 0, 0, 0, 0, 0) if errno != 0 { return errno } - if spec.Process.NoNewPrivileges && ret != 1 { - return fmt.Errorf("NoNewPrivileges expected: true, actual: false") - } - if !spec.Process.NoNewPrivileges && ret != 0 { - return fmt.Errorf("NoNewPrivileges expected: false, actual: true") - } + noNewPrivileges := ret == 1 + c.harness.Ok(spec.Process.NoNewPrivileges == noNewPrivileges, "has expected noNewPrivileges") return nil } -func validateCapabilities(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validateCapabilities(spec *rspec.Spec) error { if spec.Process == nil || spec.Process.Capabilities == nil { + c.harness.Skip(1, "process.capabilities not set") return nil } @@ -240,10 +308,10 @@ func validateCapabilities(spec *rspec.Spec, t *tap.T) error { capKey := fmt.Sprintf("CAP_%s", strings.ToUpper(cap.String())) expectedSet := expectedCaps[capKey] actuallySet := processCaps.Get(capType.capType, cap) - if expectedSet && !actuallySet { - return fmt.Errorf("expected %s capability %v not set", capType.capType, capKey) - } else if !expectedSet && actuallySet { - return fmt.Errorf("unexpected %s capability %v set", capType.capType, capKey) + if expectedSet { + c.harness.Ok(actuallySet, fmt.Sprintf("expected %s capability %v set", capType.capType, capKey)) + } else { + c.harness.Ok(!actuallySet, fmt.Sprintf("unexpected %s capability %v not set", capType.capType, capKey)) } } } @@ -251,19 +319,27 @@ func validateCapabilities(spec *rspec.Spec, t *tap.T) error { return nil } -func validateHostname(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validateHostname(spec *rspec.Spec) error { + if spec.Hostname == "" { + c.harness.Skip(1, "hostname not set") + return nil + } + hostname, err := os.Hostname() if err != nil { return err } - if spec.Hostname != "" && hostname != spec.Hostname { - return fmt.Errorf("Hostname expected: %v, actual: %v", spec.Hostname, hostname) - } + c.harness.Ok(spec.Hostname == hostname, "has expected hostname") + c.harness.YAML(map[string]string{ + "expected": spec.Hostname, + "actual": hostname, + }) return nil } -func validateRlimits(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validateRlimits(spec *rspec.Spec) error { if spec.Process == nil { + c.harness.Skip(1, "process.rlimits not set") return nil } @@ -278,20 +354,39 @@ func validateRlimits(spec *rspec.Spec, t *tap.T) error { return err } - if rlimit.Cur != r.Soft { - return specerror.NewError(specerror.PosixProcRlimitsSoftMatchCur, fmt.Errorf("%v rlimit soft expected: %v, actual: %v", r.Type, r.Soft, rlimit.Cur), rspec.Version) + rfcError, err := c.Ok(rlimit.Cur == r.Soft, specerror.PosixProcRlimitsSoftMatchCur, spec.Version, fmt.Sprintf("has expected soft %v", r.Type)) + if err != nil { + return err } - if rlimit.Max != r.Hard { - return specerror.NewError(specerror.PosixProcRlimitsHardMatchMax, fmt.Errorf("%v rlimit hard expected: %v, actual: %v", r.Type, r.Hard, rlimit.Max), rspec.Version) + c.harness.YAML(map[string]interface{}{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "type": r.Type, + "expected": r.Soft, + "actual": rlimit.Cur, + }) + + rfcError, err = c.Ok(rlimit.Max == r.Hard, specerror.PosixProcRlimitsHardMatchMax, spec.Version, fmt.Sprintf("has expected hard %v", r.Type)) + if err != nil { + return err } + c.harness.YAML(map[string]interface{}{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "type": r.Type, + "expected": r.Hard, + "actual": rlimit.Max, + }) } return nil } -func validateSysctls(spec *rspec.Spec, t *tap.T) error { - if spec.Linux == nil { +func (c *complianceTester) validateSysctls(spec *rspec.Spec) error { + if spec.Linux == nil || spec.Linux.Sysctl == nil { + c.harness.Skip(1, "linux.sysctl not set") return nil } + for k, v := range spec.Linux.Sysctl { keyPath := filepath.Join("/proc/sys", strings.Replace(k, ".", "/", -1)) vBytes, err := ioutil.ReadFile(keyPath) @@ -299,42 +394,103 @@ func validateSysctls(spec *rspec.Spec, t *tap.T) error { return err } value := strings.TrimSpace(string(bytes.Trim(vBytes, "\x00"))) - if value != v { - return fmt.Errorf("Sysctl %v value expected: %v, actual: %v", k, v, value) - } + c.harness.Ok(value == v, fmt.Sprintf("has expected sysctl %v", k)) + c.harness.YAML(map[string]string{ + "sysctl": k, + "expected": v, + "actual": value, + }) } return nil } -func testWriteAccess(path string) error { - tmpfile, err := ioutil.TempFile(path, "Test") +func testReadAccess(path string) (readable bool, err error) { + fi, err := os.Stat(path) if err != nil { - return err + return false, err + } + if fi.Mode()&os.ModeType == 0 { + return testFileReadAccess(path) } + return false, fmt.Errorf("cannot test read access for %q (mode %d)", path, fi.Mode()) +} +func testFileReadAccess(path string) (readable bool, err error) { + f, err := os.Open(path) + if err != nil { + return false, nil + } + defer f.Close() + b := make([]byte, 1) + _, err = f.Read(b) + if err == nil { + return true, nil + } else if err == io.EOF { + return false, nil + } + return false, err +} + +func testWriteAccess(path string) (writable bool, err error) { + fi, err := os.Stat(path) + if err != nil { + return false, err + } + if fi.IsDir() { + return testDirectoryWriteAccess(path) + } else if fi.Mode()&os.ModeType == 0 { + return testFileWriteAccess(path) + } + return false, fmt.Errorf("cannot test write access for %q (mode %d)", path, fi.Mode()) +} + +func testDirectoryWriteAccess(path string) (writable bool, err error) { + tmpfile, err := ioutil.TempFile(path, "Test") + if err != nil { + return false, nil + } tmpfile.Close() - os.RemoveAll(filepath.Join(path, tmpfile.Name())) + return true, os.RemoveAll(filepath.Join(path, tmpfile.Name())) +} - return nil +func testFileWriteAccess(path string) (readable bool, err error) { + err = ioutil.WriteFile(path, []byte("a"), 0644) + if err == nil { + return true, nil + } + return false, nil } -func validateRootFS(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validateRootFS(spec *rspec.Spec) error { if spec.Root == nil { + c.harness.Skip(1, "root not set") return nil } + writable, err := testDirectoryWriteAccess("/") + if err != nil { + return err + } + if spec.Root.Readonly { - err := testWriteAccess("/") - if err == nil { - return specerror.NewError(specerror.RootReadonlyImplement, fmt.Errorf("rootfs must be readonly"), rspec.Version) + rfcError, err := c.Ok(!writable, specerror.RootReadonlyImplement, spec.Version, "root filesystem is readonly") + if err != nil { + return err } - } // no need to check the else case: unwriteable root is not a spec violation + c.harness.YAML(map[string]string{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + }) + } else if !writable { + c.harness.Skip(1, "root.readonly is false but the root filesystem is still not writable") + } return nil } -func validateRootfsPropagation(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validateRootfsPropagation(spec *rspec.Spec) error { if spec.Linux == nil || spec.Linux.RootfsPropagation == "" { + c.harness.Skip(1, "linux.rootfsPropagation not set") return nil } @@ -372,36 +528,46 @@ func validateRootfsPropagation(spec *rspec.Spec, t *tap.T) error { return err } defer unix.Unmount(mountDir, unix.MNT_DETACH) - if _, err := os.Stat(filepath.Join(targetDir, filepath.Join(mountDir, filepath.Base(tmpfile.Name())))); os.IsNotExist(err) { - if spec.Linux.RootfsPropagation == "shared" { - return fmt.Errorf("rootfs should be %s, but not", spec.Linux.RootfsPropagation) - } - return nil + targetFile := filepath.Join(targetDir, filepath.Join(mountDir, filepath.Base(tmpfile.Name()))) + var exposed bool + _, err = os.Stat(targetFile) + if os.IsNotExist(err) { + exposed = false + } else if err != nil { + return err + } else { + exposed = true } if spec.Linux.RootfsPropagation == "shared" { - return nil + c.harness.Ok(exposed, fmt.Sprintf("shared root propogation exposes %q", targetFile)) + } else { + c.harness.Ok( + !exposed, + fmt.Sprintf("%s root propogation does not expose %q", spec.Linux.RootfsPropagation, targetFile), + ) } - return fmt.Errorf("rootfs should be %s, but not", spec.Linux.RootfsPropagation) case "unbindable": - if err := unix.Mount("/", targetDir, "", unix.MS_BIND|unix.MS_REC, ""); err != nil { - if err == syscall.EINVAL { - return nil - } + err = unix.Mount("/", targetDir, "", unix.MS_BIND|unix.MS_REC, "") + if err == syscall.EINVAL { + c.harness.Pass("root propagation is unbindable") + return nil + } else if err != nil { return err } defer unix.Unmount(targetDir, unix.MNT_DETACH) - return fmt.Errorf("rootfs expected to be unbindable, but not") + c.harness.Fail("root propagation is unbindable") + return nil default: - logrus.Warnf("unrecognized linux.rootfsPropagation %s", spec.Linux.RootfsPropagation) + c.harness.Skip(1, fmt.Sprintf("unrecognized linux.rootfsPropagation %s", spec.Linux.RootfsPropagation)) } return nil } -func validateDefaultFS(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validateDefaultFS(spec *rspec.Spec) error { mountInfos, err := mount.GetMounts() if err != nil { - specerror.NewError(specerror.DefaultFilesystems, err, spec.Version) + return nil } mountsMap := make(map[string]string) @@ -410,201 +576,377 @@ func validateDefaultFS(spec *rspec.Spec, t *tap.T) error { } for fs, fstype := range defaultFS { - if !(mountsMap[fs] == fstype) { - return specerror.NewError(specerror.DefaultFilesystems, fmt.Errorf("%v SHOULD exist and expected type is %v", fs, fstype), rspec.Version) + rfcError, err := c.Ok(mountsMap[fs] == fstype, specerror.DefaultFilesystems, spec.Version, fmt.Sprintf("mount %v has expected type", fs)) + if err != nil { + return err } + c.harness.YAML(map[string]string{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "mount": fs, + "expected": fstype, + "actual": mountsMap[fs], + }) } return nil } -func validateLinuxDevices(spec *rspec.Spec, t *tap.T) error { - if spec.Linux == nil { +func (c *complianceTester) validateLinuxDevices(spec *rspec.Spec) error { + if spec.Linux == nil || spec.Linux.Devices == nil { + c.harness.Skip(1, "linux.devices is not set") return nil } - for _, device := range spec.Linux.Devices { - fi, err := os.Stat(device.Path) + + for i, device := range spec.Linux.Devices { + err := c.validateDevice( + &device, + specerror.DevicesAvailable, + spec.Version, + fmt.Sprintf("%q (linux.devices[%d])", device.Path, i)) if err != nil { return err } - fStat, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return specerror.NewError(specerror.DevicesAvailable, fmt.Errorf("cannot determine state for device %s", device.Path), rspec.Version) - } - var devType string - switch fStat.Mode & syscall.S_IFMT { - case syscall.S_IFCHR: - devType = "c" - case syscall.S_IFBLK: - devType = "b" - case syscall.S_IFIFO: - devType = "p" - default: - devType = "unmatched" - } - if devType != device.Type || (devType == "c" && device.Type == "u") { - return fmt.Errorf("device %v expected type is %v, actual is %v", device.Path, device.Type, devType) - } - if devType != "p" { - dev := fStat.Rdev - major := (dev >> 8) & 0xfff - minor := (dev & 0xff) | ((dev >> 12) & 0xfff00) - if int64(major) != device.Major || int64(minor) != device.Minor { - return fmt.Errorf("%v device number expected is %v:%v, actual is %v:%v", device.Path, device.Major, device.Minor, major, minor) - } + } + + return nil +} + +func (c *complianceTester) validateDevice(device *rspec.LinuxDevice, condition specerror.Code, version string, description string) (err error) { + var exists bool + fi, err := os.Stat(device.Path) + if os.IsNotExist(err) { + exists = false + } else if err != nil { + return err + } else { + exists = true + } + rfcError, err := c.Ok(exists, condition, version, fmt.Sprintf("has a file at %s", description)) + if err != nil { + return err + } + c.harness.YAML(map[string]string{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "path": device.Path, + }) + if !exists { + return nil + } + + fStat, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("could not convert to syscall.Stat_t: %v", fi.Sys()) + } + expectedType := device.Type + if expectedType == "u" { + expectedType = "c" + } + var devType string + switch fStat.Mode & syscall.S_IFMT { + case syscall.S_IFCHR: + devType = "c" + case syscall.S_IFBLK: + devType = "b" + case syscall.S_IFIFO: + devType = "p" + default: + devType = "unmatched" + } + rfcError, err = c.Ok(devType == expectedType, condition, version, fmt.Sprintf("%s has the expected type", description)) + if err != nil { + return err + } + c.harness.YAML(map[string]string{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "path": device.Path, + "expected": expectedType, + "actual": devType, + }) + if devType != expectedType { + return nil + } + + if devType != "p" { + dev := fStat.Rdev + major := (dev >> 8) & 0xfff + minor := (dev & 0xff) | ((dev >> 12) & 0xfff00) + rfcError, err = c.Ok(int64(major) == device.Major, condition, version, fmt.Sprintf("%s has the expected major ID", description)) + if err != nil { + return err } - if device.FileMode != nil { - expectedPerm := *device.FileMode & os.ModePerm - actualPerm := fi.Mode() & os.ModePerm - if expectedPerm != actualPerm { - return fmt.Errorf("%v filemode expected is %v, actual is %v", device.Path, expectedPerm, actualPerm) - } + c.harness.YAML(map[string]interface{}{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "path": device.Path, + "expected": device.Major, + "actual": major, + }) + rfcError, err = c.Ok(int64(minor) == device.Minor, condition, version, fmt.Sprintf("%s has the expected minor ID", description)) + if err != nil { + return err } - if device.UID != nil { - if *device.UID != fStat.Uid { - return fmt.Errorf("%v uid expected is %v, actual is %v", device.Path, *device.UID, fStat.Uid) - } + c.harness.YAML(map[string]interface{}{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "path": device.Path, + "expected": device.Minor, + "actual": minor, + }) + } + + if device.FileMode == nil { + c.harness.Skip(1, fmt.Sprintf("%s has unconfigured permissions", description)) + } else { + expectedPerm := *device.FileMode & os.ModePerm + actualPerm := fi.Mode() & os.ModePerm + rfcError, err = c.Ok(actualPerm == expectedPerm, condition, version, fmt.Sprintf("%s has the expected permissions", description)) + if err != nil { + return err } - if device.GID != nil { - if *device.GID != fStat.Gid { - return fmt.Errorf("%v uid expected is %v, actual is %v", device.Path, *device.GID, fStat.Gid) - } + c.harness.YAML(map[string]interface{}{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "path": device.Path, + "expected": expectedPerm, + "actual": actualPerm, + }) + } + + if description == "/dev/console (default device)" { + c.harness.Todo().Fail("we need the major/minor from the controlling TTY") + return nil + } + + if device.UID == nil { + c.harness.Skip(1, fmt.Sprintf("%s has an unconfigured user ID", description)) + } else { + rfcError, err = c.Ok(fStat.Uid == *device.UID, condition, version, fmt.Sprintf("%s has the expected user ID", description)) + if err != nil { + return err + } + c.harness.YAML(map[string]interface{}{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "path": device.Path, + "expected": *device.UID, + "actual": fStat.Uid, + }) + } + + if device.GID == nil { + c.harness.Skip(1, fmt.Sprintf("%s has an unconfigured group ID", description)) + } else { + rfcError, err = c.Ok(fStat.Gid == *device.GID, condition, version, fmt.Sprintf("%s has the expected group ID", description)) + if err != nil { + return err } + c.harness.YAML(map[string]interface{}{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "path": device.Path, + "expected": *device.GID, + "actual": fStat.Gid, + }) } return nil } -func validateDefaultSymlinks(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validateDefaultSymlinks(spec *rspec.Spec) error { for symlink, dest := range defaultSymlinks { + var exists bool fi, err := os.Lstat(symlink) + if os.IsNotExist(err) { + exists = false + } else if err != nil { + return err + } else { + exists = true + } + rfcError, err := c.Ok(exists, specerror.DefaultRuntimeLinuxSymlinks, spec.Version, fmt.Sprintf("has a file at default symlink path %q", symlink)) if err != nil { return err } - if fi.Mode()&os.ModeSymlink != os.ModeSymlink { - return specerror.NewError(specerror.DefaultRuntimeLinuxSymlinks, - fmt.Errorf("%v is not a symbolic link as expected", symlink), - rspec.Version) + c.harness.YAML(map[string]string{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "path": symlink, + }) + if !exists { + continue } + + isSymlink := fi.Mode()&os.ModeSymlink == os.ModeSymlink + rfcError, err = c.Ok( + isSymlink, + specerror.DefaultRuntimeLinuxSymlinks, + spec.Version, + fmt.Sprintf("file at default symlink path %q is a symlink", symlink)) + if err != nil { + return err + } + c.harness.YAML(map[string]interface{}{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "path": symlink, + "mode": fi.Mode(), + }) + if !isSymlink { + continue + } + realDest, err := os.Readlink(symlink) if err != nil { return err } - if realDest != dest { - return specerror.NewError(specerror.DefaultRuntimeLinuxSymlinks, - fmt.Errorf("link destation of %v expected is %v, actual is %v", - symlink, dest, realDest), - rspec.Version) + rfcError, err = c.Ok( + realDest == dest, + specerror.DefaultRuntimeLinuxSymlinks, + spec.Version, + fmt.Sprintf("symlink at default symlink path %q has the expected target", symlink)) + if err != nil { + return err } + c.harness.YAML(map[string]string{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "path": symlink, + "expected": dest, + "actual": realDest, + }) } return nil } -func validateDefaultDevices(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validateDefaultDevices(spec *rspec.Spec) error { if spec.Process != nil && spec.Process.Terminal { - defaultDevices = append(defaultDevices, "/dev/console") + defaultDevices = append(defaultDevices, rspec.LinuxDevice{ + Path: "/dev/console", + Type: "c", + // FIXME: get the major/minor from the controlling TTY + }) } for _, device := range defaultDevices { - fi, err := os.Stat(device) + err := c.validateDevice( + &device, + specerror.DefaultDevices, + spec.Version, + fmt.Sprintf("%s (default device)", device.Path)) if err != nil { - if os.IsNotExist(err) { - return specerror.NewError(specerror.DefaultDevices, - fmt.Errorf("device node %v not found", device), - rspec.Version) - } return err } - if fi.Mode()&os.ModeDevice != os.ModeDevice { - return specerror.NewError(specerror.DefaultDevices, - fmt.Errorf("file %v is not a device as expected", device), - rspec.Version) - } } return nil } -func validateMaskedPaths(spec *rspec.Spec, t *tap.T) error { - if spec.Linux == nil { +func (c *complianceTester) validateMaskedPaths(spec *rspec.Spec) error { + if spec.Linux == nil || spec.Linux.MaskedPaths == nil { + c.harness.Skip(1, "linux.maskedPaths not set") return nil } + for _, maskedPath := range spec.Linux.MaskedPaths { - f, err := os.Open(maskedPath) - if err != nil { + readable, err := testReadAccess(maskedPath) + if err != nil && !os.IsNotExist(err) { return err } - defer f.Close() - b := make([]byte, 1) - _, err = f.Read(b) - if err != io.EOF { - return fmt.Errorf("%v should not be readable", maskedPath) - } + c.harness.Ok(!readable, fmt.Sprintf("cannot read masked path %q", maskedPath)) } + return nil } -func validateSeccomp(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validateSeccomp(spec *rspec.Spec) error { if spec.Linux == nil || spec.Linux.Seccomp == nil { + c.harness.Skip(1, "linux.seccomp not set") return nil } + for _, sys := range spec.Linux.Seccomp.Syscalls { if sys.Action == "SCMP_ACT_ERRNO" { for _, name := range sys.Names { if name == "getcwd" { _, err := os.Getwd() if err == nil { - t.Diagnostic("getcwd did not return an error") + c.harness.Skip(1, "getcwd did not return an error") } } else { - t.Skip(1, fmt.Sprintf("%s syscall returns errno", name)) + c.harness.Skip(1, fmt.Sprintf("%s syscall returns errno", name)) } } } else { - t.Skip(1, fmt.Sprintf("syscall action %s", sys.Action)) + c.harness.Skip(1, fmt.Sprintf("syscall action %s", sys.Action)) } } + return nil } -func validateROPaths(spec *rspec.Spec, t *tap.T) error { - if spec.Linux == nil { +func (c *complianceTester) validateROPaths(spec *rspec.Spec) error { + if spec.Linux == nil || spec.Linux.ReadonlyPaths == nil { + c.harness.Skip(1, "linux.readonlyPaths not set") return nil } - for _, v := range spec.Linux.ReadonlyPaths { - err := testWriteAccess(v) - if err == nil { - return fmt.Errorf("%v should be readonly", v) + + for i, path := range spec.Linux.ReadonlyPaths { + readable, err := testReadAccess(path) + if err != nil { + return err + } + if !readable { + c.harness.Skip(1, fmt.Sprintf("%q (linux.readonlyPaths[%d]) is not readable", path, i)) + } + + writable, err := testWriteAccess(path) + if err != nil && !os.IsNotExist(err) { + return err } + c.harness.Ok(!writable, fmt.Sprintf("%q (linux.readonlyPaths[%d]) is not writable", path, i)) } return nil } -func validateOOMScoreAdj(spec *rspec.Spec, t *tap.T) error { - if spec.Process != nil && spec.Process.OOMScoreAdj != nil { - expected := *spec.Process.OOMScoreAdj - f, err := os.Open("/proc/self/oom_score_adj") +func (c *complianceTester) validateOOMScoreAdj(spec *rspec.Spec) error { + if spec.Process == nil || spec.Process.OOMScoreAdj == nil { + c.harness.Skip(1, "process.oomScoreAdj not set") + return nil + } + + expected := *spec.Process.OOMScoreAdj + f, err := os.Open("/proc/self/oom_score_adj") + if err != nil { + return err + } + defer f.Close() + + s := bufio.NewScanner(f) + for s.Scan() { + err := s.Err() if err != nil { return err } - defer f.Close() - - s := bufio.NewScanner(f) - for s.Scan() { - if err := s.Err(); err != nil { - return err - } - text := strings.TrimSpace(s.Text()) - actual, err := strconv.Atoi(text) - if err != nil { - return err - } - if actual != expected { - return specerror.NewError(specerror.LinuxProcOomScoreAdjSet, fmt.Errorf("oomScoreAdj expected: %v, actual: %v", expected, actual), rspec.Version) - } + text := strings.TrimSpace(s.Text()) + actual, err := strconv.Atoi(text) + if err != nil { + return err } + rfcError, err := c.Ok(actual == expected, specerror.LinuxProcOomScoreAdjSet, spec.Version, fmt.Sprintf("has expected OOM score adjustment")) + if err != nil { + return err + } + c.harness.YAML(map[string]interface{}{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "expected": expected, + "actual": actual, + }) } return nil @@ -649,14 +991,21 @@ func getIDMappings(path string) ([]rspec.LinuxIDMapping, error) { return idMaps, nil } -func validateIDMappings(mappings []rspec.LinuxIDMapping, path string, property string) error { +func (c *complianceTester) validateIDMappings(mappings []rspec.LinuxIDMapping, path string, property string) error { + if len(mappings) == 0 { + c.harness.Skip(1, fmt.Sprintf("%s not set", property)) + return nil + } + idMaps, err := getIDMappings(path) if err != nil { - return fmt.Errorf("can not get items: %v", err) - } - if len(mappings) != 0 && len(mappings) != len(idMaps) { - return fmt.Errorf("expected %d entries in %v, but acutal is %d", len(mappings), path, len(idMaps)) + return err } + c.harness.Ok(len(idMaps) == len(mappings), fmt.Sprintf("%s has expected number of mappings", path)) + c.harness.YAML(map[string]interface{}{ + "expected": mappings, + "actual": idMaps, + }) for _, v := range mappings { exist := false for _, cv := range idMaps { @@ -665,26 +1014,26 @@ func validateIDMappings(mappings []rspec.LinuxIDMapping, path string, property s break } } - if !exist { - return fmt.Errorf("%v is not applied as expected", property) - } + c.harness.Ok(exist, fmt.Sprintf("%s has expected mapping %v", path, v)) } return nil } -func validateUIDMappings(spec *rspec.Spec, t *tap.T) error { - if spec.Linux == nil { +func (c *complianceTester) validateUIDMappings(spec *rspec.Spec) error { + if spec.Linux == nil || spec.Linux.UIDMappings == nil { + c.harness.Skip(1, "linux.uidMappings not set") return nil } - return validateIDMappings(spec.Linux.UIDMappings, "/proc/self/uid_map", "linux.uidMappings") + return c.validateIDMappings(spec.Linux.UIDMappings, "/proc/self/uid_map", "linux.uidMappings") } -func validateGIDMappings(spec *rspec.Spec, t *tap.T) error { - if spec.Linux == nil { +func (c *complianceTester) validateGIDMappings(spec *rspec.Spec) error { + if spec.Linux == nil || spec.Linux.GIDMappings == nil { + c.harness.Skip(1, "linux.gidMappings not set") return nil } - return validateIDMappings(spec.Linux.GIDMappings, "/proc/self/gid_map", "linux.gidMappings") + return c.validateIDMappings(spec.Linux.GIDMappings, "/proc/self/gid_map", "linux.gidMappings") } func mountMatch(configMount rspec.Mount, sysMount *mount.Info) error { @@ -709,67 +1058,76 @@ func mountMatch(configMount rspec.Mount, sysMount *mount.Info) error { return nil } -func validatePosixMounts(spec *rspec.Spec, t *tap.T) error { +func (c *complianceTester) validatePosixMounts(spec *rspec.Spec) error { + if spec.Mounts == nil { + c.harness.Skip(1, "mounts not set") + return nil + } + mountInfos, err := mount.GetMounts() if err != nil { return err } var mountErrs error + var configSys = make(map[int]int) var consumedSys = make(map[int]bool) highestMatchedConfig := -1 - highestMatchedSystem := -1 var j = 0 for i, configMount := range spec.Mounts { if configMount.Type == "bind" || configMount.Type == "rbind" { - // TODO: add bind or rbind check. + c.harness.Todo().Fail("we need an (r)bind spec to test against") continue } - found := false + foundInOrder := false + foundOutOfOrder := false for k, sysMount := range mountInfos[j:] { if err := mountMatch(configMount, sysMount); err == nil { - found = true + foundInOrder = true j += k + 1 + configSys[i] = j - 1 consumedSys[j-1] = true - if j > highestMatchedSystem { - highestMatchedSystem = j - 1 + if j > configSys[highestMatchedConfig] { highestMatchedConfig = i } break } } - if !found { + if err != nil { + return err + } + if !foundInOrder { if j > 0 { for k, sysMount := range mountInfos[:j-1] { if _, ok := consumedSys[k]; ok { continue } if err := mountMatch(configMount, sysMount); err == nil { - found = true + foundOutOfOrder = true break } } } - if found { - mountErrs = multierror.Append( - mountErrs, - specerror.NewError(specerror.MountsInOrder, - fmt.Errorf( - "mounts[%d] %v mounted before mounts[%d] %v", - i, - configMount, - highestMatchedConfig, - spec.Mounts[highestMatchedConfig]), - rspec.Version)) - } else { - mountErrs = multierror.Append( - mountErrs, - specerror.NewError(specerror.MountsInOrder, fmt.Errorf( - "mounts[%d] %v does not exist", - i, - configMount), rspec.Version)) - } + } + + var rfcError *rfc2119.Error + if !foundInOrder && !foundOutOfOrder { + rfcError, err = c.Ok(false, specerror.MountsInOrder, spec.Version, fmt.Sprintf("mounts[%d] (%s) found", i, configMount.Destination)) + } else { + rfcError, err = c.Ok(foundInOrder, specerror.MountsInOrder, spec.Version, fmt.Sprintf("mounts[%d] (%s) found in order", i, configMount.Destination)) + c.harness.YAML(map[string]interface{}{ + "level": rfcError.Level.String(), + "reference": rfcError.Reference, + "config": configMount, + "indexConfig": i, + "indexSystem": configSys[i], + "earlier": map[string]interface{}{ + "config": spec.Mounts[highestMatchedConfig], + "indexConfig": highestMatchedConfig, + "indexSystem": configSys[highestMatchedConfig], + }, + }) } } @@ -795,103 +1153,47 @@ func run(context *cli.Context) error { return err } - defaultValidations := []validation{ - { - test: validateRootFS, - description: "root filesystem", - }, - { - test: validateHostname, - description: "hostname", - }, - { - test: validateProcess, - description: "process", - }, + complianceLevelString := context.String("compliance-level") + complianceLevel, err := rfc2119.ParseLevel(complianceLevelString) + if err != nil { + complianceLevel = rfc2119.Must + logrus.Warningf("%s, using 'MUST' by default.", err.Error()) } - posixValidations := []validation{ - { - test: validatePosixMounts, - description: "mounts", - }, - { - test: validatePosixUser, - description: "user", - }, - { - test: validateRlimits, - description: "rlimits", - }, + c := &complianceTester{ + harness: tap.New(), + complianceLevel: complianceLevel, } - linuxValidations := []validation{ - { - test: validateCapabilities, - description: "capabilities", - }, - { - test: validateDefaultSymlinks, - description: "default symlinks", - }, - { - test: validateDefaultFS, - description: "default file system", - }, - { - test: validateDefaultDevices, - description: "default devices", - }, - { - test: validateLinuxDevices, - description: "linux devices", - }, - { - test: validateLinuxProcess, - description: "linux process", - }, - { - test: validateMaskedPaths, - description: "masked paths", - }, - { - test: validateOOMScoreAdj, - description: "oom score adj", - }, - { - test: validateSeccomp, - description: "seccomp", - }, - { - test: validateROPaths, - description: "read only paths", - }, - { - test: validateRootfsPropagation, - description: "rootfs propagation", - }, - { - test: validateSysctls, - description: "sysctls", - }, - { - test: validateUIDMappings, - description: "uid mappings", - }, - { - test: validateGIDMappings, - description: "gid mappings", - }, + c.harness.Header(0) + + defaultValidations := []validator{ + c.validateRootFS, + c.validateHostname, + c.validateProcess, } - t := tap.New() - t.Header(0) + posixValidations := []validator{ + c.validatePosixMounts, + c.validatePosixUser, + c.validateRlimits, + } - complianceLevelString := context.String("compliance-level") - complianceLevel, err := rfc2119.ParseLevel(complianceLevelString) - if err != nil { - complianceLevel = rfc2119.Must - logrus.Warningf("%s, using 'MUST' by default.", err.Error()) + linuxValidations := []validator{ + c.validateCapabilities, + c.validateDefaultSymlinks, + c.validateDefaultFS, + c.validateDefaultDevices, + c.validateLinuxDevices, + c.validateLinuxProcess, + c.validateMaskedPaths, + c.validateOOMScoreAdj, + c.validateSeccomp, + c.validateROPaths, + c.validateRootfsPropagation, + c.validateSysctls, + c.validateUIDMappings, + c.validateGIDMappings, } validations := defaultValidations @@ -902,32 +1204,13 @@ func run(context *cli.Context) error { validations = append(validations, posixValidations...) } - for _, v := range validations { - err := v.test(spec, t) - if err == nil { - t.Pass(v.description) - } else { - merr, ok := err.(*multierror.Error) - if ok { - for _, err = range merr.Errors { - if e, ok := err.(*rfc2119.Error); ok { - t.Ok(e.Level < complianceLevel, v.description) - } else { - t.Fail(v.description) - } - t.YAML(map[string]string{"error": err.Error()}) - } - } else { - if e, ok := err.(*rfc2119.Error); ok { - t.Ok(e.Level < complianceLevel, v.description) - } else { - t.Fail(v.description) - } - t.YAML(map[string]string{"error": err.Error()}) - } + for _, validation := range validations { + err := validation(spec) + if err != nil { + return err } } - t.AutoPlan() + c.harness.AutoPlan() return nil } diff --git a/error/error.go b/error/error.go index f5a90800e..f7eac6639 100644 --- a/error/error.go +++ b/error/error.go @@ -86,6 +86,36 @@ func ParseLevel(level string) (Level, error) { return l, fmt.Errorf("%q is not a valid compliance level", level) } +// String takes a RFC 2119 compliance level constant and returns a string representation. +func (level Level) String() string { + switch level { + case May: + return "MAY" + case Optional: + return "OPTIONAL" + case Should: + return "SHOULD" + case ShouldNot: + return "SHOULD NOT" + case Recommended: + return "RECOMMENDED" + case NotRecommended: + return "NOT RECOMMENDED" + case Must: + return "MUST" + case MustNot: + return "MUST NOT" + case Shall: + return "SHALL" + case ShallNot: + return "SHALL NOT" + case Required: + return "REQUIRED" + } + + panic(fmt.Sprintf("%d is not a valid compliance level", level)) +} + // Error returns the error message with specification reference. func (err *Error) Error() string { return fmt.Sprintf("%s\nRefer to: %s", err.Err.Error(), err.Reference) From 732d438d1a04ebf90bf5289029986b0759c46a8d Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 6 Apr 2018 09:20:12 -0700 Subject: [PATCH 2/4] validation: Use non-empty files in masked/readonly tests Previously we were only looking at directories. But the spec does not say these are directory-only options, so test files too. Put some content in the files, because runtimetest currently has no way to check the readability of empty files. Signed-off-by: W. Trevor King --- cmd/runtimetest/main.go | 3 +++ validation/linux_masked_paths.go | 17 ++++++++++++++--- validation/linux_readonly_paths.go | 17 ++++++++++++++--- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/cmd/runtimetest/main.go b/cmd/runtimetest/main.go index a8809ed07..74c66aef0 100644 --- a/cmd/runtimetest/main.go +++ b/cmd/runtimetest/main.go @@ -426,6 +426,9 @@ func testFileReadAccess(path string) (readable bool, err error) { if err == nil { return true, nil } else if err == io.EOF { + // Our validation/ tests only use non-empty files for read-access + // tests. So if we get an EOF on the first read, the runtime did + // successfully block readability. return false, nil } return false, err diff --git a/validation/linux_masked_paths.go b/validation/linux_masked_paths.go index 5d97f41ba..545a0e191 100644 --- a/validation/linux_masked_paths.go +++ b/validation/linux_masked_paths.go @@ -1,6 +1,7 @@ package main import ( + "io/ioutil" "os" "path/filepath" @@ -9,10 +10,20 @@ import ( func main() { g := util.GetDefaultGenerator() - g.AddLinuxMaskedPaths("/masktest") + g.AddLinuxMaskedPaths("/masked-dir") + g.AddLinuxMaskedPaths("/masked-file") err := util.RuntimeInsideValidate(g, func(path string) error { - pathName := filepath.Join(path, "masktest") - return os.MkdirAll(pathName, 0700) + testDir := filepath.Join(path, "masked-dir") + err := os.MkdirAll(testDir, 0777) + if err != nil { + return err + } + + testFile := filepath.Join(path, "masked-file") + + // runtimetest cannot check the readability of empty files, so + // write something. + return ioutil.WriteFile(testFile, []byte("secrets"), 0777) }) if err != nil { util.Fatal(err) diff --git a/validation/linux_readonly_paths.go b/validation/linux_readonly_paths.go index 6c1e37fa1..bd727e630 100644 --- a/validation/linux_readonly_paths.go +++ b/validation/linux_readonly_paths.go @@ -1,6 +1,7 @@ package main import ( + "io/ioutil" "os" "path/filepath" @@ -9,10 +10,20 @@ import ( func main() { g := util.GetDefaultGenerator() - g.AddLinuxReadonlyPaths("readonlytest") + g.AddLinuxReadonlyPaths("/readonly-dir") + g.AddLinuxReadonlyPaths("/readonly-file") err := util.RuntimeInsideValidate(g, func(path string) error { - pathName := filepath.Join(path, "readonlytest") - return os.MkdirAll(pathName, 0700) + testDir := filepath.Join(path, "readonly-dir") + err := os.MkdirAll(testDir, 0777) + if err != nil { + return err + } + + testFile := filepath.Join(path, "readonly-file") + + // runtimetest cannot check the readability of empty files, so + // write something. + return ioutil.WriteFile(testFile, []byte("immutable"), 0777) }) if err != nil { util.Fatal(err) From 4b888f281379cc9d39d0b7fffd4a1c22cd17774c Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 6 Apr 2018 10:13:47 -0700 Subject: [PATCH 3/4] runtimetest: Use ModeType as the mask in the symlink check We've been using ModeSymlink as the mask since the check landed in da250042 (runtimetest: add linux default symbolic link validation, 2016-11-30, #284). POSIX provides S_IS*(m) macros to portably interpret the mode type, but does not define values for each type [2]. Alban pointed out that st_mode is not a bitfield on Linux [1]. For example, Linux defines [3]: S_IFBLK 060000 S_IFDIR 040000 S_IFCHR 020000 So 'm&S_IFCHR == S_IFCHR', for example, would succeed for both character and block devices. Go translates the system values to a platform-agnostic bitfield [4], so the previous approach works on Go. But it may be confusing for people used to the native non-bitfield mode, so this commit moves us to an approach that does not rely on Go's using a bitfield. [1]: https://github.com/opencontainers/runtime-tools/pull/308#discussion_r179799657 [2]: http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/sys_stat.h.html [3]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/stat.h?h=v4.16#n9 [4]: https://github.com/golang/go/blob/b0d437f866eb8987cde7e6550cacd77876f36d4b/src/os/types.go#L45 Signed-off-by: W. Trevor King --- cmd/runtimetest/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/runtimetest/main.go b/cmd/runtimetest/main.go index 74c66aef0..8eaee7d28 100644 --- a/cmd/runtimetest/main.go +++ b/cmd/runtimetest/main.go @@ -782,7 +782,7 @@ func (c *complianceTester) validateDefaultSymlinks(spec *rspec.Spec) error { continue } - isSymlink := fi.Mode()&os.ModeSymlink == os.ModeSymlink + isSymlink := fi.Mode()&os.ModeType == os.ModeSymlink rfcError, err = c.Ok( isSymlink, specerror.DefaultRuntimeLinuxSymlinks, From e1ad3f006190202e6f6d516d116373c541de0bbc Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 6 Apr 2018 15:42:29 -0700 Subject: [PATCH 4/4] README: Update to reflect granular TAP output I ran these examples again with: $ runc --version runc version 1.0.0-rc5+dev commit: cc4307ab6643668ce5abc6b524e1764a54c32550 spec: 1.0.0 $ uname -r 4.15.0 I don't know what that hugetlb permission error is about, but that's what I see ;). It could be an issue with my local system. Signed-off-by: W. Trevor King --- README.md | 134 ++++++++++++++---------------------------------------- 1 file changed, 34 insertions(+), 100 deletions(-) diff --git a/README.md b/README.md index 796d1ac76..19d0e4a26 100644 --- a/README.md +++ b/README.md @@ -49,41 +49,16 @@ For the command line interface, the `RUNTIME` option selects the runtime command ``` $ sudo make RUNTIME=runc localvalidation -RUNTIME=runc tap validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t -validation/linux_rootfs_propagation_shared.t ........ 18/19 - not ok rootfs propagation - error: 'rootfs should be shared, but not' - -validation/create.t ................................... 4/4 -validation/default.t ................................ 19/19 -validation/linux_readonly_paths.t ................... 19/19 -validation/linux_masked_paths.t ..................... 18/19 - not ok masked paths - error: /masktest should not be readable - -validation/mounts.t ................................... 0/1 - Skipped: 1 - TODO: mounts generation options have not been implemented - -validation/process.t ................................ 19/19 -validation/root_readonly_false.t .................... 19/19 -validation/linux_sysctl.t ........................... 19/19 -validation/linux_devices.t .......................... 19/19 -validation/linux_gid_mappings.t ..................... 18/19 - not ok gid mappings - -validation/process_oom_score_adj.t .................. 19/19 -validation/process_capabilities.t ................... 19/19 -validation/process_rlimits.t ........................ 19/19 -validation/root_readonly_true.t ...................failed to create the container -rootfsPropagation=unbindable is not supported +RUNTIME=runc tap validation/pidfile.t validation/linux_cgroups_hugetlb.t validation/linux_cgroups_memory.t validation/linux_rootfs_propagation_shared.t validation/kill.t validation/create.t validation/poststart.t validation/linux_cgroups_network.t validation/poststop_fail.t validation/linux_readonly_paths.t validation/prestart_fail.t validation/hooks_stdin.t validation/default.t validation/linux_masked_paths.t validation/poststop.t validation/misc_props.t validation/prestart.t validation/poststart_fail.t validation/mounts.t validation/linux_cgroups_relative_pids.t validation/process_user.t validation/process.t validation/hooks.t validation/process_capabilities_fail.t validation/process_rlimits_fail.t validation/linux_cgroups_relative_cpus.t validation/process_rlimits.t validation/linux_cgroups_relative_blkio.t validation/linux_sysctl.t validation/linux_seccomp.t validation/linux_devices.t validation/start.t validation/linux_cgroups_pids.t validation/process_capabilities.t validation/process_oom_score_adj.t validation/linux_cgroups_relative_hugetlb.t validation/linux_cgroups_cpus.t validation/linux_cgroups_relative_memory.t validation/state.t validation/root_readonly_true.t validation/linux_cgroups_blkio.t validation/linux_rootfs_propagation_unbindable.t validation/delete.t validation/linux_cgroups_relative_network.t validation/hostname.t validation/killsig.t validation/linux_uid_mappings.t +validation/pidfile.t .failed to create the container +container_linux.go:348: starting container process caused "process_linux.go:402: container init caused \"process_linux.go:367: setting cgroup config for procHooks process caused \\\"failed to write 56892210544640 to hugetlb.1GB.limit_in_bytes: open /sys/fs/cgroup/hugetlb/cgrouptest/hugetlb.1GB.limit_in_bytes: permission denied\\\"\"" exit status 1 -validation/root_readonly_true.t ..................... 19/19 -validation/linux_rootfs_propagation_unbindable.t ...... 0/1 - not ok validation/linux_rootfs_propagation_unbindable.t +validation/pidfile.t .................................. 1/1 315ms +validation/linux_cgroups_hugetlb.t .................... 0/1 + not ok validation/linux_cgroups_hugetlb.t timeout: 30000 - file: validation/linux_rootfs_propagation_unbindable.t - command: validation/linux_rootfs_propagation_unbindable.t + file: validation/linux_cgroups_hugetlb.t + command: validation/linux_cgroups_hugetlb.t args: [] stdio: - 0 @@ -92,31 +67,21 @@ validation/linux_rootfs_propagation_unbindable.t ...... 0/1 cwd: /…/go/src/github.com/opencontainers/runtime-tools exitCode: 1 -validation/hostname.t ...................failed to create the container -User namespace mappings specified, but USER namespace isn't enabled in the config -exit status 1 -validation/hostname.t ............................... 19/19 -validation/linux_uid_mappings.t ....................... 0/1 - not ok validation/linux_uid_mappings.t - timeout: 30000 - file: validation/linux_uid_mappings.t - command: validation/linux_uid_mappings.t - args: [] - stdio: - - 0 - - pipe - - 2 - cwd: /…/go/src/github.com/opencontainers/runtime-tools - exitCode: 1 +validation/linux_cgroups_memory.t ..................... 9/9 +validation/linux_rootfs_propagation_shared.t ...... 252/282 + not ok shared root propogation exposes "/target348456609/mount892511628/example376408222" -total ............................................. 267/273 + Skipped: 29 + /dev/null (default device) has unconfigured permissions +… +total ........................................... 4381/4962 - 267 passing (31s) - 1 pending - 5 failing + 4381 passing (1m) + 567 pending + 14 failing -make: *** [Makefile:43: localvalidation] Error 1 +make: *** [Makefile:44: localvalidation] Error 1 ``` You can also run an individual test executable directly: @@ -124,58 +89,27 @@ You can also run an individual test executable directly: ```console $ RUNTIME=runc validation/default.t TAP version 13 -ok 1 - root filesystem -ok 2 - hostname -ok 3 - process -ok 4 - mounts -ok 5 - user -ok 6 - rlimits -ok 7 - capabilities -ok 8 - default symlinks -ok 9 - default file system -ok 10 - default devices -ok 11 - linux devices -ok 12 - linux process -ok 13 - masked paths -ok 14 - oom score adj -ok 15 - read only paths -ok 16 - rootfs propagation -ok 17 - sysctls -ok 18 - uid mappings -ok 19 - gid mappings -1..19 +ok 1 - has expected hostname + --- + { + "actual": "mrsdalloway", + "expected": "mrsdalloway" + } + ... +… +ok 287 # SKIP linux.gidMappings not set +1..287 ``` If you cannot install node-tap, you can probably run the test suite with another [TAP consumer][tap-consumers]. For example, with [`prove`][prove]: ```console -$ sudo make TAP='prove -Q -j9' RUNTIME=runc localvalidation -RUNTIME=runc prove -Q -j9 validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t -failed to create the container -rootfsPropagation=unbindable is not supported -exit status 1 -failed to create the container -User namespace mappings specified, but USER namespace isn't enabled in the config -exit status 1 - -Test Summary Report -------------------- -validation/linux_rootfs_propagation_shared.t (Wstat: 0 Tests: 19 Failed: 1) - Failed test: 16 -validation/linux_masked_paths.t (Wstat: 0 Tests: 19 Failed: 1) - Failed test: 13 -validation/linux_rootfs_propagation_unbindable.t (Wstat: 256 Tests: 0 Failed: 0) - Non-zero exit status: 1 - Parse errors: No plan found in TAP output -validation/linux_uid_mappings.t (Wstat: 256 Tests: 0 Failed: 0) - Non-zero exit status: 1 - Parse errors: No plan found in TAP output -validation/linux_gid_mappings.t (Wstat: 0 Tests: 19 Failed: 1) - Failed test: 19 -Files=18, Tests=271, 6 wallclock secs ( 0.06 usr 0.01 sys + 0.59 cusr 0.24 csys = 0.90 CPU) -Result: FAIL -make: *** [Makefile:43: localvalidation] Error 1 +$ sudo make TAP='prove -Q -j9' RUNTIME=runc VALIDATION_TESTS=validation/pidfile.t localvalidation +RUNTIME=runc prove -Q -j9 validation/pidfile.t +All tests successful. +Files=1, Tests=1, 0 wallclock secs ( 0.01 usr 0.01 sys + 0.03 cusr 0.03 csys = 0.08 CPU) +Result: PASS ``` [bundle]: https://github.com/opencontainers/runtime-spec/blob/master/bundle.md