Skip to content

Commit f256851

Browse files
author
Zhou Hao
authored
Merge pull request #637 from kinvolk/dongsu/test-cgroups
validation: test cgroups with different input values
2 parents e963cf4 + 0a7749a commit f256851

27 files changed

+604
-176
lines changed

‎generate/config.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,8 @@ func (g *Generator) initConfigLinuxResourcesBlockIO() {
9494
}
9595
}
9696

97-
func (g *Generator) initConfigLinuxResourcesCPU() {
97+
// InitConfigLinuxResourcesCPU initializes CPU of Linux resources
98+
func (g *Generator) InitConfigLinuxResourcesCPU() {
9899
g.initConfigLinuxResources()
99100
if g.Config.Linux.Resources.CPU == nil {
100101
g.Config.Linux.Resources.CPU = &rspec.LinuxCPU{}

‎generate/generate.go

+7-7
Original file line numberDiff line numberDiff line change
@@ -703,43 +703,43 @@ func (g *Generator) DropLinuxResourcesBlockIOThrottleWriteIOPSDevice(major int64
703703

704704
// SetLinuxResourcesCPUShares sets g.Config.Linux.Resources.CPU.Shares.
705705
func (g *Generator) SetLinuxResourcesCPUShares(shares uint64) {
706-
g.initConfigLinuxResourcesCPU()
706+
g.InitConfigLinuxResourcesCPU()
707707
g.Config.Linux.Resources.CPU.Shares = &shares
708708
}
709709

710710
// SetLinuxResourcesCPUQuota sets g.Config.Linux.Resources.CPU.Quota.
711711
func (g *Generator) SetLinuxResourcesCPUQuota(quota int64) {
712-
g.initConfigLinuxResourcesCPU()
712+
g.InitConfigLinuxResourcesCPU()
713713
g.Config.Linux.Resources.CPU.Quota = &quota
714714
}
715715

716716
// SetLinuxResourcesCPUPeriod sets g.Config.Linux.Resources.CPU.Period.
717717
func (g *Generator) SetLinuxResourcesCPUPeriod(period uint64) {
718-
g.initConfigLinuxResourcesCPU()
718+
g.InitConfigLinuxResourcesCPU()
719719
g.Config.Linux.Resources.CPU.Period = &period
720720
}
721721

722722
// SetLinuxResourcesCPURealtimeRuntime sets g.Config.Linux.Resources.CPU.RealtimeRuntime.
723723
func (g *Generator) SetLinuxResourcesCPURealtimeRuntime(time int64) {
724-
g.initConfigLinuxResourcesCPU()
724+
g.InitConfigLinuxResourcesCPU()
725725
g.Config.Linux.Resources.CPU.RealtimeRuntime = &time
726726
}
727727

728728
// SetLinuxResourcesCPURealtimePeriod sets g.Config.Linux.Resources.CPU.RealtimePeriod.
729729
func (g *Generator) SetLinuxResourcesCPURealtimePeriod(period uint64) {
730-
g.initConfigLinuxResourcesCPU()
730+
g.InitConfigLinuxResourcesCPU()
731731
g.Config.Linux.Resources.CPU.RealtimePeriod = &period
732732
}
733733

734734
// SetLinuxResourcesCPUCpus sets g.Config.Linux.Resources.CPU.Cpus.
735735
func (g *Generator) SetLinuxResourcesCPUCpus(cpus string) {
736-
g.initConfigLinuxResourcesCPU()
736+
g.InitConfigLinuxResourcesCPU()
737737
g.Config.Linux.Resources.CPU.Cpus = cpus
738738
}
739739

740740
// SetLinuxResourcesCPUMems sets g.Config.Linux.Resources.CPU.Mems.
741741
func (g *Generator) SetLinuxResourcesCPUMems(mems string) {
742-
g.initConfigLinuxResourcesCPU()
742+
g.InitConfigLinuxResourcesCPU()
743743
g.Config.Linux.Resources.CPU.Mems = mems
744744
}
745745

‎validation/linux_cgroups_blkio.go

+50-8
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,72 @@
11
package main
22

33
import (
4+
"fmt"
5+
"runtime"
6+
7+
"github.com/mndrix/tap-go"
48
"github.com/opencontainers/runtime-tools/cgroups"
59
"github.com/opencontainers/runtime-tools/validation/util"
610
)
711

8-
func main() {
12+
func testBlkioCgroups(rate uint64, isEmpty bool) error {
913
var weight uint16 = 500
1014
var leafWeight uint16 = 300
11-
var major, minor int64 = 8, 0
12-
var rate uint64 = 102400
15+
16+
// It's assumed that a device /dev/sda (8:0) exists on the test system.
17+
// The minor number must be always 0, as it's only allowed to set blkio
18+
// weights to a whole block device /dev/sda, not to partitions like sda1.
19+
var major int64 = 8
20+
var minor int64
21+
22+
t := tap.New()
23+
t.Header(0)
24+
defer t.AutoPlan()
25+
1326
g, err := util.GetDefaultGenerator()
1427
if err != nil {
15-
util.Fatal(err)
28+
return err
1629
}
30+
1731
g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath)
18-
g.SetLinuxResourcesBlockIOWeight(weight)
19-
g.SetLinuxResourcesBlockIOLeafWeight(leafWeight)
32+
33+
if !isEmpty {
34+
g.SetLinuxResourcesBlockIOWeight(weight)
35+
g.SetLinuxResourcesBlockIOLeafWeight(leafWeight)
36+
}
37+
2038
g.AddLinuxResourcesBlockIOWeightDevice(major, minor, weight)
2139
g.AddLinuxResourcesBlockIOLeafWeightDevice(major, minor, leafWeight)
2240
g.AddLinuxResourcesBlockIOThrottleReadBpsDevice(major, minor, rate)
2341
g.AddLinuxResourcesBlockIOThrottleWriteBpsDevice(major, minor, rate)
2442
g.AddLinuxResourcesBlockIOThrottleReadIOPSDevice(major, minor, rate)
2543
g.AddLinuxResourcesBlockIOThrottleWriteIOPSDevice(major, minor, rate)
26-
err = util.RuntimeOutsideValidate(g, util.ValidateLinuxResourcesBlockIO)
44+
45+
err = util.RuntimeOutsideValidate(g, t, util.ValidateLinuxResourcesBlockIO)
2746
if err != nil {
28-
util.Fatal(err)
47+
return err
48+
}
49+
return nil
50+
}
51+
52+
func main() {
53+
if "linux" != runtime.GOOS {
54+
util.Fatal(fmt.Errorf("linux-specific cgroup test"))
55+
}
56+
57+
cases := []struct {
58+
rate uint64
59+
isEmpty bool
60+
}{
61+
{102400, false},
62+
{204800, false},
63+
{102400, true},
64+
{204800, true},
65+
}
66+
67+
for _, c := range cases {
68+
if err := testBlkioCgroups(c.rate, c.isEmpty); err != nil {
69+
util.Fatal(err)
70+
}
2971
}
3072
}

‎validation/linux_cgroups_cpus.go

+105-40
Original file line numberDiff line numberDiff line change
@@ -2,67 +2,132 @@ package main
22

33
import (
44
"fmt"
5+
"os"
6+
"path/filepath"
7+
"runtime"
58

6-
rspec "github.com/opencontainers/runtime-spec/specs-go"
9+
"github.com/mndrix/tap-go"
710
"github.com/opencontainers/runtime-tools/cgroups"
811
"github.com/opencontainers/runtime-tools/validation/util"
912
)
1013

11-
func main() {
12-
var shares uint64 = 1024
13-
var period uint64 = 100000
14-
var quota int64 = 50000
15-
var cpus, mems string = "0-1", "0"
16-
g, err := util.GetDefaultGenerator()
17-
if err != nil {
18-
util.Fatal(err)
14+
const (
15+
defaultRealtimePeriod uint64 = 1000000
16+
defaultRealtimeRuntime int64 = 950000
17+
)
18+
19+
func testCPUCgroups() error {
20+
t := tap.New()
21+
t.Header(0)
22+
defer t.AutoPlan()
23+
24+
CPUrange := fmt.Sprintf("0-%d", runtime.NumCPU()-1)
25+
26+
// Test with different combinations of values.
27+
// NOTE: most systems have only one memory node (mems=="0"), so we cannot
28+
// simply test with multiple values of mems.
29+
cases := []struct {
30+
shares uint64
31+
period uint64
32+
quota int64
33+
cpus string
34+
mems string
35+
}{
36+
{1024, 100000, 50000, "0", "0"},
37+
{1024, 100000, 50000, CPUrange, "0"},
38+
{1024, 100000, 200000, "0", "0"},
39+
{1024, 100000, 200000, CPUrange, "0"},
40+
{1024, 500000, 50000, "0", "0"},
41+
{1024, 500000, 50000, CPUrange, "0"},
42+
{1024, 500000, 200000, "0", "0"},
43+
{1024, 500000, 200000, CPUrange, "0"},
44+
{2048, 100000, 50000, "0", "0"},
45+
{2048, 100000, 50000, CPUrange, "0"},
46+
{2048, 100000, 200000, "0", "0"},
47+
{2048, 100000, 200000, CPUrange, "0"},
48+
{2048, 500000, 50000, "0", "0"},
49+
{2048, 500000, 50000, CPUrange, "0"},
50+
{2048, 500000, 200000, "0", "0"},
51+
{2048, 500000, 200000, CPUrange, "0"},
1952
}
20-
g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath)
21-
g.SetLinuxResourcesCPUShares(shares)
22-
g.SetLinuxResourcesCPUQuota(quota)
23-
g.SetLinuxResourcesCPUPeriod(period)
24-
g.SetLinuxResourcesCPUCpus(cpus)
25-
g.SetLinuxResourcesCPUMems(mems)
26-
err = util.RuntimeOutsideValidate(g, func(config *rspec.Spec, state *rspec.State) error {
27-
cg, err := cgroups.FindCgroup()
28-
if err != nil {
29-
return err
30-
}
31-
lcd, err := cg.GetCPUData(state.Pid, config.Linux.CgroupsPath)
53+
54+
for _, c := range cases {
55+
g, err := util.GetDefaultGenerator()
3256
if err != nil {
33-
return err
57+
return fmt.Errorf("cannot get default config from generator: %v", err)
3458
}
3559

36-
if lcd.Shares == nil {
37-
return fmt.Errorf("unable to get cpu shares, lcd.Shares == %v", lcd.Shares)
60+
g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath)
61+
62+
if c.shares > 0 {
63+
g.SetLinuxResourcesCPUShares(c.shares)
3864
}
39-
if *lcd.Shares != shares {
40-
return fmt.Errorf("cpus shares limit is not set correctly, expect: %d, actual: %d", shares, *lcd.Shares)
65+
66+
if c.period > 0 {
67+
g.SetLinuxResourcesCPUPeriod(c.period)
4168
}
4269

43-
if lcd.Quota == nil {
44-
return fmt.Errorf("unable to get cpu quota, lcd.Quota == %v", lcd.Quota)
70+
if c.quota > 0 {
71+
g.SetLinuxResourcesCPUQuota(c.quota)
4572
}
46-
if *lcd.Quota != quota {
47-
return fmt.Errorf("cpus quota is not set correctly, expect: %d, actual: %d", quota, *lcd.Quota)
73+
74+
if c.cpus != "" {
75+
g.SetLinuxResourcesCPUCpus(c.cpus)
4876
}
4977

50-
if lcd.Period == nil {
51-
return fmt.Errorf("unable to get cpu period, lcd.Period == %v", lcd.Period)
78+
if c.mems != "" {
79+
g.SetLinuxResourcesCPUMems(c.mems)
5280
}
53-
if *lcd.Period != period {
54-
return fmt.Errorf("cpus period is not set correctly, expect: %d, actual: %d", period, *lcd.Period)
81+
82+
// NOTE: On most systems where CONFIG_RT_GROUP & CONFIG_RT_GROUP_SCHED are not enabled,
83+
// the following tests will fail, because sysfs knobs like
84+
// /sys/fs/cgroup/cpu,cpuacct/cpu.rt_{period,runtime}_us do not exist.
85+
// So we need to check if the sysfs knobs exist before setting the variables.
86+
if _, err := os.Stat(filepath.Join(util.CPUCgroupPrefix, "cpu.rt_period_us")); !os.IsNotExist(err) {
87+
g.SetLinuxResourcesCPURealtimePeriod(defaultRealtimePeriod)
5588
}
56-
if lcd.Cpus != cpus {
57-
return fmt.Errorf("cpus cpus is not set correctly, expect: %s, actual: %s", cpus, lcd.Cpus)
89+
90+
if _, err := os.Stat(filepath.Join(util.CPUCgroupPrefix, "cpu.rt_runtime_us")); !os.IsNotExist(err) {
91+
g.SetLinuxResourcesCPURealtimeRuntime(defaultRealtimeRuntime)
5892
}
59-
if lcd.Mems != mems {
60-
return fmt.Errorf("cpus mems is not set correctly, expect: %s, actual: %s", mems, lcd.Mems)
93+
94+
if err := util.RuntimeOutsideValidate(g, t, util.ValidateLinuxResourcesCPU); err != nil {
95+
return fmt.Errorf("cannot validate CPU cgroups: %v", err)
6196
}
62-
return nil
63-
})
97+
}
98+
99+
return nil
100+
}
64101

102+
func testEmptyCPU() error {
103+
t := tap.New()
104+
t.Header(0)
105+
defer t.AutoPlan()
106+
107+
g, err := util.GetDefaultGenerator()
65108
if err != nil {
109+
return fmt.Errorf("cannot get default config from generator: %v", err)
110+
}
111+
g.InitConfigLinuxResourcesCPU()
112+
g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath)
113+
114+
if err := util.RuntimeOutsideValidate(g, t, util.ValidateLinuxResourcesCPUEmpty); err != nil {
115+
return fmt.Errorf("cannot validate empty CPU cgroups: %v", err)
116+
}
117+
118+
return nil
119+
}
120+
121+
func main() {
122+
if "linux" != runtime.GOOS {
123+
util.Fatal(fmt.Errorf("linux-specific cgroup test"))
124+
}
125+
126+
if err := testCPUCgroups(); err != nil {
127+
util.Fatal(err)
128+
}
129+
130+
if err := testEmptyCPU(); err != nil {
66131
util.Fatal(err)
67132
}
68133
}

‎validation/linux_cgroups_devices.go

+8-2
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,18 @@
11
package main
22

33
import (
4+
"github.com/mndrix/tap-go"
45
"github.com/opencontainers/runtime-tools/cgroups"
56
"github.com/opencontainers/runtime-tools/validation/util"
67
)
78

89
func main() {
910
var major1, minor1, major2, minor2, major3, minor3 int64 = 10, 229, 8, 20, 10, 200
11+
12+
t := tap.New()
13+
t.Header(0)
14+
defer t.AutoPlan()
15+
1016
g, err := util.GetDefaultGenerator()
1117
if err != nil {
1218
util.Fatal(err)
@@ -15,8 +21,8 @@ func main() {
1521
g.AddLinuxResourcesDevice(true, "c", &major1, &minor1, "rwm")
1622
g.AddLinuxResourcesDevice(true, "b", &major2, &minor2, "rw")
1723
g.AddLinuxResourcesDevice(true, "b", &major3, &minor3, "r")
18-
err = util.RuntimeOutsideValidate(g, util.ValidateLinuxResourcesDevices)
24+
err = util.RuntimeOutsideValidate(g, t, util.ValidateLinuxResourcesDevices)
1925
if err != nil {
20-
util.Fatal(err)
26+
t.Fail(err.Error())
2127
}
2228
}

0 commit comments

Comments
 (0)