From 35b5b55c686080de64facf127d6d6a5ca9a0fe6b Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Fri, 3 Dec 2021 13:51:17 +0000 Subject: [PATCH 1/4] v2: Fix inotify leak when cgroup is deleted When running on cgroup2, we currently leak the inotify instance (and goroutine blocked on read) used to monitor 'memory.events' on any container exit. This is highly problematic when containers are automatically restarted because we will exhaust either the fd limit or system-wide inotify instance limit. When a process exits, there is no memory event and even when the cgroup is deleted, the inotify read is also not unblocked. This is not the case when containerd is running on cgroup (v1) because that uses a different mechanism for notification and detects cgroup deletion. Fulfill the contract on cgroup2 by additionally monitoring cgroup.events for process exit. When the last process exits the kernel signals an event on 'cgroup.events'. For robustness we check both 'cgroup.events' and 'memory.events' on any notification and also handle ENOENT/ENODEV errors from read/open of 'memory.events'. We signal exit up the stack by closing the error channel. Strangely, the error channel was not previously being returned to the caller. Signed-off-by: Jeremi Piotrowski --- v2/manager.go | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/v2/manager.go b/v2/manager.go index b56989bb..c8cf0e93 100644 --- a/v2/manager.go +++ b/v2/manager.go @@ -560,6 +560,22 @@ func (c *Manager) freeze(path string, state State) error { } } +func (c *Manager) isCgroupEmpty() bool { + // In case of any error we return true so that we exit and don't leak resources + out := make(map[string]interface{}) + if err := readKVStatsFile(c.path, "cgroup.events", out); err != nil { + return true + } + if v, ok := out["populated"]; ok { + populated, ok := v.(uint64) + if !ok { + return true + } + return populated == 0 + } + return true +} + // MemoryEventFD returns inotify file descriptor and 'memory.events' inotify watch descriptor func (c *Manager) MemoryEventFD() (int, uint32, error) { fpath := filepath.Join(c.path, "memory.events") @@ -568,9 +584,15 @@ func (c *Manager) MemoryEventFD() (int, uint32, error) { return 0, 0, errors.New("failed to create inotify fd") } wd, err := syscall.InotifyAddWatch(fd, fpath, unix.IN_MODIFY) - if wd < 0 { + if err != nil { syscall.Close(fd) - return 0, 0, fmt.Errorf("failed to add inotify watch for %q", fpath) + return 0, 0, fmt.Errorf("failed to add inotify watch for %q: %w", fpath, err) + } + // monitor to detect process exit/cgroup deletion + evpath := filepath.Join(c.path, "cgroup.events") + if _, err = syscall.InotifyAddWatch(fd, evpath, unix.IN_MODIFY); err != nil { + syscall.Close(fd) + return 0, 0, fmt.Errorf("failed to add inotify watch for %q: %w", evpath, err) } return fd, uint32(wd), nil @@ -578,22 +600,21 @@ func (c *Manager) MemoryEventFD() (int, uint32, error) { func (c *Manager) EventChan() (<-chan Event, <-chan error) { ec := make(chan Event) - errCh := make(chan error) + errCh := make(chan error, 1) go c.waitForEvents(ec, errCh) - return ec, nil + return ec, errCh } func (c *Manager) waitForEvents(ec chan<- Event, errCh chan<- error) { - fd, wd, err := c.MemoryEventFD() - - defer syscall.InotifyRmWatch(fd, wd) - defer syscall.Close(fd) + defer close(errCh) + fd, _, err := c.MemoryEventFD() if err != nil { errCh <- err return } + defer syscall.Close(fd) for { buffer := make([]byte, syscall.SizeofInotifyEvent*10) @@ -643,7 +664,13 @@ func (c *Manager) waitForEvents(ec chan<- Event, errCh chan<- error) { } ec <- e } else { - errCh <- err + // When cgroup is deleted read may return -ENODEV instead of -ENOENT from open. + if _, statErr := os.Lstat(filepath.Join(c.path, "memory.events")); !os.IsNotExist(statErr) { + errCh <- err + } + return + } + if c.isCgroupEmpty() { return } } From 6a46df25065d551a8d998495c34263354c84f2c1 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Mon, 31 Jan 2022 14:25:00 +0100 Subject: [PATCH 2/4] v2: manager: factor out memory.events parsing This makes waitForEvents() more shorter and more readable. Signed-off-by: Jeremi Piotrowski --- v2/manager.go | 74 +++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/v2/manager.go b/v2/manager.go index c8cf0e93..28655f8e 100644 --- a/v2/manager.go +++ b/v2/manager.go @@ -606,6 +606,41 @@ func (c *Manager) EventChan() (<-chan Event, <-chan error) { return ec, errCh } +func parseMemoryEvents(out map[string]interface{}) (Event, error) { + e := Event{} + if v, ok := out["high"]; ok { + e.High, ok = v.(uint64) + if !ok { + return Event{}, fmt.Errorf("cannot convert high to uint64: %+v", v) + } + } + if v, ok := out["low"]; ok { + e.Low, ok = v.(uint64) + if !ok { + return Event{}, fmt.Errorf("cannot convert low to uint64: %+v", v) + } + } + if v, ok := out["max"]; ok { + e.Max, ok = v.(uint64) + if !ok { + return Event{}, fmt.Errorf("cannot convert max to uint64: %+v", v) + } + } + if v, ok := out["oom"]; ok { + e.OOM, ok = v.(uint64) + if !ok { + return Event{}, fmt.Errorf("cannot convert oom to uint64: %+v", v) + } + } + if v, ok := out["oom_kill"]; ok { + e.OOMKill, ok = v.(uint64) + if !ok { + return Event{}, fmt.Errorf("cannot convert oom_kill to uint64: %+v", v) + } + } + return e, nil +} + func (c *Manager) waitForEvents(ec chan<- Event, errCh chan<- error) { defer close(errCh) @@ -626,41 +661,10 @@ func (c *Manager) waitForEvents(ec chan<- Event, errCh chan<- error) { if bytesRead >= syscall.SizeofInotifyEvent { out := make(map[string]interface{}) if err := readKVStatsFile(c.path, "memory.events", out); err == nil { - e := Event{} - if v, ok := out["high"]; ok { - e.High, ok = v.(uint64) - if !ok { - errCh <- fmt.Errorf("cannot convert high to uint64: %+v", v) - return - } - } - if v, ok := out["low"]; ok { - e.Low, ok = v.(uint64) - if !ok { - errCh <- fmt.Errorf("cannot convert low to uint64: %+v", v) - return - } - } - if v, ok := out["max"]; ok { - e.Max, ok = v.(uint64) - if !ok { - errCh <- fmt.Errorf("cannot convert max to uint64: %+v", v) - return - } - } - if v, ok := out["oom"]; ok { - e.OOM, ok = v.(uint64) - if !ok { - errCh <- fmt.Errorf("cannot convert oom to uint64: %+v", v) - return - } - } - if v, ok := out["oom_kill"]; ok { - e.OOMKill, ok = v.(uint64) - if !ok { - errCh <- fmt.Errorf("cannot convert oom_kill to uint64: %+v", v) - return - } + e, err := parseMemoryEvents(out) + if err != nil { + errCh <- err + return } ec <- e } else { From cf1f978b93bf784118d3ab7dec6a47b8204918c2 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Thu, 27 Jan 2022 14:04:06 +0000 Subject: [PATCH 3/4] v2: flip error handling for readKVStat("memory.events") to reduce indentation Signed-off-by: Jeremi Piotrowski --- v2/manager.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/v2/manager.go b/v2/manager.go index 28655f8e..afed14c6 100644 --- a/v2/manager.go +++ b/v2/manager.go @@ -660,20 +660,19 @@ func (c *Manager) waitForEvents(ec chan<- Event, errCh chan<- error) { } if bytesRead >= syscall.SizeofInotifyEvent { out := make(map[string]interface{}) - if err := readKVStatsFile(c.path, "memory.events", out); err == nil { - e, err := parseMemoryEvents(out) - if err != nil { - errCh <- err - return - } - ec <- e - } else { + if err := readKVStatsFile(c.path, "memory.events", out); err != nil { // When cgroup is deleted read may return -ENODEV instead of -ENOENT from open. if _, statErr := os.Lstat(filepath.Join(c.path, "memory.events")); !os.IsNotExist(statErr) { errCh <- err } return } + e, err := parseMemoryEvents(out) + if err != nil { + errCh <- err + return + } + ec <- e if c.isCgroupEmpty() { return } From a7d6888aa30218c8aff15d979eb3f6aec0b7979c Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Thu, 27 Jan 2022 11:04:36 +0000 Subject: [PATCH 4/4] v2: add test case for Manager.EventChan() behavior This test case demonstrates that now the oom monitor goroutine terminates when a cgroup is removed. Systemd is responsible for cgroup removal so a "cat" process is spawned in a scope, with stdin attached to a pipe. When the pipe is closed the process terminates, systemd notices and removes the cgroup, which results in the test case finishing. Signed-off-by: Jeremi Piotrowski --- go.mod | 5 +-- go.sum | 26 +++++++++++++--- v2/manager_test.go | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 v2/manager_test.go diff --git a/go.mod b/go.mod index b3df917d..80d3f6ea 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,8 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/opencontainers/runtime-spec v1.0.2 github.com/sirupsen/logrus v1.8.1 - github.com/stretchr/testify v1.6.1 + github.com/stretchr/testify v1.7.0 github.com/urfave/cli v1.22.2 - golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c + go.uber.org/goleak v1.1.12 + golang.org/x/sys v0.0.0-20210510120138-977fb7262007 ) diff --git a/go.sum b/go.sum index 20ed896a..cda30b49 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,7 @@ github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -38,43 +39,60 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/v2/manager_test.go b/v2/manager_test.go new file mode 100644 index 00000000..35d104ca --- /dev/null +++ b/v2/manager_test.go @@ -0,0 +1,76 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v2 + +import ( + "fmt" + "os/exec" + "testing" + "time" + + "go.uber.org/goleak" +) + +func TestEventChanCleanupOnCgroupRemoval(t *testing.T) { + checkCgroupMode(t) + + cmd := exec.Command("cat") + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatalf("Failed to create cat process: %v", err) + } + if err := cmd.Start(); err != nil { + t.Fatalf("Failed to start cat process: %v", err) + } + proc := cmd.Process + if proc == nil { + t.Fatal("Process is nil") + } + + group := fmt.Sprintf("testing-watcher-%d.scope", proc.Pid) + c, err := NewSystemd("", group, proc.Pid, &Resources{}) + if err != nil { + t.Fatalf("Failed to init new cgroup manager: %v", err) + } + + evCh, errCh := c.EventChan() + + // give event goroutine a chance to start + time.Sleep(500 * time.Millisecond) + + if err := stdin.Close(); err != nil { + t.Fatalf("Failed closing stdin: %v", err) + } + if err := cmd.Wait(); err != nil { + t.Fatalf("Failed waiting for cmd: %v", err) + } + + done := false + for !done { + select { + case <-evCh: + case err := <-errCh: + if err != nil { + t.Fatalf("Unexpected error on error channel: %v", err) + } + done = true + case <-time.After(5 * time.Second): + t.Fatal("Timed out") + } + } + goleak.VerifyNone(t) +}