From f05d46b88ba70327549c1a875132dbb8d8dbe81b Mon Sep 17 00:00:00 2001 From: Justin Kulikauskas <44813129+JustinKuli@users.noreply.github.com> Date: Fri, 14 Jun 2024 03:43:13 +0000 Subject: [PATCH] Add Compliance Events Many details about the event emitted and functions that verify the form of the fields are taken from other policy repositories. Many of the helpful functions added to the toolkit are similarly not original, but are an evolution of functions used in other repositories' tests. Signed-off-by: Justin Kulikauskas <44813129+JustinKuli@users.noreply.github.com> --- Makefile | 4 + api/v1beta1/policycore_types.go | 24 +++ pkg/compliance/k8sEventEmitter.go | 132 ++++++++++++++ pkg/testutils/courtesies.go | 36 ++++ pkg/testutils/courtesies_test.go | 155 +++++++++++++++++ pkg/testutils/toolkit.go | 138 ++++++++++++++- .../api/v1beta1/fakepolicy_types.go | 39 ++++- ...en-cluster-management.io_fakepolicies.yaml | 13 +- test/fakepolicy/config/deploy.yaml | 13 +- .../controllers/fakepolicy_controller.go | 81 ++++++++- test/fakepolicy/main.go | 2 +- .../test/compliance/compEvMutator_test.go | 123 +++++++++++++ .../test/compliance/complianceEvent_test.go | 163 ++++++++++++++++++ test/fakepolicy/test/compliance/suite_test.go | 84 +++++++++ 14 files changed, 988 insertions(+), 19 deletions(-) create mode 100644 pkg/compliance/k8sEventEmitter.go create mode 100644 pkg/testutils/courtesies_test.go create mode 100644 test/fakepolicy/test/compliance/compEvMutator_test.go create mode 100644 test/fakepolicy/test/compliance/complianceEvent_test.go create mode 100644 test/fakepolicy/test/compliance/suite_test.go diff --git a/Makefile b/Makefile index 2b23318..b856606 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,10 @@ test-basicsuite: manifests generate $(GINKGO) $(ENVTEST) ## Run just the basic s KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GINKGO) \ --coverpkg=./... --covermode=count --coverprofile=cover-basic.out ./test/fakepolicy/test/basic +test-compsuite: manifests generate $(GINKGO) $(ENVTEST) ## Run just the compliance event tests + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GINKGO) \ + --coverpkg=./... --covermode=count --coverprofile=cover-comp.out ./test/fakepolicy/test/compliance + .PHONY: fuzz-test fuzz-test: go test ./api/v1beta1 -fuzz=FuzzMatchesExcludeAll -fuzztime=20s diff --git a/api/v1beta1/policycore_types.go b/api/v1beta1/policycore_types.go index 2acde7a..7386478 100644 --- a/api/v1beta1/policycore_types.go +++ b/api/v1beta1/policycore_types.go @@ -193,3 +193,27 @@ type PolicyCore struct { Spec PolicyCoreSpec `json:"spec,omitempty"` Status PolicyCoreStatus `json:"status,omitempty"` } + +//+kubebuilder:object:generate=false + +// PolicyLike is an interface that policies should implement so that they can +// benefit from some of the general tools in the nucleus. +type PolicyLike interface { + client.Object + + // The ComplianceState (Compliant/NonCompliant) of the specific policy. + ComplianceState() ComplianceState + + // A human-readable string describing the current state of the policy, and why it is either + // Compliant or NonCompliant. + ComplianceMessage() string + + // The "parent" object on this cluster for the specific policy. Generally a Policy, in the API + // GroupVersion `policy.open-cluster-management.io/v1`. For namespaced kinds of policies, this + // will usually be the owner of the policy. For cluster-scoped policies, this must be stored + // some other way. + Parent() metav1.OwnerReference + + // The namespace of the "parent" object. + ParentNamespace() string +} diff --git a/pkg/compliance/k8sEventEmitter.go b/pkg/compliance/k8sEventEmitter.go new file mode 100644 index 0000000..076b5df --- /dev/null +++ b/pkg/compliance/k8sEventEmitter.go @@ -0,0 +1,132 @@ +// Copyright Contributors to the Open Cluster Management project + +package compliance + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + nucleusv1beta1 "open-cluster-management.io/governance-policy-nucleus/api/v1beta1" +) + +// K8sEmitter is an emitter of Kubernetes events which the policy framework +// watches for in order to aggregate and report policy status. +type K8sEmitter struct { + // Client is a Kubernetes client for the cluster where the compliance events + // will be created. + Client client.Client + + // Source contains optional information for where the event comes from. + Source corev1.EventSource + + // Mutators modify the Event after the fields are initially set, but before + // it is created on the cluster. They are run in the order they are defined. + Mutators []func(corev1.Event) (corev1.Event, error) +} + +// Emit creates the Kubernetes Event on the cluster. It returns an error if the +// API call fails. +func (e K8sEmitter) Emit(ctx context.Context, pl nucleusv1beta1.PolicyLike) error { + _, err := e.EmitEvent(ctx, pl) + + return err +} + +// EmitEvent creates the Kubernetes Event on the cluster. It returns the Event +// that was (at least) attempted to be created, and an error if the API call +// fails. +func (e K8sEmitter) EmitEvent(ctx context.Context, pl nucleusv1beta1.PolicyLike) (*corev1.Event, error) { + plGVK := pl.GetObjectKind().GroupVersionKind() + time := time.Now() + + // This event name matches the convention of recorders from client-go + name := fmt.Sprintf("%v.%x", pl.Parent().Name, time.UnixNano()) + + // The reason must match a pattern looked for by the policy framework + var reason string + if ns := pl.GetNamespace(); ns != "" { + reason = "policy: " + ns + "/" + pl.GetName() + } else { + reason = "policy: " + pl.GetName() + } + + // The message must begin with the compliance, then should go into a descriptive message + message := string(pl.ComplianceState()) + "; " + pl.ComplianceMessage() + + evType := "Normal" + if pl.ComplianceState() != nucleusv1beta1.Compliant { + evType = "Warning" + } + + src := corev1.EventSource{ + Component: e.Source.Component, + Host: e.Source.Host, + } + + // These fields are required for the event to function as expected + if src.Component == "" { + src.Component = "policy-nucleus-default" + } + + if src.Host == "" { + src.Host = "policy-nucleus-default" + } + + event := corev1.Event{ + TypeMeta: metav1.TypeMeta{ + Kind: "Event", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: pl.ParentNamespace(), + Labels: pl.GetLabels(), + Annotations: pl.GetAnnotations(), + }, + InvolvedObject: corev1.ObjectReference{ + Kind: pl.Parent().Kind, + Namespace: pl.ParentNamespace(), + Name: pl.Parent().Name, + UID: pl.Parent().UID, + APIVersion: pl.Parent().APIVersion, + }, + Reason: reason, + Message: message, + Source: src, + FirstTimestamp: metav1.NewTime(time), + LastTimestamp: metav1.NewTime(time), + Count: 1, + Type: evType, + EventTime: metav1.NewMicroTime(time), + Series: nil, + Action: "ComplianceStateUpdate", + Related: &corev1.ObjectReference{ + Kind: plGVK.Kind, + Namespace: pl.GetNamespace(), + Name: pl.GetName(), + UID: pl.GetUID(), + APIVersion: plGVK.GroupVersion().String(), + ResourceVersion: pl.GetResourceVersion(), + }, + ReportingController: src.Component, + ReportingInstance: src.Host, + } + + for _, mutatorFunc := range e.Mutators { + var err error + + event, err = mutatorFunc(event) + if err != nil { + return nil, err + } + } + + err := e.Client.Create(ctx, &event) + + return &event, err +} diff --git a/pkg/testutils/courtesies.go b/pkg/testutils/courtesies.go index 797ebc1..6f11fb9 100644 --- a/pkg/testutils/courtesies.go +++ b/pkg/testutils/courtesies.go @@ -3,13 +3,49 @@ package testutils import ( + "regexp" + "time" + + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) +// ObjNN returns a NamespacedName for the given Object. func ObjNN(obj client.Object) types.NamespacedName { return types.NamespacedName{ Namespace: obj.GetNamespace(), Name: obj.GetName(), } } + +// EventFilter filters the given events. Any of the filter parameters can be passed an empty +// value to ignore that field when filtering. The msg parameter will be compiled into a regex if +// possible. The since parameter checks against the event's EventTime - but if the event does not +// specify an EventTime, it will not be filtered out. +func EventFilter(evs []corev1.Event, evType, msg string, since time.Time) []corev1.Event { + msgRegex, err := regexp.Compile(msg) + if err != nil { + msgRegex = regexp.MustCompile(regexp.QuoteMeta(msg)) + } + + ans := make([]corev1.Event, 0) + + for _, ev := range evs { + if evType != "" && ev.Type != evType { + continue + } + + if !msgRegex.MatchString(ev.Message) { + continue + } + + if !ev.EventTime.IsZero() && since.After(ev.EventTime.Time) { + continue + } + + ans = append(ans, ev) + } + + return ans +} diff --git a/pkg/testutils/courtesies_test.go b/pkg/testutils/courtesies_test.go new file mode 100644 index 0000000..49d3dc3 --- /dev/null +++ b/pkg/testutils/courtesies_test.go @@ -0,0 +1,155 @@ +// Copyright Contributors to the Open Cluster Management project + +package testutils + +import ( + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestObjNN(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + inpObj client.Object + wantName string + wantNS string + }{ + "namespaced unstructured": { + inpObj: &unstructured.Unstructured{Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "foo", + "namespace": "world", + }, + }}, + wantName: "foo", + wantNS: "world", + }, + "cluster-scoped unstructured": { + inpObj: &unstructured.Unstructured{Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "bar", + }, + }}, + wantName: "bar", + wantNS: "", + }, + "(namespaced) configmap": { + inpObj: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "my-cm", + Namespace: "kube-one", + }}, + wantName: "my-cm", + wantNS: "kube-one", + }, + "(cluster-scoped) node": { + inpObj: &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "unit-tests-only", + }}, + wantName: "unit-tests-only", + wantNS: "", + }, + } + + for name, tcase := range tests { + got := ObjNN(tcase.inpObj) + + if got.Name != tcase.wantName { + t.Errorf("Wanted name '%v', got '%v' in test '%v'", tcase.wantName, got.Name, name) + } + + if got.Namespace != tcase.wantNS { + t.Errorf("Wanted namespace '%v', got '%v' in test '%v'", tcase.wantNS, got.Namespace, name) + } + } +} + +func TestEventFilter(t *testing.T) { + t.Parallel() + + now := time.Now() + old := now.Add(-time.Minute) + veryOld := now.Add(-time.Hour) + + sampleEvents := []corev1.Event{{ + Message: "hello", + Type: "Normal", + EventTime: metav1.NewMicroTime(veryOld), + }, { + Message: "goodbye", + Type: "Warning", + EventTime: metav1.NewMicroTime(old), + }, { + Message: "carpe diem [", + Type: "Normal", + EventTime: metav1.NewMicroTime(now), + }, { + Message: "what time is it?", + Type: "Warning", + }} + + tests := map[string]struct { + inpType string + inpMsg string + inpSince time.Time + wantIdxs []int + }{ + "#NoFilter": { + inpType: "", + inpMsg: "", + inpSince: time.Time{}, + wantIdxs: []int{0, 1, 2, 3}, + }, + "recent events, plus the one with no time specified": { + inpType: "", + inpMsg: "", + inpSince: now.Add(-5 * time.Minute), + wantIdxs: []int{1, 2, 3}, + }, + "only warnings": { + inpType: "Warning", + inpMsg: "", + inpSince: time.Time{}, + wantIdxs: []int{1, 3}, + }, + "basic regex for a space": { + inpType: "", + inpMsg: ".* .*", + inpSince: time.Time{}, + wantIdxs: []int{2, 3}, + }, + "just a space": { + inpType: "", + inpMsg: " ", + inpSince: time.Time{}, + wantIdxs: []int{2, 3}, + }, + "invalid inescaped regex": { + inpType: "", + inpMsg: "[", + inpSince: time.Time{}, + wantIdxs: []int{2}, + }, + } + + for name, tcase := range tests { + got := EventFilter(sampleEvents, tcase.inpType, tcase.inpMsg, tcase.inpSince) + + if len(got) != len(tcase.wantIdxs) { + t.Fatalf("Expected %v events to be returned, got %v in test %v: got events: %v", + len(tcase.wantIdxs), len(got), name, got) + } + + for i, wantIdx := range tcase.wantIdxs { + if sampleEvents[wantIdx].String() != got[i].String() { + t.Errorf("Mismatch on item #%v in test %v. Expected '%v' got '%v'", + i, name, sampleEvents[wantIdx], got[i]) + } + } + } +} diff --git a/pkg/testutils/toolkit.go b/pkg/testutils/toolkit.go index a3ae601..3fb4f43 100644 --- a/pkg/testutils/toolkit.go +++ b/pkg/testutils/toolkit.go @@ -5,11 +5,15 @@ package testutils import ( "context" "fmt" + "regexp" + "sort" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" gomegaTypes "github.com/onsi/gomega/types" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -22,6 +26,8 @@ type Toolkit struct { BackgroundCtx context.Context //nolint:containedctx // this is for convenience } +// NewToolkit returns a toolkit using the given Client, with some basic defaults. +// This is the preferred way to get a Toolkit instance, to avoid unset fields. func NewToolkit(client client.Client) Toolkit { return Toolkit{ Client: client, @@ -36,11 +42,8 @@ func NewToolkit(client client.Client) Toolkit { // cleanlyCreate creates the given object, and registers a callback to delete the object which // Ginkgo will call at the appropriate time. The error from the `Create` call is returned (so it // can be checked) and the `Delete` callback handles 'NotFound' errors as a success. -func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object) error { - // Save and then re-set the GVK because the API call removes it - savedGVK := obj.GetObjectKind().GroupVersionKind() +func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { createErr := tk.Create(ctx, obj) - obj.GetObjectKind().SetGroupVersionKind(savedGVK) if createErr == nil { ginkgo.DeferCleanup(func() { @@ -59,6 +62,83 @@ func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object) error { return createErr } +// Create uses the toolkit's client to save the object in the Kubernetes cluster. +// The only change in behavior is that it saves and restores the object's type +// information, which might otherwise be stripped during the API call. +func (tk Toolkit) Create( + ctx context.Context, obj client.Object, opts ...client.CreateOption, +) error { + savedGVK := obj.GetObjectKind().GroupVersionKind() + err := tk.Client.Create(ctx, obj, opts...) + obj.GetObjectKind().SetGroupVersionKind(savedGVK) + + return err +} + +// Patch uses the toolkit's client to patch the object in the Kubernetes cluster. +// The only change in behavior is that it saves and restores the object's type +// information, which might otherwise be stripped during the API call. +func (tk Toolkit) Patch( + ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption, +) error { + savedGVK := obj.GetObjectKind().GroupVersionKind() + err := tk.Client.Patch(ctx, obj, patch, opts...) + obj.GetObjectKind().SetGroupVersionKind(savedGVK) + + return err +} + +// Update uses the toolkit's client to update the object in the Kubernetes cluster. +// The only change in behavior is that it saves and restores the object's type +// information, which might otherwise be stripped during the API call. +func (tk Toolkit) Update( + ctx context.Context, obj client.Object, opts ...client.UpdateOption, +) error { + savedGVK := obj.GetObjectKind().GroupVersionKind() + err := tk.Client.Update(ctx, obj, opts...) + obj.GetObjectKind().SetGroupVersionKind(savedGVK) + + return err +} + +// This regular expression is copied from +// https://github.com/open-cluster-management-io/governance-policy-framework-addon/blob/v0.13.0/controllers/statussync/policy_status_sync.go#L220 +var compEventRegex = regexp.MustCompile(`(?i)^policy:\s*(?:([a-z0-9.-]+)\s*\/)?(.+)`) + +// GetComplianceEvents queries the cluster and returns a sorted list of the Kubernetes +// compliance events for the given policy. +func (tk Toolkit) GetComplianceEvents( + ctx context.Context, ns string, parentUID types.UID, templateName string, +) ([]corev1.Event, error) { + list := &corev1.EventList{} + + err := tk.List(ctx, list, client.InNamespace(ns)) + if err != nil { + return nil, err + } + + events := make([]corev1.Event, 0) + + for _, event := range list.Items { + event := event + + if event.InvolvedObject.UID != parentUID { + continue + } + + submatch := compEventRegex.FindStringSubmatch(event.Reason) + if len(submatch) >= 3 && submatch[2] == templateName { + events = append(events, event) + } + } + + sort.SliceStable(events, func(i, j int) bool { + return events[i].Name < events[j].Name + }) + + return events, nil +} + // EC runs assertions on asynchronous behavior, both *E*ventually and *C*onsistently, // using the polling and timeout settings of the toolkit. Its usage should feel familiar // to gomega users, simply skip the `.Should(...)` call and put your matcher as the second @@ -66,11 +146,57 @@ func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object) error { func (tk Toolkit) EC( actualOrCtx interface{}, matcher gomegaTypes.GomegaMatcher, optionalDescription ...interface{}, ) bool { + ginkgo.GinkgoHelper() + + // Add where the failure occurred to the description + eDesc := make([]interface{}, 1) + cDesc := make([]interface{}, 1) + + switch len(optionalDescription) { + case 0: + eDesc[0] = "Failed in Eventually" + cDesc[0] = "Failed in Consistently" + case 1: + if origDescFunc, ok := optionalDescription[0].(func() string); ok { + eDesc[0] = func() string { + return "Failed in Eventually; " + origDescFunc() + } + cDesc[0] = func() string { + return "Failed in Consistently; " + origDescFunc() + } + } else { + eDesc[0] = "Failed in Eventually; " + optionalDescription[0].(string) + cDesc[0] = "Failed in Consistently; " + optionalDescription[0].(string) + } + default: + eDesc[0] = "Failed in Eventually; " + optionalDescription[0].(string) + eDesc = append(eDesc, optionalDescription[1:]...) //nolint: makezero // appending is definitely correct + + cDesc[0] = "Failed in Consistently; " + optionalDescription[0].(string) + cDesc = append(cDesc, optionalDescription[1:]...) //nolint: makezero // appending is definitely correct + } + gomega.Eventually( actualOrCtx, tk.EventuallyTimeout, tk.EventuallyPoll, - ).Should(matcher, optionalDescription...) + ).Should(matcher, eDesc...) return gomega.Consistently( actualOrCtx, tk.ConsistentlyTimeout, tk.ConsistentlyPoll, - ).Should(matcher, optionalDescription...) + ).Should(matcher, cDesc...) +} + +// RegisterDebugMessage returns a pointer to a string which will be logged at the +// end of the test only if the test fails. This is particularly useful for logging +// information only once in an Eventually or Consistently function. +// Note: using a custom description message may be a better practice overall. +func RegisterDebugMessage() *string { + var debugMsg string + + ginkgo.DeferCleanup(func() { + if ginkgo.CurrentSpecReport().Failed() { + ginkgo.GinkgoWriter.Println(debugMsg) + } + }) + + return &debugMsg } diff --git a/test/fakepolicy/api/v1beta1/fakepolicy_types.go b/test/fakepolicy/api/v1beta1/fakepolicy_types.go index 32ea63e..fd5d6ce 100644 --- a/test/fakepolicy/api/v1beta1/fakepolicy_types.go +++ b/test/fakepolicy/api/v1beta1/fakepolicy_types.go @@ -12,11 +12,18 @@ import ( type FakePolicySpec struct { nucleusv1beta1.PolicyCoreSpec `json:",inline"` - // targetConfigMaps defines the ConfigMaps which should be examined by this policy + // TargetConfigMaps defines the ConfigMaps which should be examined by this policy TargetConfigMaps nucleusv1beta1.Target `json:"targetConfigMaps,omitempty"` - // targetUsingReflection defines whether to use reflection to find the ConfigMaps + // TargetUsingReflection defines whether to use reflection to find the ConfigMaps TargetUsingReflection bool `json:"targetUsingReflection,omitempty"` + + // DesiredConfigMapName - if this name is not found, the policy will report a violation + DesiredConfigMapName string `json:"desiredConfigMapName,omitempty"` + + // EventAnnotation - if provided, this value will be annotated on the compliance + // events, under the "policy.open-cluster-management.io/test" key + EventAnnotation string `json:"eventAnnotation,omitempty"` } //+kubebuilder:validation:Optional @@ -41,6 +48,34 @@ type FakePolicy struct { Status FakePolicyStatus `json:"status,omitempty"` } +// ensure FakePolicy implements PolicyLike +var _ nucleusv1beta1.PolicyLike = (*FakePolicy)(nil) + +func (f FakePolicy) ComplianceState() nucleusv1beta1.ComplianceState { + return f.Status.ComplianceState +} + +func (f FakePolicy) ComplianceMessage() string { + idx, compCond := f.Status.GetCondition("Compliant") + if idx == -1 { + return "" + } + + return compCond.Message +} + +func (f FakePolicy) Parent() metav1.OwnerReference { + if len(f.OwnerReferences) == 0 { + return metav1.OwnerReference{} + } + + return f.OwnerReferences[0] +} + +func (f FakePolicy) ParentNamespace() string { + return f.Namespace +} + //+kubebuilder:object:root=true // FakePolicyList contains a list of FakePolicy diff --git a/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml b/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml index 104214a..58afa30 100644 --- a/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml +++ b/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml @@ -39,6 +39,15 @@ spec: spec: description: FakePolicySpec defines the desired state of FakePolicy properties: + desiredConfigMapName: + description: DesiredConfigMapName - if this name is not found, the + policy will report a violation + type: string + eventAnnotation: + description: |- + EventAnnotation - if provided, this value will be annotated on the compliance + events, under the "policy.open-cluster-management.io/test" key + type: string namespaceSelector: description: |- NamespaceSelector indicates which namespaces on the cluster this policy @@ -130,7 +139,7 @@ spec: - Critical type: string targetConfigMaps: - description: targetConfigMaps defines the ConfigMaps which should + description: TargetConfigMaps defines the ConfigMaps which should be examined by this policy properties: exclude: @@ -196,7 +205,7 @@ spec: type: object x-kubernetes-map-type: atomic targetUsingReflection: - description: targetUsingReflection defines whether to use reflection + description: TargetUsingReflection defines whether to use reflection to find the ConfigMaps type: boolean type: object diff --git a/test/fakepolicy/config/deploy.yaml b/test/fakepolicy/config/deploy.yaml index 90a6203..79dc8a5 100644 --- a/test/fakepolicy/config/deploy.yaml +++ b/test/fakepolicy/config/deploy.yaml @@ -48,6 +48,15 @@ spec: spec: description: FakePolicySpec defines the desired state of FakePolicy properties: + desiredConfigMapName: + description: DesiredConfigMapName - if this name is not found, the + policy will report a violation + type: string + eventAnnotation: + description: |- + EventAnnotation - if provided, this value will be annotated on the compliance + events, under the "policy.open-cluster-management.io/test" key + type: string namespaceSelector: description: |- NamespaceSelector indicates which namespaces on the cluster this policy @@ -139,7 +148,7 @@ spec: - Critical type: string targetConfigMaps: - description: targetConfigMaps defines the ConfigMaps which should + description: TargetConfigMaps defines the ConfigMaps which should be examined by this policy properties: exclude: @@ -205,7 +214,7 @@ spec: type: object x-kubernetes-map-type: atomic targetUsingReflection: - description: targetUsingReflection defines whether to use reflection + description: TargetUsingReflection defines whether to use reflection to find the ConfigMaps type: boolean type: object diff --git a/test/fakepolicy/controllers/fakepolicy_controller.go b/test/fakepolicy/controllers/fakepolicy_controller.go index ab0999f..7425d08 100644 --- a/test/fakepolicy/controllers/fakepolicy_controller.go +++ b/test/fakepolicy/controllers/fakepolicy_controller.go @@ -19,16 +19,22 @@ import ( nucleusv1alpha1 "open-cluster-management.io/governance-policy-nucleus/api/v1alpha1" nucleusv1beta1 "open-cluster-management.io/governance-policy-nucleus/api/v1beta1" + "open-cluster-management.io/governance-policy-nucleus/pkg/compliance" fakev1beta1 "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/api/v1beta1" ) -// FakePolicyReconciler reconciles a FakePolicy object +// FakePolicyReconciler reconciles a FakePolicy object. +// NOTE: it does not watch anything other than FakePolcies, so it will not react +// to other changes in the cluster - update something on the policy to make it +// re-reconcile. type FakePolicyReconciler struct { client.Client Scheme *runtime.Scheme DynamicClient *dynamic.DynamicClient } +const mutatorAnno string = "policy.open-cluster-management.io/test" + // Usual RBAC for fakepolicy: //+kubebuilder:rbac:groups=policy.open-cluster-management.io,resources=fakepolicies,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=policy.open-cluster-management.io,resources=fakepolicies/status,verbs=get;update;patch @@ -42,11 +48,13 @@ type FakePolicyReconciler struct { func (r *FakePolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) + log.Info("Starting a reconcile") policy := &fakev1beta1.FakePolicy{} if err := r.Get(ctx, req.NamespacedName, policy); err != nil { if errors.IsNotFound(err) { - // Request object not found, probably deleted + log.Info("Request object not found, probably deleted") + return ctrl.Result{}, nil } @@ -55,7 +63,33 @@ func (r *FakePolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - r.doSelections(ctx, policy) + cmFound := r.doSelections(ctx, policy) + + policy.Status.SelectionComplete = true + + complianceCondition := metav1.Condition{ + Type: "Compliant", + Status: metav1.ConditionTrue, + Reason: "Found", + Message: "the desired configmap was found", + } + + policy.Status.ComplianceState = nucleusv1beta1.Compliant + + if !cmFound { + complianceCondition.Status = metav1.ConditionFalse + complianceCondition.Reason = "NotFound" + complianceCondition.Message = "the desired configmap was missing" + policy.Status.ComplianceState = nucleusv1beta1.NonCompliant + } + + changed := policy.Status.UpdateCondition(complianceCondition) + + if !changed { + log.Info("No change; no compliance event to emit") + + return ctrl.Result{}, nil + } if err := r.Status().Update(ctx, policy); err != nil { log.Error(err, "Failed to update status") @@ -63,10 +97,41 @@ func (r *FakePolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - return ctrl.Result{}, nil + emitter := compliance.K8sEmitter{ + Client: r.Client, + } + + if policy.Spec.EventAnnotation != "" { + emitter.Mutators = []func(inpEv corev1.Event) (corev1.Event, error){ + func(inpEv corev1.Event) (corev1.Event, error) { + if inpEv.Annotations == nil { + inpEv.Annotations = make(map[string]string) + } + + inpEv.Annotations[mutatorAnno] = policy.Spec.EventAnnotation + + return inpEv, nil + }, + } + + // it's cheating a bit to put this here but it's helpful to test that + // the events work both when this is and when this is not specified + emitter.Source = corev1.EventSource{ + Component: policy.Spec.EventAnnotation, + Host: policy.Spec.EventAnnotation, + } + } + + ev, err := emitter.EmitEvent(ctx, policy) + + log.Info("Event emitted", "eventName", ev.Name) + + return ctrl.Result{}, err } -func (r *FakePolicyReconciler) doSelections(ctx context.Context, policy *fakev1beta1.FakePolicy) { +func (r *FakePolicyReconciler) doSelections( + ctx context.Context, policy *fakev1beta1.FakePolicy, +) (configMapFound bool) { log := log.FromContext(ctx) nsCond := metav1.Condition{ @@ -150,6 +215,10 @@ func (r *FakePolicyReconciler) doSelections(ctx context.Context, policy *fakev1b clientCMs := make([]string, len(clientMatchedCMs)) for i, cm := range dynamicMatchedCMs { clientCMs[i] = cm.GetNamespace() + "/" + cm.GetName() + + if cm.GetName() == policy.Spec.DesiredConfigMapName { + configMapFound = true + } } slices.Sort(clientCMs) @@ -159,7 +228,7 @@ func (r *FakePolicyReconciler) doSelections(ctx context.Context, policy *fakev1b policy.Status.UpdateCondition(clientCond) - policy.Status.SelectionComplete = true + return configMapFound } type configMapResList struct { diff --git a/test/fakepolicy/main.go b/test/fakepolicy/main.go index c81d02e..ee518cc 100644 --- a/test/fakepolicy/main.go +++ b/test/fakepolicy/main.go @@ -77,7 +77,7 @@ func Run(parentCtx context.Context, cfg *rest.Config) error { cfg, err = ctrl.GetConfig() if err != nil { - setupLog.Error(err, "unable to get kubernetes config") + setupLog.Error(err, "unable to get Kubernetes config") return err } diff --git a/test/fakepolicy/test/compliance/compEvMutator_test.go b/test/fakepolicy/test/compliance/compEvMutator_test.go new file mode 100644 index 0000000..a6f3e49 --- /dev/null +++ b/test/fakepolicy/test/compliance/compEvMutator_test.go @@ -0,0 +1,123 @@ +// Copyright Contributors to the Open Cluster Management project + +package compliance + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "open-cluster-management.io/governance-policy-nucleus/pkg/testutils" + fakev1beta1 "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/api/v1beta1" + . "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/test/utils" +) + +var _ = Describe("Compliance Events with a Mutator", Ordered, func() { + const testNS string = "mutator-comp-test" + const annoKey string = "policy.open-cluster-management.io/test" + + var ( + parent *corev1.ConfigMap + policy *fakev1beta1.FakePolicy + ) + + BeforeAll(func(ctx SpecContext) { + ns := &corev1.Namespace{ + // TypeMeta is not required here; the Client can infer it + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + }, + } + Expect(tk.CleanlyCreate(ctx, ns)).To(Succeed()) + + parent = &corev1.ConfigMap{ + // TypeMeta is useful here for the OwnerReference + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "parent", + Namespace: testNS, + }, + } + Expect(tk.CleanlyCreate(ctx, parent)).To(Succeed()) + + sample := SampleFakePolicy() + policy = &sample + policy.Name += "-mutator-comp-test" + policy.Namespace = testNS + policy.Spec.DesiredConfigMapName = "hello-world" + policy.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: parent.APIVersion, + Kind: parent.Kind, + Name: parent.Name, + UID: parent.UID, + }} + Expect(tk.CleanlyCreate(ctx, policy)).To(Succeed()) + }) + + It("Should start NonCompliant", func(ctx SpecContext) { + tk.EC(func(g Gomega) string { + g.Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + return string(policy.Status.ComplianceState) + }, Equal("NonCompliant")) + }) + + It("Should emit one NonCompliant event without the annotation", func(ctx SpecContext) { + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + evs = testutils.EventFilter(evs, "Warning", "NonCompliant", time.Time{}) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + if _, ok := ev.Annotations[annoKey]; ok { + continue + } + + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) + + It("Should emit with the annotation, when eventAnnotation is set", func(ctx SpecContext) { + Expect(tk.CleanlyCreate(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "hello-world", + Namespace: "default", + Labels: map[string]string{"sample": ""}, + }})).To(Succeed()) + + // Refresh the test's copy of the policy + Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + By("Setting the eventAnnotation field on the policy") + policy.Spec.EventAnnotation = "borogoves" + Expect(tk.Update(ctx, policy)).To(Succeed()) + + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + evs = testutils.EventFilter(evs, "Normal", "^Compliant", time.Time{}) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + if ev.Annotations[annoKey] != "borogoves" { + continue + } + + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) +}) diff --git a/test/fakepolicy/test/compliance/complianceEvent_test.go b/test/fakepolicy/test/compliance/complianceEvent_test.go new file mode 100644 index 0000000..6754aec --- /dev/null +++ b/test/fakepolicy/test/compliance/complianceEvent_test.go @@ -0,0 +1,163 @@ +// Copyright Contributors to the Open Cluster Management project + +package compliance + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "open-cluster-management.io/governance-policy-nucleus/pkg/testutils" + fakev1beta1 "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/api/v1beta1" + . "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/test/utils" +) + +var _ = Describe("Classic Compliance Events", Ordered, func() { + const testNS string = "classic-comp-test" + + var ( + parent *corev1.ConfigMap + policy *fakev1beta1.FakePolicy + ) + + BeforeAll(func(ctx SpecContext) { + ns := &corev1.Namespace{ + // TypeMeta is not required here; the Client can infer it + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + }, + } + Expect(tk.CleanlyCreate(ctx, ns)).To(Succeed()) + + parent = &corev1.ConfigMap{ + // TypeMeta is useful here for the OwnerReference + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "parent", + Namespace: testNS, + }, + } + Expect(tk.CleanlyCreate(ctx, parent)).To(Succeed()) + + sample := SampleFakePolicy() + policy = &sample + policy.Name += "-classic-comp-test" + policy.Namespace = testNS + policy.Spec.DesiredConfigMapName = "hello-world" + policy.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: parent.APIVersion, + Kind: parent.Kind, + Name: parent.Name, + UID: parent.UID, + }} + Expect(tk.CleanlyCreate(ctx, policy)).To(Succeed()) + }) + + It("Should start NonCompliant", func(ctx SpecContext) { + tk.EC(func(g Gomega) string { + g.Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + return string(policy.Status.ComplianceState) + }, Equal("NonCompliant")) + }) + + It("Should emit one NonCompliant event", func(ctx SpecContext) { + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + evs = testutils.EventFilter(evs, "Warning", "NonCompliant", time.Time{}) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) + + It("Should emit one Compliant event after the configmap is created", func(ctx SpecContext) { + Expect(tk.CleanlyCreate(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "hello-world", + Namespace: "default", + Labels: map[string]string{"sample": ""}, + }})).To(Succeed()) + + // Refresh the test's copy of the policy + Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + By("Setting an annotation on the policy to trigger a re-reconcile") + policy.SetAnnotations(map[string]string{ + "classic-comp-test-1": "1", + }) + Expect(tk.Update(ctx, policy)).To(Succeed()) + + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + evs = testutils.EventFilter(evs, "Normal", "^Compliant", time.Time{}) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) + + It("Should emit a NonCompliant event after the configmap is deleted", func(ctx SpecContext) { + By("Ensuring that the configmap is gone") + Eventually(func() string { + cm := corev1.ConfigMap{} + _ = tk.Get(ctx, types.NamespacedName{Name: "hello-world", Namespace: "default"}, &cm) + + return cm.Name + }, "1s", "50ms").Should(BeEmpty()) + + // Refresh the test's copy of the policy + Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + By("Patching an annotation on the policy to trigger a re-reconcile") + patch := `[{"op":"replace","path":"/metadata/annotations/classic-comp-test-1","value":"2"}]` + err := tk.Patch(ctx, policy, client.RawPatch(types.JSONPatchType, []byte(patch))) + Expect(err).NotTo(HaveOccurred()) + + // This is just an example usage of this function, not an actual case where it was necessary. + // It's _obvious_ (/s) that the test must use a filter for 2 seconds ago; if it filtered to + // only 1 second ago, then since the Consistently runs for a whole second, it would always + // get an empty list near the end, and fail. + debugMsg := testutils.RegisterDebugMessage() + + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + *debugMsg = "unfiltered events: " + for _, ev := range evs { + *debugMsg += fmt.Sprintf("(%v: %v), ", ev.Name, ev.Message) + } + + evs = testutils.EventFilter(evs, "Warning", "NonCompliant", + time.Now().Add(-2*time.Second)) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) +}) diff --git a/test/fakepolicy/test/compliance/suite_test.go b/test/fakepolicy/test/compliance/suite_test.go new file mode 100644 index 0000000..1d153f5 --- /dev/null +++ b/test/fakepolicy/test/compliance/suite_test.go @@ -0,0 +1,84 @@ +// Copyright Contributors to the Open Cluster Management project + +package compliance + +import ( + "context" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "open-cluster-management.io/governance-policy-nucleus/pkg/testutils" + "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy" + fakev1beta1 "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/api/v1beta1" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + cfg *rest.Config + k8sClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + tk testutils.Toolkit +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Compliance Details Suite") +} + +var _ = BeforeSuite(func() { + format.TruncatedDiff = false + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = fakev1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + tk = testutils.NewToolkit(k8sClient) + tk.BackgroundCtx = ctx + + go func() { + defer GinkgoRecover() + Expect(fakepolicy.Run(ctx, cfg)).To(Succeed()) + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +})