diff --git a/Makefile b/Makefile index 2b23318..b856606 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,10 @@ test-basicsuite: manifests generate $(GINKGO) $(ENVTEST) ## Run just the basic s KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GINKGO) \ --coverpkg=./... --covermode=count --coverprofile=cover-basic.out ./test/fakepolicy/test/basic +test-compsuite: manifests generate $(GINKGO) $(ENVTEST) ## Run just the compliance event tests + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GINKGO) \ + --coverpkg=./... --covermode=count --coverprofile=cover-comp.out ./test/fakepolicy/test/compliance + .PHONY: fuzz-test fuzz-test: go test ./api/v1beta1 -fuzz=FuzzMatchesExcludeAll -fuzztime=20s diff --git a/api/v1beta1/policycore_types.go b/api/v1beta1/policycore_types.go index 2acde7a..7386478 100644 --- a/api/v1beta1/policycore_types.go +++ b/api/v1beta1/policycore_types.go @@ -193,3 +193,27 @@ type PolicyCore struct { Spec PolicyCoreSpec `json:"spec,omitempty"` Status PolicyCoreStatus `json:"status,omitempty"` } + +//+kubebuilder:object:generate=false + +// PolicyLike is an interface that policies should implement so that they can +// benefit from some of the general tools in the nucleus. +type PolicyLike interface { + client.Object + + // The ComplianceState (Compliant/NonCompliant) of the specific policy. + ComplianceState() ComplianceState + + // A human-readable string describing the current state of the policy, and why it is either + // Compliant or NonCompliant. + ComplianceMessage() string + + // The "parent" object on this cluster for the specific policy. Generally a Policy, in the API + // GroupVersion `policy.open-cluster-management.io/v1`. For namespaced kinds of policies, this + // will usually be the owner of the policy. For cluster-scoped policies, this must be stored + // some other way. + Parent() metav1.OwnerReference + + // The namespace of the "parent" object. + ParentNamespace() string +} diff --git a/pkg/compliance/k8sEventEmitter.go b/pkg/compliance/k8sEventEmitter.go new file mode 100644 index 0000000..076b5df --- /dev/null +++ b/pkg/compliance/k8sEventEmitter.go @@ -0,0 +1,132 @@ +// Copyright Contributors to the Open Cluster Management project + +package compliance + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + nucleusv1beta1 "open-cluster-management.io/governance-policy-nucleus/api/v1beta1" +) + +// K8sEmitter is an emitter of Kubernetes events which the policy framework +// watches for in order to aggregate and report policy status. +type K8sEmitter struct { + // Client is a Kubernetes client for the cluster where the compliance events + // will be created. + Client client.Client + + // Source contains optional information for where the event comes from. + Source corev1.EventSource + + // Mutators modify the Event after the fields are initially set, but before + // it is created on the cluster. They are run in the order they are defined. + Mutators []func(corev1.Event) (corev1.Event, error) +} + +// Emit creates the Kubernetes Event on the cluster. It returns an error if the +// API call fails. +func (e K8sEmitter) Emit(ctx context.Context, pl nucleusv1beta1.PolicyLike) error { + _, err := e.EmitEvent(ctx, pl) + + return err +} + +// EmitEvent creates the Kubernetes Event on the cluster. It returns the Event +// that was (at least) attempted to be created, and an error if the API call +// fails. +func (e K8sEmitter) EmitEvent(ctx context.Context, pl nucleusv1beta1.PolicyLike) (*corev1.Event, error) { + plGVK := pl.GetObjectKind().GroupVersionKind() + time := time.Now() + + // This event name matches the convention of recorders from client-go + name := fmt.Sprintf("%v.%x", pl.Parent().Name, time.UnixNano()) + + // The reason must match a pattern looked for by the policy framework + var reason string + if ns := pl.GetNamespace(); ns != "" { + reason = "policy: " + ns + "/" + pl.GetName() + } else { + reason = "policy: " + pl.GetName() + } + + // The message must begin with the compliance, then should go into a descriptive message + message := string(pl.ComplianceState()) + "; " + pl.ComplianceMessage() + + evType := "Normal" + if pl.ComplianceState() != nucleusv1beta1.Compliant { + evType = "Warning" + } + + src := corev1.EventSource{ + Component: e.Source.Component, + Host: e.Source.Host, + } + + // These fields are required for the event to function as expected + if src.Component == "" { + src.Component = "policy-nucleus-default" + } + + if src.Host == "" { + src.Host = "policy-nucleus-default" + } + + event := corev1.Event{ + TypeMeta: metav1.TypeMeta{ + Kind: "Event", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: pl.ParentNamespace(), + Labels: pl.GetLabels(), + Annotations: pl.GetAnnotations(), + }, + InvolvedObject: corev1.ObjectReference{ + Kind: pl.Parent().Kind, + Namespace: pl.ParentNamespace(), + Name: pl.Parent().Name, + UID: pl.Parent().UID, + APIVersion: pl.Parent().APIVersion, + }, + Reason: reason, + Message: message, + Source: src, + FirstTimestamp: metav1.NewTime(time), + LastTimestamp: metav1.NewTime(time), + Count: 1, + Type: evType, + EventTime: metav1.NewMicroTime(time), + Series: nil, + Action: "ComplianceStateUpdate", + Related: &corev1.ObjectReference{ + Kind: plGVK.Kind, + Namespace: pl.GetNamespace(), + Name: pl.GetName(), + UID: pl.GetUID(), + APIVersion: plGVK.GroupVersion().String(), + ResourceVersion: pl.GetResourceVersion(), + }, + ReportingController: src.Component, + ReportingInstance: src.Host, + } + + for _, mutatorFunc := range e.Mutators { + var err error + + event, err = mutatorFunc(event) + if err != nil { + return nil, err + } + } + + err := e.Client.Create(ctx, &event) + + return &event, err +} diff --git a/pkg/testutils/courtesies.go b/pkg/testutils/courtesies.go index 797ebc1..6f11fb9 100644 --- a/pkg/testutils/courtesies.go +++ b/pkg/testutils/courtesies.go @@ -3,13 +3,49 @@ package testutils import ( + "regexp" + "time" + + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) +// ObjNN returns a NamespacedName for the given Object. func ObjNN(obj client.Object) types.NamespacedName { return types.NamespacedName{ Namespace: obj.GetNamespace(), Name: obj.GetName(), } } + +// EventFilter filters the given events. Any of the filter parameters can be passed an empty +// value to ignore that field when filtering. The msg parameter will be compiled into a regex if +// possible. The since parameter checks against the event's EventTime - but if the event does not +// specify an EventTime, it will not be filtered out. +func EventFilter(evs []corev1.Event, evType, msg string, since time.Time) []corev1.Event { + msgRegex, err := regexp.Compile(msg) + if err != nil { + msgRegex = regexp.MustCompile(regexp.QuoteMeta(msg)) + } + + ans := make([]corev1.Event, 0) + + for _, ev := range evs { + if evType != "" && ev.Type != evType { + continue + } + + if !msgRegex.MatchString(ev.Message) { + continue + } + + if !ev.EventTime.IsZero() && since.After(ev.EventTime.Time) { + continue + } + + ans = append(ans, ev) + } + + return ans +} diff --git a/pkg/testutils/courtesies_test.go b/pkg/testutils/courtesies_test.go new file mode 100644 index 0000000..49d3dc3 --- /dev/null +++ b/pkg/testutils/courtesies_test.go @@ -0,0 +1,155 @@ +// Copyright Contributors to the Open Cluster Management project + +package testutils + +import ( + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestObjNN(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + inpObj client.Object + wantName string + wantNS string + }{ + "namespaced unstructured": { + inpObj: &unstructured.Unstructured{Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "foo", + "namespace": "world", + }, + }}, + wantName: "foo", + wantNS: "world", + }, + "cluster-scoped unstructured": { + inpObj: &unstructured.Unstructured{Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "bar", + }, + }}, + wantName: "bar", + wantNS: "", + }, + "(namespaced) configmap": { + inpObj: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "my-cm", + Namespace: "kube-one", + }}, + wantName: "my-cm", + wantNS: "kube-one", + }, + "(cluster-scoped) node": { + inpObj: &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "unit-tests-only", + }}, + wantName: "unit-tests-only", + wantNS: "", + }, + } + + for name, tcase := range tests { + got := ObjNN(tcase.inpObj) + + if got.Name != tcase.wantName { + t.Errorf("Wanted name '%v', got '%v' in test '%v'", tcase.wantName, got.Name, name) + } + + if got.Namespace != tcase.wantNS { + t.Errorf("Wanted namespace '%v', got '%v' in test '%v'", tcase.wantNS, got.Namespace, name) + } + } +} + +func TestEventFilter(t *testing.T) { + t.Parallel() + + now := time.Now() + old := now.Add(-time.Minute) + veryOld := now.Add(-time.Hour) + + sampleEvents := []corev1.Event{{ + Message: "hello", + Type: "Normal", + EventTime: metav1.NewMicroTime(veryOld), + }, { + Message: "goodbye", + Type: "Warning", + EventTime: metav1.NewMicroTime(old), + }, { + Message: "carpe diem [", + Type: "Normal", + EventTime: metav1.NewMicroTime(now), + }, { + Message: "what time is it?", + Type: "Warning", + }} + + tests := map[string]struct { + inpType string + inpMsg string + inpSince time.Time + wantIdxs []int + }{ + "#NoFilter": { + inpType: "", + inpMsg: "", + inpSince: time.Time{}, + wantIdxs: []int{0, 1, 2, 3}, + }, + "recent events, plus the one with no time specified": { + inpType: "", + inpMsg: "", + inpSince: now.Add(-5 * time.Minute), + wantIdxs: []int{1, 2, 3}, + }, + "only warnings": { + inpType: "Warning", + inpMsg: "", + inpSince: time.Time{}, + wantIdxs: []int{1, 3}, + }, + "basic regex for a space": { + inpType: "", + inpMsg: ".* .*", + inpSince: time.Time{}, + wantIdxs: []int{2, 3}, + }, + "just a space": { + inpType: "", + inpMsg: " ", + inpSince: time.Time{}, + wantIdxs: []int{2, 3}, + }, + "invalid inescaped regex": { + inpType: "", + inpMsg: "[", + inpSince: time.Time{}, + wantIdxs: []int{2}, + }, + } + + for name, tcase := range tests { + got := EventFilter(sampleEvents, tcase.inpType, tcase.inpMsg, tcase.inpSince) + + if len(got) != len(tcase.wantIdxs) { + t.Fatalf("Expected %v events to be returned, got %v in test %v: got events: %v", + len(tcase.wantIdxs), len(got), name, got) + } + + for i, wantIdx := range tcase.wantIdxs { + if sampleEvents[wantIdx].String() != got[i].String() { + t.Errorf("Mismatch on item #%v in test %v. Expected '%v' got '%v'", + i, name, sampleEvents[wantIdx], got[i]) + } + } + } +} diff --git a/pkg/testutils/toolkit.go b/pkg/testutils/toolkit.go index f8ded15..7d3d5fe 100644 --- a/pkg/testutils/toolkit.go +++ b/pkg/testutils/toolkit.go @@ -5,11 +5,15 @@ package testutils import ( "context" "fmt" + "regexp" + "sort" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" gomegaTypes "github.com/onsi/gomega/types" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -19,9 +23,11 @@ type Toolkit struct { EventuallyTimeout string ConsistentlyPoll string ConsistentallyTimeout string - BackgroundCtx context.Context //nolint:containedctx + BackgroundCtx context.Context //nolint:containedctx // this is for convenience } +// NewToolkit returns a toolkit using the given Client, with some basic defaults. +// This is the preferred way to get a Toolkit instance, to avoid unset fields. func NewToolkit(client client.Client) Toolkit { return Toolkit{ Client: client, @@ -36,11 +42,8 @@ func NewToolkit(client client.Client) Toolkit { // cleanlyCreate creates the given object, and registers a callback to delete the object which // Ginkgo will call at the appropriate time. The error from the `Create` call is returned (so it // can be checked) and the `Delete` callback handles 'NotFound' errors as a success. -func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object) error { - // Save and then re-set the GVK because the API call removes it - savedGVK := obj.GetObjectKind().GroupVersionKind() +func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { createErr := tk.Create(ctx, obj) - obj.GetObjectKind().SetGroupVersionKind(savedGVK) if createErr == nil { ginkgo.DeferCleanup(func() { @@ -59,6 +62,83 @@ func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object) error { return createErr } +// Create uses the toolkit's client to save the object in the Kubernetes cluster. +// The only change in behavior is that it saves and restores the object's type +// information, which might otherwise be stripped during the API call. +func (tk Toolkit) Create( + ctx context.Context, obj client.Object, opts ...client.CreateOption, +) error { + savedGVK := obj.GetObjectKind().GroupVersionKind() + err := tk.Client.Create(ctx, obj, opts...) + obj.GetObjectKind().SetGroupVersionKind(savedGVK) + + return err +} + +// Patch uses the toolkit's client to patch the object in the Kubernetes cluster. +// The only change in behavior is that it saves and restores the object's type +// information, which might otherwise be stripped during the API call. +func (tk Toolkit) Patch( + ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption, +) error { + savedGVK := obj.GetObjectKind().GroupVersionKind() + err := tk.Client.Patch(ctx, obj, patch, opts...) + obj.GetObjectKind().SetGroupVersionKind(savedGVK) + + return err +} + +// Update uses the toolkit's client to update the object in the Kubernetes cluster. +// The only change in behavior is that it saves and restores the object's type +// information, which might otherwise be stripped during the API call. +func (tk Toolkit) Update( + ctx context.Context, obj client.Object, opts ...client.UpdateOption, +) error { + savedGVK := obj.GetObjectKind().GroupVersionKind() + err := tk.Client.Update(ctx, obj, opts...) + obj.GetObjectKind().SetGroupVersionKind(savedGVK) + + return err +} + +// This regular expression is copied from +// https://github.com/open-cluster-management-io/governance-policy-framework-addon/blob/v0.13.0/controllers/statussync/policy_status_sync.go#L220 +var compEventRegex = regexp.MustCompile(`(?i)^policy:\s*(?:([a-z0-9.-]+)\s*\/)?(.+)`) + +// GetComplianceEvents queries the cluster and returns a sorted list of the Kubernetes +// compliance events for the given policy. +func (tk Toolkit) GetComplianceEvents( + ctx context.Context, ns string, parentUID types.UID, templateName string, +) ([]corev1.Event, error) { + list := &corev1.EventList{} + + err := tk.List(ctx, list, client.InNamespace(ns)) + if err != nil { + return nil, err + } + + events := make([]corev1.Event, 0) + + for _, event := range list.Items { + event := event + + if event.InvolvedObject.UID != parentUID { + continue + } + + submatch := compEventRegex.FindStringSubmatch(event.Reason) + if len(submatch) >= 3 && submatch[2] == templateName { + events = append(events, event) + } + } + + sort.SliceStable(events, func(i, j int) bool { + return events[i].Name < events[j].Name + }) + + return events, nil +} + // EC runs assertions on asynchronous behavior, both *E*ventually and *C*onsistently, // using the polling and timeout settings of the toolkit. Its usage should feel familiar // to gomega users, simply skip the `.Should(...)` call and put your matcher as the second @@ -66,11 +146,57 @@ func (tk Toolkit) CleanlyCreate(ctx context.Context, obj client.Object) error { func (tk Toolkit) EC( actualOrCtx interface{}, matcher gomegaTypes.GomegaMatcher, optionalDescription ...interface{}, ) bool { + ginkgo.GinkgoHelper() + + // Add where the failure occurred to the description + eDesc := make([]interface{}, 1) + cDesc := make([]interface{}, 1) + + switch len(optionalDescription) { + case 0: + eDesc[0] = "Failed in Eventually" + cDesc[0] = "Failed in Consistently" + case 1: + if origDescFunc, ok := optionalDescription[0].(func() string); ok { + eDesc[0] = func() string { + return "Failed in Eventually; " + origDescFunc() + } + cDesc[0] = func() string { + return "Failed in Consistently; " + origDescFunc() + } + } else { + eDesc[0] = "Failed in Eventually; " + optionalDescription[0].(string) + cDesc[0] = "Failed in Consistently; " + optionalDescription[0].(string) + } + default: + eDesc[0] = "Failed in Eventually; " + optionalDescription[0].(string) + eDesc = append(eDesc, optionalDescription[1:]...) //nolint: makezero // appending is definitely correct + + cDesc[0] = "Failed in Consistently; " + optionalDescription[0].(string) + cDesc = append(cDesc, optionalDescription[1:]...) //nolint: makezero // appending is definitely correct + } + gomega.Eventually( actualOrCtx, tk.EventuallyTimeout, tk.EventuallyPoll, - ).Should(matcher, optionalDescription...) + ).Should(matcher, eDesc...) return gomega.Consistently( actualOrCtx, tk.ConsistentallyTimeout, tk.ConsistentlyPoll, - ).Should(matcher, optionalDescription...) + ).Should(matcher, cDesc...) +} + +// RegisterDebugMessage returns a pointer to a string which will be logged at the +// end of the test only if the test fails. This is particularly useful for logging +// information only once in an Eventually or Consistently function. +// Note: using a custom description message may be a better practice overall. +func RegisterDebugMessage() *string { + var debugMsg string + + ginkgo.DeferCleanup(func() { + if ginkgo.CurrentSpecReport().Failed() { + ginkgo.GinkgoWriter.Println(debugMsg) + } + }) + + return &debugMsg } diff --git a/test/fakepolicy/api/v1beta1/fakepolicy_types.go b/test/fakepolicy/api/v1beta1/fakepolicy_types.go index 32ea63e..fd5d6ce 100644 --- a/test/fakepolicy/api/v1beta1/fakepolicy_types.go +++ b/test/fakepolicy/api/v1beta1/fakepolicy_types.go @@ -12,11 +12,18 @@ import ( type FakePolicySpec struct { nucleusv1beta1.PolicyCoreSpec `json:",inline"` - // targetConfigMaps defines the ConfigMaps which should be examined by this policy + // TargetConfigMaps defines the ConfigMaps which should be examined by this policy TargetConfigMaps nucleusv1beta1.Target `json:"targetConfigMaps,omitempty"` - // targetUsingReflection defines whether to use reflection to find the ConfigMaps + // TargetUsingReflection defines whether to use reflection to find the ConfigMaps TargetUsingReflection bool `json:"targetUsingReflection,omitempty"` + + // DesiredConfigMapName - if this name is not found, the policy will report a violation + DesiredConfigMapName string `json:"desiredConfigMapName,omitempty"` + + // EventAnnotation - if provided, this value will be annotated on the compliance + // events, under the "policy.open-cluster-management.io/test" key + EventAnnotation string `json:"eventAnnotation,omitempty"` } //+kubebuilder:validation:Optional @@ -41,6 +48,34 @@ type FakePolicy struct { Status FakePolicyStatus `json:"status,omitempty"` } +// ensure FakePolicy implements PolicyLike +var _ nucleusv1beta1.PolicyLike = (*FakePolicy)(nil) + +func (f FakePolicy) ComplianceState() nucleusv1beta1.ComplianceState { + return f.Status.ComplianceState +} + +func (f FakePolicy) ComplianceMessage() string { + idx, compCond := f.Status.GetCondition("Compliant") + if idx == -1 { + return "" + } + + return compCond.Message +} + +func (f FakePolicy) Parent() metav1.OwnerReference { + if len(f.OwnerReferences) == 0 { + return metav1.OwnerReference{} + } + + return f.OwnerReferences[0] +} + +func (f FakePolicy) ParentNamespace() string { + return f.Namespace +} + //+kubebuilder:object:root=true // FakePolicyList contains a list of FakePolicy diff --git a/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml b/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml index 104214a..58afa30 100644 --- a/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml +++ b/test/fakepolicy/config/crd/bases/policy.open-cluster-management.io_fakepolicies.yaml @@ -39,6 +39,15 @@ spec: spec: description: FakePolicySpec defines the desired state of FakePolicy properties: + desiredConfigMapName: + description: DesiredConfigMapName - if this name is not found, the + policy will report a violation + type: string + eventAnnotation: + description: |- + EventAnnotation - if provided, this value will be annotated on the compliance + events, under the "policy.open-cluster-management.io/test" key + type: string namespaceSelector: description: |- NamespaceSelector indicates which namespaces on the cluster this policy @@ -130,7 +139,7 @@ spec: - Critical type: string targetConfigMaps: - description: targetConfigMaps defines the ConfigMaps which should + description: TargetConfigMaps defines the ConfigMaps which should be examined by this policy properties: exclude: @@ -196,7 +205,7 @@ spec: type: object x-kubernetes-map-type: atomic targetUsingReflection: - description: targetUsingReflection defines whether to use reflection + description: TargetUsingReflection defines whether to use reflection to find the ConfigMaps type: boolean type: object diff --git a/test/fakepolicy/config/deploy.yaml b/test/fakepolicy/config/deploy.yaml index 90a6203..79dc8a5 100644 --- a/test/fakepolicy/config/deploy.yaml +++ b/test/fakepolicy/config/deploy.yaml @@ -48,6 +48,15 @@ spec: spec: description: FakePolicySpec defines the desired state of FakePolicy properties: + desiredConfigMapName: + description: DesiredConfigMapName - if this name is not found, the + policy will report a violation + type: string + eventAnnotation: + description: |- + EventAnnotation - if provided, this value will be annotated on the compliance + events, under the "policy.open-cluster-management.io/test" key + type: string namespaceSelector: description: |- NamespaceSelector indicates which namespaces on the cluster this policy @@ -139,7 +148,7 @@ spec: - Critical type: string targetConfigMaps: - description: targetConfigMaps defines the ConfigMaps which should + description: TargetConfigMaps defines the ConfigMaps which should be examined by this policy properties: exclude: @@ -205,7 +214,7 @@ spec: type: object x-kubernetes-map-type: atomic targetUsingReflection: - description: targetUsingReflection defines whether to use reflection + description: TargetUsingReflection defines whether to use reflection to find the ConfigMaps type: boolean type: object diff --git a/test/fakepolicy/controllers/fakepolicy_controller.go b/test/fakepolicy/controllers/fakepolicy_controller.go index ab0999f..7425d08 100644 --- a/test/fakepolicy/controllers/fakepolicy_controller.go +++ b/test/fakepolicy/controllers/fakepolicy_controller.go @@ -19,16 +19,22 @@ import ( nucleusv1alpha1 "open-cluster-management.io/governance-policy-nucleus/api/v1alpha1" nucleusv1beta1 "open-cluster-management.io/governance-policy-nucleus/api/v1beta1" + "open-cluster-management.io/governance-policy-nucleus/pkg/compliance" fakev1beta1 "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/api/v1beta1" ) -// FakePolicyReconciler reconciles a FakePolicy object +// FakePolicyReconciler reconciles a FakePolicy object. +// NOTE: it does not watch anything other than FakePolcies, so it will not react +// to other changes in the cluster - update something on the policy to make it +// re-reconcile. type FakePolicyReconciler struct { client.Client Scheme *runtime.Scheme DynamicClient *dynamic.DynamicClient } +const mutatorAnno string = "policy.open-cluster-management.io/test" + // Usual RBAC for fakepolicy: //+kubebuilder:rbac:groups=policy.open-cluster-management.io,resources=fakepolicies,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=policy.open-cluster-management.io,resources=fakepolicies/status,verbs=get;update;patch @@ -42,11 +48,13 @@ type FakePolicyReconciler struct { func (r *FakePolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) + log.Info("Starting a reconcile") policy := &fakev1beta1.FakePolicy{} if err := r.Get(ctx, req.NamespacedName, policy); err != nil { if errors.IsNotFound(err) { - // Request object not found, probably deleted + log.Info("Request object not found, probably deleted") + return ctrl.Result{}, nil } @@ -55,7 +63,33 @@ func (r *FakePolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - r.doSelections(ctx, policy) + cmFound := r.doSelections(ctx, policy) + + policy.Status.SelectionComplete = true + + complianceCondition := metav1.Condition{ + Type: "Compliant", + Status: metav1.ConditionTrue, + Reason: "Found", + Message: "the desired configmap was found", + } + + policy.Status.ComplianceState = nucleusv1beta1.Compliant + + if !cmFound { + complianceCondition.Status = metav1.ConditionFalse + complianceCondition.Reason = "NotFound" + complianceCondition.Message = "the desired configmap was missing" + policy.Status.ComplianceState = nucleusv1beta1.NonCompliant + } + + changed := policy.Status.UpdateCondition(complianceCondition) + + if !changed { + log.Info("No change; no compliance event to emit") + + return ctrl.Result{}, nil + } if err := r.Status().Update(ctx, policy); err != nil { log.Error(err, "Failed to update status") @@ -63,10 +97,41 @@ func (r *FakePolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - return ctrl.Result{}, nil + emitter := compliance.K8sEmitter{ + Client: r.Client, + } + + if policy.Spec.EventAnnotation != "" { + emitter.Mutators = []func(inpEv corev1.Event) (corev1.Event, error){ + func(inpEv corev1.Event) (corev1.Event, error) { + if inpEv.Annotations == nil { + inpEv.Annotations = make(map[string]string) + } + + inpEv.Annotations[mutatorAnno] = policy.Spec.EventAnnotation + + return inpEv, nil + }, + } + + // it's cheating a bit to put this here but it's helpful to test that + // the events work both when this is and when this is not specified + emitter.Source = corev1.EventSource{ + Component: policy.Spec.EventAnnotation, + Host: policy.Spec.EventAnnotation, + } + } + + ev, err := emitter.EmitEvent(ctx, policy) + + log.Info("Event emitted", "eventName", ev.Name) + + return ctrl.Result{}, err } -func (r *FakePolicyReconciler) doSelections(ctx context.Context, policy *fakev1beta1.FakePolicy) { +func (r *FakePolicyReconciler) doSelections( + ctx context.Context, policy *fakev1beta1.FakePolicy, +) (configMapFound bool) { log := log.FromContext(ctx) nsCond := metav1.Condition{ @@ -150,6 +215,10 @@ func (r *FakePolicyReconciler) doSelections(ctx context.Context, policy *fakev1b clientCMs := make([]string, len(clientMatchedCMs)) for i, cm := range dynamicMatchedCMs { clientCMs[i] = cm.GetNamespace() + "/" + cm.GetName() + + if cm.GetName() == policy.Spec.DesiredConfigMapName { + configMapFound = true + } } slices.Sort(clientCMs) @@ -159,7 +228,7 @@ func (r *FakePolicyReconciler) doSelections(ctx context.Context, policy *fakev1b policy.Status.UpdateCondition(clientCond) - policy.Status.SelectionComplete = true + return configMapFound } type configMapResList struct { diff --git a/test/fakepolicy/main.go b/test/fakepolicy/main.go index c81d02e..ee518cc 100644 --- a/test/fakepolicy/main.go +++ b/test/fakepolicy/main.go @@ -77,7 +77,7 @@ func Run(parentCtx context.Context, cfg *rest.Config) error { cfg, err = ctrl.GetConfig() if err != nil { - setupLog.Error(err, "unable to get kubernetes config") + setupLog.Error(err, "unable to get Kubernetes config") return err } diff --git a/test/fakepolicy/test/compliance/compEvMutator_test.go b/test/fakepolicy/test/compliance/compEvMutator_test.go new file mode 100644 index 0000000..a6f3e49 --- /dev/null +++ b/test/fakepolicy/test/compliance/compEvMutator_test.go @@ -0,0 +1,123 @@ +// Copyright Contributors to the Open Cluster Management project + +package compliance + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "open-cluster-management.io/governance-policy-nucleus/pkg/testutils" + fakev1beta1 "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/api/v1beta1" + . "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/test/utils" +) + +var _ = Describe("Compliance Events with a Mutator", Ordered, func() { + const testNS string = "mutator-comp-test" + const annoKey string = "policy.open-cluster-management.io/test" + + var ( + parent *corev1.ConfigMap + policy *fakev1beta1.FakePolicy + ) + + BeforeAll(func(ctx SpecContext) { + ns := &corev1.Namespace{ + // TypeMeta is not required here; the Client can infer it + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + }, + } + Expect(tk.CleanlyCreate(ctx, ns)).To(Succeed()) + + parent = &corev1.ConfigMap{ + // TypeMeta is useful here for the OwnerReference + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "parent", + Namespace: testNS, + }, + } + Expect(tk.CleanlyCreate(ctx, parent)).To(Succeed()) + + sample := SampleFakePolicy() + policy = &sample + policy.Name += "-mutator-comp-test" + policy.Namespace = testNS + policy.Spec.DesiredConfigMapName = "hello-world" + policy.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: parent.APIVersion, + Kind: parent.Kind, + Name: parent.Name, + UID: parent.UID, + }} + Expect(tk.CleanlyCreate(ctx, policy)).To(Succeed()) + }) + + It("Should start NonCompliant", func(ctx SpecContext) { + tk.EC(func(g Gomega) string { + g.Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + return string(policy.Status.ComplianceState) + }, Equal("NonCompliant")) + }) + + It("Should emit one NonCompliant event without the annotation", func(ctx SpecContext) { + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + evs = testutils.EventFilter(evs, "Warning", "NonCompliant", time.Time{}) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + if _, ok := ev.Annotations[annoKey]; ok { + continue + } + + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) + + It("Should emit with the annotation, when eventAnnotation is set", func(ctx SpecContext) { + Expect(tk.CleanlyCreate(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "hello-world", + Namespace: "default", + Labels: map[string]string{"sample": ""}, + }})).To(Succeed()) + + // Refresh the test's copy of the policy + Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + By("Setting the eventAnnotation field on the policy") + policy.Spec.EventAnnotation = "borogoves" + Expect(tk.Update(ctx, policy)).To(Succeed()) + + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + evs = testutils.EventFilter(evs, "Normal", "^Compliant", time.Time{}) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + if ev.Annotations[annoKey] != "borogoves" { + continue + } + + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) +}) diff --git a/test/fakepolicy/test/compliance/complianceEvent_test.go b/test/fakepolicy/test/compliance/complianceEvent_test.go new file mode 100644 index 0000000..6754aec --- /dev/null +++ b/test/fakepolicy/test/compliance/complianceEvent_test.go @@ -0,0 +1,163 @@ +// Copyright Contributors to the Open Cluster Management project + +package compliance + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "open-cluster-management.io/governance-policy-nucleus/pkg/testutils" + fakev1beta1 "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/api/v1beta1" + . "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/test/utils" +) + +var _ = Describe("Classic Compliance Events", Ordered, func() { + const testNS string = "classic-comp-test" + + var ( + parent *corev1.ConfigMap + policy *fakev1beta1.FakePolicy + ) + + BeforeAll(func(ctx SpecContext) { + ns := &corev1.Namespace{ + // TypeMeta is not required here; the Client can infer it + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + }, + } + Expect(tk.CleanlyCreate(ctx, ns)).To(Succeed()) + + parent = &corev1.ConfigMap{ + // TypeMeta is useful here for the OwnerReference + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "parent", + Namespace: testNS, + }, + } + Expect(tk.CleanlyCreate(ctx, parent)).To(Succeed()) + + sample := SampleFakePolicy() + policy = &sample + policy.Name += "-classic-comp-test" + policy.Namespace = testNS + policy.Spec.DesiredConfigMapName = "hello-world" + policy.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: parent.APIVersion, + Kind: parent.Kind, + Name: parent.Name, + UID: parent.UID, + }} + Expect(tk.CleanlyCreate(ctx, policy)).To(Succeed()) + }) + + It("Should start NonCompliant", func(ctx SpecContext) { + tk.EC(func(g Gomega) string { + g.Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + return string(policy.Status.ComplianceState) + }, Equal("NonCompliant")) + }) + + It("Should emit one NonCompliant event", func(ctx SpecContext) { + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + evs = testutils.EventFilter(evs, "Warning", "NonCompliant", time.Time{}) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) + + It("Should emit one Compliant event after the configmap is created", func(ctx SpecContext) { + Expect(tk.CleanlyCreate(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "hello-world", + Namespace: "default", + Labels: map[string]string{"sample": ""}, + }})).To(Succeed()) + + // Refresh the test's copy of the policy + Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + By("Setting an annotation on the policy to trigger a re-reconcile") + policy.SetAnnotations(map[string]string{ + "classic-comp-test-1": "1", + }) + Expect(tk.Update(ctx, policy)).To(Succeed()) + + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + evs = testutils.EventFilter(evs, "Normal", "^Compliant", time.Time{}) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) + + It("Should emit a NonCompliant event after the configmap is deleted", func(ctx SpecContext) { + By("Ensuring that the configmap is gone") + Eventually(func() string { + cm := corev1.ConfigMap{} + _ = tk.Get(ctx, types.NamespacedName{Name: "hello-world", Namespace: "default"}, &cm) + + return cm.Name + }, "1s", "50ms").Should(BeEmpty()) + + // Refresh the test's copy of the policy + Expect(tk.Get(ctx, testutils.ObjNN(policy), policy)).To(Succeed()) + + By("Patching an annotation on the policy to trigger a re-reconcile") + patch := `[{"op":"replace","path":"/metadata/annotations/classic-comp-test-1","value":"2"}]` + err := tk.Patch(ctx, policy, client.RawPatch(types.JSONPatchType, []byte(patch))) + Expect(err).NotTo(HaveOccurred()) + + // This is just an example usage of this function, not an actual case where it was necessary. + // It's _obvious_ (/s) that the test must use a filter for 2 seconds ago; if it filtered to + // only 1 second ago, then since the Consistently runs for a whole second, it would always + // get an empty list near the end, and fail. + debugMsg := testutils.RegisterDebugMessage() + + tk.EC(func(g Gomega) []string { + evs, err := tk.GetComplianceEvents(ctx, testNS, parent.UID, policy.Name) + g.Expect(err).NotTo(HaveOccurred()) + + *debugMsg = "unfiltered events: " + for _, ev := range evs { + *debugMsg += fmt.Sprintf("(%v: %v), ", ev.Name, ev.Message) + } + + evs = testutils.EventFilter(evs, "Warning", "NonCompliant", + time.Now().Add(-2*time.Second)) + + names := make([]string, 0, len(evs)) + for _, ev := range evs { + names = append(names, ev.Name) + } + + return names + }, HaveLen(1)) + }) +}) diff --git a/test/fakepolicy/test/compliance/suite_test.go b/test/fakepolicy/test/compliance/suite_test.go new file mode 100644 index 0000000..1d153f5 --- /dev/null +++ b/test/fakepolicy/test/compliance/suite_test.go @@ -0,0 +1,84 @@ +// Copyright Contributors to the Open Cluster Management project + +package compliance + +import ( + "context" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "open-cluster-management.io/governance-policy-nucleus/pkg/testutils" + "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy" + fakev1beta1 "open-cluster-management.io/governance-policy-nucleus/test/fakepolicy/api/v1beta1" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + cfg *rest.Config + k8sClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + tk testutils.Toolkit +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Compliance Details Suite") +} + +var _ = BeforeSuite(func() { + format.TruncatedDiff = false + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = fakev1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + tk = testutils.NewToolkit(k8sClient) + tk.BackgroundCtx = ctx + + go func() { + defer GinkgoRecover() + Expect(fakepolicy.Run(ctx, cfg)).To(Succeed()) + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +})