Skip to content

Commit

Permalink
operator: Add OpenShift CloudCredentials support for AWS STS (#11524)
Browse files Browse the repository at this point in the history
Co-authored-by: Periklis Tsirakidis <[email protected]>
  • Loading branch information
JoaoBraveCoding and periklis authored Jan 26, 2024
1 parent de4f56e commit fe4ba0c
Show file tree
Hide file tree
Showing 46 changed files with 1,549 additions and 119 deletions.
1 change: 1 addition & 0 deletions operator/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
## Main

- [11524](https://github.com/grafana/loki/pull/11524) **JoaoBraveCoding**, **periklis**: Add OpenShift cloud credentials support for AWS STS
- [11513](https://github.com/grafana/loki/pull/11513) **btaani**: Add a custom metric that collects Lokistacks requiring a schema upgrade
- [11718](https://github.com/grafana/loki/pull/11718) **periklis**: Upgrade k8s.io, sigs.k8s.io and openshift deps
- [11671](https://github.com/grafana/loki/pull/11671) **JoaoBraveCoding**: Update mixins to fix structured metadata dashboards
Expand Down
7 changes: 7 additions & 0 deletions operator/apis/config/v1/projectconfig_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ type OpenShiftFeatureGates struct {

// Dashboards enables the loki-mixin dashboards into the OpenShift Console
Dashboards bool `json:"dashboards,omitempty"`

// ManagedAuthEnv enabled when the operator installation is on OpenShift STS clusters.
ManagedAuthEnv bool
}

func (o OpenShiftFeatureGates) ManagedAuthEnabled() bool {
return o.Enabled && o.ManagedAuthEnv
}

// FeatureGates is the supported set of all operator feature gates.
Expand Down
6 changes: 6 additions & 0 deletions operator/apis/loki/v1/lokistack_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -1062,6 +1062,12 @@ const (
ReasonMissingObjectStorageSecret LokiStackConditionReason = "MissingObjectStorageSecret"
// ReasonInvalidObjectStorageSecret when the format of the secret is invalid.
ReasonInvalidObjectStorageSecret LokiStackConditionReason = "InvalidObjectStorageSecret"
// ReasonMissingCredentialsRequest when the required request for managed auth credentials to object
// storage is missing.
ReasonMissingCredentialsRequest LokiStackConditionReason = "MissingCredentialsRequest"
// ReasonMissingManagedAuthSecret when the required secret for managed auth credentials to object
// storage is missing.
ReasonMissingManagedAuthSecret LokiStackConditionReason = "MissingManagedAuthenticationSecret"
// ReasonInvalidObjectStorageSchema when the spec contains an invalid schema(s).
ReasonInvalidObjectStorageSchema LokiStackConditionReason = "InvalidObjectStorageSchema"
// ReasonMissingObjectStorageCAConfigMap when the required configmap to verify object storage
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ metadata:
features.operators.openshift.io/fips-compliant: "false"
features.operators.openshift.io/proxy-aware: "true"
features.operators.openshift.io/tls-profiles: "true"
features.operators.openshift.io/token-auth-aws: "false"
features.operators.openshift.io/token-auth-aws: "true"
features.operators.openshift.io/token-auth-azure: "false"
features.operators.openshift.io/token-auth-gcp: "false"
operators.operatorframework.io/builder: operator-sdk-unknown
Expand Down Expand Up @@ -1463,6 +1463,16 @@ spec:
- patch
- update
- watch
- apiGroups:
- cloudcredential.openshift.io
resources:
- credentialsrequests
verbs:
- create
- delete
- get
- list
- watch
- apiGroups:
- config.openshift.io
resources:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1443,6 +1443,16 @@ spec:
- patch
- update
- watch
- apiGroups:
- cloudcredential.openshift.io
resources:
- credentialsrequests
verbs:
- create
- delete
- get
- list
- watch
- apiGroups:
- config.openshift.io
resources:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ metadata:
features.operators.openshift.io/fips-compliant: "false"
features.operators.openshift.io/proxy-aware: "true"
features.operators.openshift.io/tls-profiles: "true"
features.operators.openshift.io/token-auth-aws: "false"
features.operators.openshift.io/token-auth-aws: "true"
features.operators.openshift.io/token-auth-azure: "false"
features.operators.openshift.io/token-auth-gcp: "false"
olm.skipRange: '>=5.7.0-0 <5.9.0'
Expand Down Expand Up @@ -1448,6 +1448,16 @@ spec:
- patch
- update
- watch
- apiGroups:
- cloudcredential.openshift.io
resources:
- credentialsrequests
verbs:
- create
- delete
- get
- list
- watch
- apiGroups:
- config.openshift.io
resources:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ metadata:
features.operators.openshift.io/fips-compliant: "false"
features.operators.openshift.io/proxy-aware: "true"
features.operators.openshift.io/tls-profiles: "true"
features.operators.openshift.io/token-auth-aws: "false"
features.operators.openshift.io/token-auth-aws: "true"
features.operators.openshift.io/token-auth-azure: "false"
features.operators.openshift.io/token-auth-gcp: "false"
repository: https://github.com/grafana/loki/tree/main/operator
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ metadata:
features.operators.openshift.io/fips-compliant: "false"
features.operators.openshift.io/proxy-aware: "true"
features.operators.openshift.io/tls-profiles: "true"
features.operators.openshift.io/token-auth-aws: "false"
features.operators.openshift.io/token-auth-aws: "true"
features.operators.openshift.io/token-auth-azure: "false"
features.operators.openshift.io/token-auth-gcp: "false"
olm.skipRange: '>=5.7.0-0 <5.9.0'
Expand Down
10 changes: 10 additions & 0 deletions operator/config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,16 @@ rules:
- patch
- update
- watch
- apiGroups:
- cloudcredential.openshift.io
resources:
- credentialsrequests
verbs:
- create
- delete
- get
- list
- watch
- apiGroups:
- config.openshift.io
resources:
Expand Down
71 changes: 71 additions & 0 deletions operator/controllers/loki/credentialsrequests_controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package controllers

import (
"context"

"github.com/go-logr/logr"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
"github.com/grafana/loki/operator/controllers/loki/internal/lokistack"
"github.com/grafana/loki/operator/controllers/loki/internal/management/state"
"github.com/grafana/loki/operator/internal/external/k8s"
"github.com/grafana/loki/operator/internal/handlers"
)

// CredentialsRequestsReconciler reconciles a single CredentialsRequest resource for each LokiStack request.
type CredentialsRequestsReconciler struct {
client.Client
Scheme *runtime.Scheme
Log logr.Logger
}

// Reconcile creates a single CredentialsRequest per LokiStack for the OpenShift cloud-credentials-operator (CCO) to
// provide a managed cloud credentials Secret. On successful creation, the LokiStack resource is annotated
// with `loki.grafana.com/credentials-request-secret-ref` that refers to the secret provided by CCO. If the LokiStack
// resource is not found its accompanying CredentialsRequest resource is deleted.
func (r *CredentialsRequestsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
var stack lokiv1.LokiStack
if err := r.Client.Get(ctx, req.NamespacedName, &stack); err != nil {
if apierrors.IsNotFound(err) {
return ctrl.Result{}, handlers.DeleteCredentialsRequest(ctx, r.Client, req.NamespacedName)
}
return ctrl.Result{}, err
}

managed, err := state.IsManaged(ctx, req, r.Client)
if err != nil {
return ctrl.Result{}, err
}
if !managed {
r.Log.Info("Skipping reconciliation for unmanaged LokiStack resource", "name", req.String())
// Stop requeueing for unmanaged LokiStack custom resources
return ctrl.Result{}, nil
}

secretRef, err := handlers.CreateCredentialsRequest(ctx, r.Client, req.NamespacedName)
if err != nil {
return ctrl.Result{}, err
}

if err := lokistack.AnnotateForCredentialsRequest(ctx, r.Client, req.NamespacedName, secretRef); err != nil {
return ctrl.Result{}, err
}

return ctrl.Result{}, nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *CredentialsRequestsReconciler) SetupWithManager(mgr ctrl.Manager) error {
b := ctrl.NewControllerManagedBy(mgr)
return r.buildController(k8s.NewCtrlBuilder(b))
}

func (r *CredentialsRequestsReconciler) buildController(bld k8s.Builder) error {
return bld.
For(&lokiv1.LokiStack{}).
Complete(r)
}
155 changes: 155 additions & 0 deletions operator/controllers/loki/credentialsrequests_controller_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
package controllers

import (
"context"
"testing"

cloudcredentialsv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1"
"github.com/stretchr/testify/require"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
"github.com/grafana/loki/operator/internal/external/k8s/k8sfakes"
"github.com/grafana/loki/operator/internal/manifests/storage"
)

func TestCredentialsRequestController_RegistersCustomResource_WithDefaultPredicates(t *testing.T) {
b := &k8sfakes.FakeBuilder{}
k := &k8sfakes.FakeClient{}
c := &CredentialsRequestsReconciler{Client: k, Scheme: scheme}

b.ForReturns(b)
b.OwnsReturns(b)

err := c.buildController(b)
require.NoError(t, err)

// Require only one For-Call for the custom resource
require.Equal(t, 1, b.ForCallCount())

// Require For-call with LokiStack resource
obj, _ := b.ForArgsForCall(0)
require.Equal(t, &lokiv1.LokiStack{}, obj)
}

func TestCredentialsRequestController_DeleteCredentialsRequest_WhenLokiStackNotFound(t *testing.T) {
k := &k8sfakes.FakeClient{}
c := &CredentialsRequestsReconciler{Client: k, Scheme: scheme}
r := ctrl.Request{
NamespacedName: types.NamespacedName{
Name: "my-stack",
Namespace: "ns",
},
}

// Set managed auth environment
t.Setenv("ROLEARN", "a-role-arn")

k.GetStub = func(_ context.Context, key types.NamespacedName, _ client.Object, _ ...client.GetOption) error {
if key.Name == r.Name && key.Namespace == r.Namespace {
return apierrors.NewNotFound(schema.GroupResource{}, "lokistack not found")
}
return nil
}

res, err := c.Reconcile(context.Background(), r)
require.NoError(t, err)
require.Equal(t, ctrl.Result{}, res)
require.Equal(t, 1, k.DeleteCallCount())
}

func TestCredentialsRequestController_CreateCredentialsRequest_WhenLokiStackNotAnnotated(t *testing.T) {
k := &k8sfakes.FakeClient{}
c := &CredentialsRequestsReconciler{Client: k, Scheme: scheme}
r := ctrl.Request{
NamespacedName: types.NamespacedName{
Name: "my-stack",
Namespace: "ns",
},
}
s := lokiv1.LokiStack{
ObjectMeta: metav1.ObjectMeta{
Name: "my-stack",
Namespace: "ns",
},
Spec: lokiv1.LokiStackSpec{
ManagementState: lokiv1.ManagementStateManaged,
},
}

// Set managed auth environment
t.Setenv("ROLEARN", "a-role-arn")

k.GetStub = func(_ context.Context, key types.NamespacedName, out client.Object, _ ...client.GetOption) error {
if key.Name == r.Name && key.Namespace == r.Namespace {
k.SetClientObject(out, &s)
return nil
}
return apierrors.NewNotFound(schema.GroupResource{}, "lokistack not found")
}

k.CreateStub = func(_ context.Context, o client.Object, _ ...client.CreateOption) error {
_, isCredReq := o.(*cloudcredentialsv1.CredentialsRequest)
if !isCredReq {
return apierrors.NewBadRequest("something went wrong creating a credentials request")
}
return nil
}

k.UpdateStub = func(_ context.Context, o client.Object, _ ...client.UpdateOption) error {
stack, ok := o.(*lokiv1.LokiStack)
if !ok {
return apierrors.NewBadRequest("something went wrong creating a credentials request")
}

_, hasSecretRef := stack.Annotations[storage.AnnotationCredentialsRequestsSecretRef]
if !hasSecretRef {
return apierrors.NewBadRequest("something went updating the lokistack annotations")
}
return nil
}

res, err := c.Reconcile(context.Background(), r)
require.NoError(t, err)
require.Equal(t, ctrl.Result{}, res)
require.Equal(t, 1, k.CreateCallCount())
require.Equal(t, 1, k.UpdateCallCount())
}

func TestCredentialsRequestController_SkipsUnmanaged(t *testing.T) {
k := &k8sfakes.FakeClient{}
c := &CredentialsRequestsReconciler{Client: k, Scheme: scheme}
r := ctrl.Request{
NamespacedName: types.NamespacedName{
Name: "my-stack",
Namespace: "ns",
},
}

s := lokiv1.LokiStack{
ObjectMeta: metav1.ObjectMeta{
Name: "my-stack",
Namespace: "ns",
},
Spec: lokiv1.LokiStackSpec{
ManagementState: lokiv1.ManagementStateUnmanaged,
},
}

k.GetStub = func(_ context.Context, key types.NamespacedName, out client.Object, _ ...client.GetOption) error {
if key.Name == s.Name && key.Namespace == s.Namespace {
k.SetClientObject(out, &s)
return nil
}
return apierrors.NewNotFound(schema.GroupResource{}, "something not found")
}

res, err := c.Reconcile(context.Background(), r)
require.NoError(t, err)
require.Equal(t, ctrl.Result{}, res)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package lokistack

import (
"context"

"github.com/ViaQ/logerr/v2/kverrors"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/grafana/loki/operator/internal/external/k8s"
"github.com/grafana/loki/operator/internal/manifests/storage"
)

// AnnotateForCredentialsRequest adds the `loki.grafana.com/credentials-request-secret-ref` annotation
// to the named Lokistack. If no LokiStack is found, then skip reconciliation. Or else return an error.
func AnnotateForCredentialsRequest(ctx context.Context, k k8s.Client, key client.ObjectKey, secretRef string) error {
stack, err := getLokiStack(ctx, k, key)
if stack == nil || err != nil {
return err
}

if val, ok := stack.Annotations[storage.AnnotationCredentialsRequestsSecretRef]; ok && val == secretRef {
return nil
}

if err := updateAnnotation(ctx, k, stack, storage.AnnotationCredentialsRequestsSecretRef, secretRef); err != nil {
return kverrors.Wrap(err, "failed to update lokistack `credentialsRequestSecretRef` annotation", "key", key)
}

return nil
}
Loading

0 comments on commit fe4ba0c

Please sign in to comment.