Implement Secret storage backend as gRPC server

Makefile

@@ -17,7 +17,7 @@ all: generate build-all-images test-unit test-lint ## Default: generate all, bui
 # Building #
 ############
 
-APPS = gateway k8s-engine hub-js argo-runner helm-runner cloudsql-runner populator terraform-runner argo-actions gitlab-api-runner
+APPS = gateway k8s-engine hub-js argo-runner helm-runner cloudsql-runner populator terraform-runner argo-actions gitlab-api-runner secret-storage-backend
 TESTS = e2e
 INFRA = json-go-gen graphql-schema-linter jinja2 merger
 
@@ -148,7 +148,7 @@ image-security-scan: build-all-images ## Build the docker images and check for v
 # Generating #
 ##############
 
-generate: gen-go-api-from-ocf-spec gen-k8s-resources gen-graphql-resources gen-go-source-code gen-docs ## Run all generators
+generate: gen-go-api-from-ocf-spec gen-k8s-resources gen-graphql-resources gen-go-source-code gen-docs gen-grpc-resources ## Run all generators
 .PHONY: generate
 
 gen-go-api-from-ocf-spec: ## Generate Go code from OCF JSON Schemas
@@ -163,6 +163,10 @@ gen-graphql-resources: ## Generate code from GraphQL schema
 	./hack/gen-graphql-resources.sh
 .PHONY: gen-graphql-resources
 
+gen-grpc-resources: ## Generate gRPC + ProtoBuf Go code for client and server
+	./hack/gen-grpc-resources.sh
+.PHONY: gen-proto-source-code
+
 gen-go-source-code:
 	go generate -x ./...
 .PHONY: gen-go-source-code

cmd/k8s-engine/README.md

@@ -41,26 +41,26 @@ query {
 
 ## Configuration
 
-| Name                            | Required | Default                          | Description                                                                                                  |
-|---------------------------------|----------|----------------------------------|--------------------------------------------------------------------------------------------------------------|
-| APP_ENABLE_LEADER_ELECTION      | no       | `false`                          | Enable leader election for Kubernetes controller. This ensures only 1 controller is active at any time point |
-| APP_LEADER_ELECTION_NAMESPACE   | no       |                                  | Set the Kubernetes namespace, in which the leader election ConfigMap is created                              |
-| APP_GRAPHQL_ADDR                | no       | `:8080`                          | TCP address the metrics endpoint binds to                                                                    |
-| APP_GRAPHQL_ADDR                | no       | `8081`                           | TCP address the metrics endpoint binds to                                                                    |
-| APP_HEALTHZ_ADDR                | no       | `:8082`                          | TCP address the health probes endpoint binds to                                                              |
-| APP_LOGGER_DEV_MODE             | no       | `false`                          | Enable development mode logging                                                                              |
-| APP_MAX_CONCURRENT_RECONCILES   | no       | `1`                              | Maximum number of concurrent reconcile loops in the controller                                               |
-| APP_MAX_RETRY_FOR_FAILED_ACTION | no       | `15`                             | Maximum number of retries for failed Action reconcile process                                                |
-| APP_GRAPHQLGATEWAY_ENDPOINT     | no       | `http://capact-gateway/graphql`  | Endpoint of the Capact Gateway                                                                              |
-| APP_GRAPHQLGATEWAY_USERNAME     | yes      |                                  | Basic auth username used to authenticate at the Capact Gateway                                              |
-| APP_GRAPHQLGATEWAY_PASSWORD     | yes      |                                  | Basic auth password used to authenticate at the Capact Gateway                                              |
-| APP_BUILTIN_RUNNER_TIMEOUT      | no       | `30m`                            | Set the timeout for the workflow execution of the builtin runners                                            |
-| APP_BUILTIN_RUNNER_IMAGE        | yes      |                                  | Set the image of the builtin runner                                                                          |
-| APP_CLUSTER_POLICY_NAME         | no       | `capact-engine-cluster-policy`   | Name of the ConfigMap with cluster policy                                                                    |
-| APP_CLUSTER_POLICY_NAMESPACE    | no       | `capact-system`                  | Namespace of the ConfigMap with cluster policy                                                               |
-| APP_RENDERER_RENDER_TIMEOUT     | no       | `10m`                            | Maximum time for rendering process. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".          |
-| APP_RENDERER_MAX_DEPTH          | no       | `50`                             | Maximum number of allowed nested workflows to be processed.                                                  |
-| KUBECONFIG                      | no       | `~/.kube/config`                 | Path to kubeconfig file                                                                                      |
+| Name                            | Required | Default                         | Description                                                                                                  |
+|---------------------------------|----------|---------------------------------|--------------------------------------------------------------------------------------------------------------|
+| APP_ENABLE_LEADER_ELECTION      | no       | `false`                         | Enable leader election for Kubernetes controller. This ensures only 1 controller is active at any time point |
+| APP_LEADER_ELECTION_NAMESPACE   | no       |                                 | Set the Kubernetes namespace, in which the leader election ConfigMap is created                              |
+| APP_GRAPHQL_ADDR                | no       | `:8080`                         | TCP address the GraphQL endpoint binds to                                                                    |
+| APP_METRICS_ADDR                | no       | `:8081`                         | TCP address the metrics endpoint binds to                                                                    |
+| APP_HEALTHZ_ADDR                | no       | `:8082`                         | TCP address the health probes endpoint binds to                                                              |
+| APP_LOGGER_DEV_MODE             | no       | `false`                         | Enable development mode logging                                                                              |
+| APP_MAX_CONCURRENT_RECONCILES   | no       | `1`                             | Maximum number of concurrent reconcile loops in the controller                                               |
+| APP_MAX_RETRY_FOR_FAILED_ACTION | no       | `15`                            | Maximum number of retries for failed Action reconcile process                                                |
+| APP_GRAPHQLGATEWAY_ENDPOINT     | no       | `http://capact-gateway/graphql` | Endpoint of the Capact Gateway                                                                               |
+| APP_GRAPHQLGATEWAY_USERNAME     | yes      |                                 | Basic auth username used to authenticate at the Capact Gateway                                               |
+| APP_GRAPHQLGATEWAY_PASSWORD     | yes      |                                 | Basic auth password used to authenticate at the Capact Gateway                                               |
+| APP_BUILTIN_RUNNER_TIMEOUT      | no       | `30m`                           | Set the timeout for the workflow execution of the builtin runners                                            |
+| APP_BUILTIN_RUNNER_IMAGE        | yes      |                                 | Set the image of the builtin runner                                                                          |
+| APP_CLUSTER_POLICY_NAME         | no       | `capact-engine-cluster-policy`  | Name of the ConfigMap with cluster policy                                                                    |
+| APP_CLUSTER_POLICY_NAMESPACE    | no       | `capact-system`                 | Namespace of the ConfigMap with cluster policy                                                               |
+| APP_RENDERER_RENDER_TIMEOUT     | no       | `10m`                           | Maximum time for rendering process. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".          |
+| APP_RENDERER_MAX_DEPTH          | no       | `50`                            | Maximum number of allowed nested workflows to be processed.                                                  |
+| KUBECONFIG                      | no       | `~/.kube/config`                | Path to kubeconfig file                                                                                      |
 
 ## Development
 

cmd/secret-storage-backend/README.md

@@ -0,0 +1,58 @@
+# Secret Storage Backend
+
+## Overview
+
+Secret Storage Backend is a service which handles multiple secret storages for TypeInstances.
+
+This service is implemented according to the [Delegated Storage](../../docs/proposal/20211207-delegated-storage.md) concept.
+
+## Prerequisites
+
+- [Go](https://golang.org)
+
+## Usage
+
+### AWS Secrets Manager provider
+
+By default, the Secret Storage Backend has the `aws_secretsmanager` provider enabled.
+
+1. Create AWS security credentials with `SecretsManagerReadWrite` policy.
+2. Export environment variables:
+
+   ```bash
+    export AWS_ACCESS_KEY_ID="{accessKey}"
+    export AWS_SECRET_ACCESS_KEY="{secretKey}"
+    ```
+3. Run the server:
+
+    ```bash
+    APP_LOGGER_DEV_MODE=true go run ./cmd/secret-storage-backend/main.go
+    ```
+
+The server will listen to gRPC calls according to the [Storage Backend Protocol Buffers schema](../../pkg/hub/api/grpc/storage_backend.proto).
+To perform such calls, you can use e.g. [Insomnia](https://insomnia.rest/) tool.
+
+### Dotenv provider
+
+To run the server with `dotenv` provider enabled, which stores data in files, execute:
+
+   ```bash
+   APP_SUPPORTED_PROVIDERS=dotenv,aws_secretsmanager APP_LOGGER_DEV_MODE=true go run ./cmd/secret-storage-backend/main.go
+   ```
+
+> **NOTE:** You can enable multiple providers, separating them by comma, such as: `APP_SUPPORTED_PROVIDERS=aws_secretsmanager,dotenv`.
+
+## Configuration
+
+| Name                    | Required | Default              | Description                                                                                                                   |
+|-------------------------|----------|----------------------|-------------------------------------------------------------------------------------------------------------------------------|
+| APP_GRPC_ADDR           | no       | `:50051`             | TCP address the gRPC server binds to                                                                                          |
+| APP_HEALTHZ_ADDR        | no       | `:8082`              | TCP address the health probes endpoint binds to                                                                               |
+| APP_SUPPORTED_PROVIDERS | no       | `aws_secretsmanager` | Supported secret providers separated by `,`. A given provider must be passed in additional parameters of gRPC request inputs. |
+| APP_LOGGER_DEV_MODE     | no       | `false`              | Enable development mode logging                                                                                               |
+
+To configure providers, use environmental variables described in the [Providers](https://github.com/SpectralOps/teller#providers) paragraph for Teller's Readme.
+
+## Development
+
+To read more about development, see the [Development guide](https://capact.io/community/development/development-guide).

cmd/secret-storage-backend/main.go

@@ -0,0 +1,103 @@
+package main
+
+import (
+	"log"
+	"net"
+
+	"capact.io/capact/internal/healthz"
+	"capact.io/capact/internal/logger"
+	secret_storage_backend "capact.io/capact/internal/secret-storage-backend"
+	"capact.io/capact/pkg/hub/api/grpc/storage_backend"
+	"github.com/pkg/errors"
+	tellerpkg "github.com/spectralops/teller/pkg"
+	tellercore "github.com/spectralops/teller/pkg/core"
+	"github.com/vrischmann/envconfig"
+	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/grpc"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
+)
+
+// Config holds application related configuration.
+type Config struct {
+	// Address is the TCP address the gRPC server binds to.
+	GRPCAddr string `envconfig:"default=:50051"`
+
+	// HealthzAddr is the TCP address the health probes endpoint binds to.
+	HealthzAddr string `envconfig:"default=:8082"`
+
+	SupportedProviders []string `envconfig:"default=aws_secretsmanager"`
+
+	Logger logger.Config
+}
+
+const appName = "secret-storage-backend"
+
+func main() {
+	var cfg Config
+	err := envconfig.InitWithPrefix(&cfg, "APP")
+	exitOnError(err, "while loading configuration")
+
+	ctx := signals.SetupSignalHandler()
+
+	// setup logger
+	unnamedLogger, err := logger.New(cfg.Logger)
+	exitOnError(err, "while creating zap logger")
+
+	logger := unnamedLogger.Named(appName)
+
+	// setup servers
+	parallelServers := new(errgroup.Group)
+
+	healthzServer := healthz.NewHTTPServer(logger, cfg.HealthzAddr, appName)
+	parallelServers.Go(func() error { return healthzServer.Start(ctx) })
+
+	providers, err := loadProviders(cfg.SupportedProviders)
+	exitOnError(err, "while loading providers")
+
+	handler := secret_storage_backend.NewHandler(logger, providers)
+	exitOnError(err, "while creating new handler")
+
+	listenCfg := net.ListenConfig{}
+	listener, err := listenCfg.Listen(ctx, "tcp", cfg.GRPCAddr)
+	exitOnError(err, "while listening")
+
+	srv := grpc.NewServer()
+	storage_backend.RegisterStorageBackendServer(srv, handler)
+
+	go func() {
+		<-ctx.Done()
+		logger.Info("Stopping server gracefully")
+		srv.GracefulStop()
+	}()
+
+	parallelServers.Go(func() error {
+		logger.Info("Starting TCP server", zap.String("addr", cfg.GRPCAddr))
+		return srv.Serve(listener)
+	})
+
+	err = parallelServers.Wait()
+	exitOnError(err, "while waiting for servers to finish gracefully")
+}
+
+func exitOnError(err error, context string) {
+	if err != nil {
+		log.Fatalf("%s: %v", context, err)
+	}
+}
+
+func loadProviders(providerNames []string) (map[string]tellercore.Provider, error) {
+	builtInProviders := tellerpkg.BuiltinProviders{}
+	providersMap := map[string]tellercore.Provider{}
+
+	for _, providerName := range providerNames {
+		provider, err := builtInProviders.GetProvider(providerName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "while loading provider %q", provider)
+		}
+
+		providersMap[providerName] = provider
+	}
+
+	return providersMap, nil
+}

docs/proposal/20211207-delegated-storage.md

@@ -475,100 +475,95 @@ Capact Local Hub calls proper storage backend service while accessing the TypeIn
 
     <details> <summary>Protocol Buffers definition</summary>
 
-    ```proto
+    ```protobuf
     syntax = "proto3";
-    option go_package = "./";
-    package storagebackend;
-
-    import "google/protobuf/any.proto";
-
-    message TypeInstanceResourceVersion {
-      uint32 resource_version = 1;
-      google.protobuf.Any value = 2;
-    }
-
-    message TypeInstance {
-      string id = 1;
-
-      TypeInstanceResourceVersion resource_version = 2;
-    }
-
+    option go_package = "./storage_backend";
+    package storage_backend;
+
     message OnCreateRequest {
-      TypeInstance typeinstance = 1;
-      google.protobuf.Any additional_parameters = 2;
+      string typeinstance_id = 1;
+      bytes value = 2;
+      bytes additional_parameters = 3;
     }
-
+    
     message OnCreateResponse {
-      google.protobuf.Any additional_parameters = 1;
+      optional bytes additional_parameters = 1;
     }
-
-    message OnUpdateData {
-      TypeInstanceResourceVersion resource_version = 1;
-      google.protobuf.Any additional_parameters = 2;
+    
+    message TypeInstanceResourceVersion {
+      uint32 resource_version = 1;
+      bytes value = 2;
     }
-
+    
     message OnUpdateRequest {
       string typeinstance_id = 1;
-
-      OnUpdateData old_data = 2;
-      OnUpdateData new_data = 3;
+      uint32 new_resource_version = 2;
+      bytes new_value = 3;
+      optional bytes additional_parameters = 4;
     }
-
+    
     message OnUpdateResponse {
-      google.protobuf.Any additional_parameters = 1;
+      optional bytes additional_parameters = 1;
     }
-
+    
     message OnDeleteRequest {
       string typeinstance_id = 1;
-      google.protobuf.Any additional_parameters = 2;
+      bytes additional_parameters = 2;
     }
-
+    
     message OnDeleteResponse {}
-
+    
     message GetValueRequest {
       string typeinstance_id = 1;
-      string resource_version_id = 2;
-      google.protobuf.Any additional_parameters = 3;
+      uint32 resource_version = 2;
+      bytes additional_parameters = 3;
     }
-
+    
     message GetValueResponse {
-      google.protobuf.Any value = 1;
+      optional bytes value = 1;
     }
-
-
+    
+    
     // lock messages
-
+    
     message GetLockedByRequest {
       string typeinstance_id = 1;
-      google.protobuf.Any additional_parameters = 2;
+      bytes additional_parameters = 2;
     }
-
+    
     message GetLockedByResponse {
-      string locked_by = 1;
+      optional string locked_by = 1;
     }
-
-    message OnLockUnlockRequest {
+    
+    message OnLockRequest {
       string typeinstance_id = 1;
-      google.protobuf.Any additional_parameters = 2;
+      bytes additional_parameters = 2;
       string locked_by = 3;
     }
-
-    message OnLockUnlockResponse {}
-
+
+    message OnLockResponse {}
+
+    message OnUnlockRequest {
+      string typeinstance_id = 1;
+      bytes additional_parameters = 2;
+    }
+
+    message OnUnlockResponse {}
+
     // services
-
+    
     service StorageBackend {
       // value
       rpc GetValue(GetValueRequest) returns (GetValueResponse);
       rpc OnCreate(OnCreateRequest) returns (OnCreateResponse);
       rpc OnUpdate(OnUpdateRequest) returns (OnUpdateResponse);
       rpc OnDelete(OnDeleteRequest) returns (OnDeleteResponse);
-
+    
       // lock
       rpc GetLockedBy(GetLockedByRequest) returns (GetLockedByResponse);
-      rpc OnLock(OnLockUnlockRequest) returns (OnLockUnlockResponse);
-      rpc OnUnlock(OnLockUnlockRequest) returns (OnLockUnlockResponse);
-    } 
+      rpc OnLock(OnLockRequest) returns (OnLockRequest);
+      rpc OnUnlock(OnUnlockRequest) returns (OnUnlockRequest);
+    }
     ```
 
     </details>