Dataplane Audit logging m2

cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java

@@ -34,6 +34,7 @@
 import com.google.inject.name.Named;
 import com.google.inject.name.Names;
 import com.google.inject.util.Modules;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisherService;
 import io.cdap.cdap.api.feature.FeatureFlagsProvider;
 import io.cdap.cdap.app.deploy.Configurator;
 import io.cdap.cdap.app.deploy.Manager;
@@ -82,6 +83,7 @@
 import io.cdap.cdap.gateway.handlers.VersionHandler;
 import io.cdap.cdap.gateway.handlers.WorkflowHttpHandler;
 import io.cdap.cdap.gateway.handlers.WorkflowStatsSLAHttpHandler;
+import io.cdap.cdap.gateway.handlers.meta.AuditLogPublisherHandler;
 import io.cdap.cdap.gateway.handlers.meta.RemotePrivilegesHandler;
 import io.cdap.cdap.internal.app.deploy.ConfiguratorFactory;
 import io.cdap.cdap.internal.app.deploy.ConfiguratorFactoryProvider;
@@ -152,6 +154,7 @@
 import io.cdap.cdap.scheduler.CoreSchedulerService;
 import io.cdap.cdap.scheduler.Scheduler;
 import io.cdap.cdap.securestore.spi.SecretStore;
+import io.cdap.cdap.security.auth.service.DefaultAuditLogPublisherService;
 import io.cdap.cdap.security.encryption.guice.DataStorageAeadEncryptionModule;
 import io.cdap.cdap.security.impersonation.DefaultOwnerAdmin;
 import io.cdap.cdap.security.impersonation.DefaultUGIProvider;
@@ -434,6 +437,8 @@ protected void configure() {
       bind(EventWriterProvider.class).to(EventWriterExtensionProvider.class);
       bind(MetricsProvider.class).to(SparkProgramStatusMetricsProvider.class);
 
+      bind(AuditLogPublisherService.class).to(DefaultAuditLogPublisherService.class);
+
       Multibinder<HttpHandler> handlerBinder = Multibinder.newSetBinder(
           binder(), HttpHandler.class, Names.named(Constants.AppFabric.HANDLERS_BINDING));
 
@@ -461,6 +466,7 @@ protected void configure() {
       handlerBinder.addBinding().to(AuthorizationHandler.class);
       handlerBinder.addBinding().to(SecureStoreHandler.class);
       handlerBinder.addBinding().to(RemotePrivilegesHandler.class);
+      handlerBinder.addBinding().to(AuditLogPublisherHandler.class);
       handlerBinder.addBinding().to(OperationalStatsHttpHandler.class);
       handlerBinder.addBinding().to(ProfileHttpHandler.class);
       handlerBinder.addBinding().to(ProvisionerHttpHandler.class);

cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AuthorizationModule.java

@@ -27,6 +27,7 @@
 import com.google.inject.assistedinject.FactoryModuleBuilder;
 import io.cdap.cdap.api.Admin;
 import io.cdap.cdap.api.Transactional;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisher;
 import io.cdap.cdap.api.data.DatasetContext;
 import io.cdap.cdap.api.metrics.MetricsCollectionService;
 import io.cdap.cdap.api.security.store.SecureStoreManager;
@@ -43,6 +44,7 @@
 import io.cdap.cdap.security.authorization.DefaultAuthorizationContext;
 import io.cdap.cdap.security.authorization.DelegatingPermissionManager;
 import io.cdap.cdap.security.authorization.DelegatingRoleController;
+import io.cdap.cdap.security.authorization.RemoteAuditLogPublisher;
 import io.cdap.cdap.security.authorization.RoleController;
 import io.cdap.cdap.security.spi.authorization.AccessController;
 import io.cdap.cdap.security.spi.authorization.AuthorizationContext;

cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/InMemoryProgramRunnerModule.java

@@ -24,6 +24,7 @@
 import com.google.inject.multibindings.MapBinder;
 import com.google.inject.name.Named;
 import io.cdap.cdap.api.artifact.ArtifactManager;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisher;
 import io.cdap.cdap.app.runtime.ProgramRunner;
 import io.cdap.cdap.app.runtime.ProgramRunnerFactory;
 import io.cdap.cdap.app.runtime.ProgramRuntimeProvider;
@@ -43,6 +44,8 @@
 import io.cdap.cdap.proto.ProgramType;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+
+import io.cdap.cdap.security.authorization.RemoteAuditLogPublisher;
 import org.apache.twill.api.ServiceAnnouncer;
 import org.apache.twill.common.Cancellable;
 import org.apache.twill.discovery.Discoverable;

cdap-app-fabric/src/main/java/io/cdap/cdap/app/preview/DefaultPreviewRunnerManager.java

@@ -27,6 +27,7 @@
 import com.google.inject.Scopes;
 import com.google.inject.name.Named;
 import com.google.inject.util.Modules;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisherService;
 import io.cdap.cdap.api.security.store.SecureStore;
 import io.cdap.cdap.app.guice.ProgramRunnerRuntimeModule;
 import io.cdap.cdap.common.NotFoundException;
@@ -55,6 +56,7 @@
 import io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule;
 import io.cdap.cdap.proto.id.ApplicationId;
 import io.cdap.cdap.security.auth.context.AuthenticationContextModules;
+import io.cdap.cdap.security.auth.service.DefaultAuditLogPublisherService;
 import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule;
 import io.cdap.cdap.security.guice.preview.PreviewSecureStoreModule;
 import java.net.InetAddress;
@@ -194,6 +196,12 @@ protected void configure() {
             bind(LogAppender.class).to(PreviewTMSLogAppender.class).in(Scopes.SINGLETON);
           }
         },
+        new AbstractModule() {
+          @Override
+          protected void configure() {
+            bind(AuditLogPublisherService.class).to(DefaultAuditLogPublisherService.class).in(Scopes.SINGLETON);
+          }
+        },
         new MessagingServerRuntimeModule().getInMemoryModules(),
         Modules.override(new MetadataReaderWriterModules().getInMemoryModules())
             .with(new AbstractModule() {

cdap-app-fabric/src/main/java/io/cdap/cdap/gateway/handlers/meta/AuditLogPublisherHandler.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright © 2016-2021 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.cdap.gateway.handlers.meta;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisherService;
+import io.cdap.cdap.security.spi.authorization.AuditLogContext;
+import io.cdap.http.HttpResponder;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Type;
+import java.nio.charset.StandardCharsets;
+import java.util.Queue;
+import java.util.concurrent.LinkedBlockingDeque;
+import javax.inject.Inject;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+
+/**
+ * An HTTP Handler that runs inside the master and communicates with
+ * {@link AuditLogPublisherService} to send the audit logs received to publish.
+ *
+ */
+@Path(AbstractRemoteSystemOpsHandler.VERSION + "/execute")
+public class AuditLogPublisherHandler extends AbstractRemoteSystemOpsHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(AuditLogPublisherHandler.class);
+  private final AuditLogPublisherService auditLogPublisherService;
+
+  @Inject
+  AuditLogPublisherHandler(AuditLogPublisherService auditLogPublisherService) {
+    this.auditLogPublisherService = auditLogPublisherService;
+  }
+
+  @POST
+  @Path("/publishbatch")
+  public void publishBatch(FullHttpRequest request, HttpResponder responder) throws Exception {
+    LOG.debug("SANKET in handler publishbatch  for {}", request.content().toString(StandardCharsets.UTF_8));
+    Type queueType = new TypeToken<LinkedBlockingDeque<AuditLogContext>>(){}.getType();
+    Queue<AuditLogContext> deserializedQueue =
+      new Gson().fromJson(request.content().toString(StandardCharsets.UTF_8), queueType);
+    LOG.debug("SANKET in handler publishbatch , q size {}", deserializedQueue.size());
+    auditLogPublisherService.addAuditContexts(deserializedQueue);
+    responder.sendStatus(HttpResponseStatus.OK);
+  }
+}

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/DefaultPreviewManager.java

@@ -31,6 +31,7 @@
 import com.google.inject.name.Names;
 import com.google.inject.util.Modules;
 import io.cdap.cdap.api.annotation.Name;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisher;
 import io.cdap.cdap.api.metrics.MetricsCollectionService;
 import io.cdap.cdap.api.security.AccessException;
 import io.cdap.cdap.app.preview.PreviewConfigModule;
@@ -93,6 +94,7 @@
 import io.cdap.cdap.security.auth.context.AuthenticationContextModules;
 import io.cdap.cdap.security.authorization.AccessControllerInstantiator;
 import io.cdap.cdap.security.authorization.DefaultContextAccessEnforcer;
+import io.cdap.cdap.security.authorization.RemoteAuditLogPublisher;
 import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule;
 import io.cdap.cdap.security.impersonation.DefaultOwnerAdmin;
 import io.cdap.cdap.security.impersonation.DefaultUGIProvider;
@@ -388,6 +390,7 @@ protected void configure() {
             bind(LevelDBTableService.class).toInstance(previewLevelDBTableService);
             bind(RemoteExecutionLogProcessor.class).to(LogAppenderLogProcessor.class)
                 .in(Scopes.SINGLETON);
+            bind(AuditLogPublisher.class).to(RemoteAuditLogPublisher.class).in(Scopes.SINGLETON);
           }
 
           @Provides

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/preview/PreviewRunnerTwillRunnable.java

@@ -30,6 +30,7 @@
 import com.google.inject.Module;
 import com.google.inject.Scopes;
 import com.google.inject.assistedinject.FactoryModuleBuilder;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisher;
 import io.cdap.cdap.api.common.Bytes;
 import io.cdap.cdap.app.deploy.Configurator;
 import io.cdap.cdap.app.preview.PreviewConfigModule;
@@ -70,6 +71,7 @@
 import io.cdap.cdap.proto.id.NamespaceId;
 import io.cdap.cdap.security.auth.context.AuthenticationContextModules;
 import io.cdap.cdap.security.authorization.AuthorizationEnforcementModule;
+import io.cdap.cdap.security.authorization.RemoteAuditLogPublisher;
 import io.cdap.cdap.security.guice.SecureStoreClientModule;
 import io.cdap.cdap.security.impersonation.CurrentUGIProvider;
 import io.cdap.cdap.security.impersonation.UGIProvider;
@@ -267,6 +269,7 @@ protected void configure() {
         bind(ArtifactLocalizerClient.class).in(Scopes.SINGLETON);
         // Preview runner pods should not have any elevated privileges, so use the current UGI.
         bind(UGIProvider.class).to(CurrentUGIProvider.class);
+        bind(AuditLogPublisher.class).to(RemoteAuditLogPublisher.class).in(Scopes.SINGLETON);
       }
     });
 

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java

@@ -25,6 +25,8 @@
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.Module;
+import com.google.inject.Scopes;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisher;
 import io.cdap.cdap.api.metrics.MetricsCollectionService;
 import io.cdap.cdap.common.conf.CConfiguration;
 import io.cdap.cdap.common.conf.Constants;
@@ -48,6 +50,7 @@
 import io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule;
 import io.cdap.cdap.proto.id.NamespaceId;
 import io.cdap.cdap.security.auth.context.AuthenticationContextModules;
+import io.cdap.cdap.security.authorization.RemoteAuditLogPublisher;
 import io.cdap.cdap.security.guice.CoreSecurityModule;
 import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule;
 import java.io.File;
@@ -115,6 +118,7 @@ protected void configure() {
           bind(DiscoveryServiceClient.class)
               .toProvider(
                   new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
+          bind(AuditLogPublisher.class).to(RemoteAuditLogPublisher.class).in(Scopes.SINGLETON);
         }
       });
       modules.add(new RemoteLogAppenderModule());

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerTwillRunnable.java

@@ -25,6 +25,8 @@
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.Module;
+import com.google.inject.Scopes;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisher;
 import io.cdap.cdap.api.feature.FeatureFlagsProvider;
 import io.cdap.cdap.app.guice.DistributedArtifactManagerModule;
 import io.cdap.cdap.common.conf.CConfiguration;
@@ -53,6 +55,7 @@
 import io.cdap.cdap.proto.id.NamespaceId;
 import io.cdap.cdap.security.auth.TokenManager;
 import io.cdap.cdap.security.auth.context.AuthenticationContextModules;
+import io.cdap.cdap.security.authorization.RemoteAuditLogPublisher;
 import io.cdap.cdap.security.guice.CoreSecurityModule;
 import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule;
 import java.io.File;
@@ -133,6 +136,7 @@ protected void configure() {
           bind(DiscoveryServiceClient.class)
               .toProvider(
                   new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
+          bind(AuditLogPublisher.class).to(RemoteAuditLogPublisher.class).in(Scopes.SINGLETON);
         }
       });
       modules.add(new RemoteLogAppenderModule());

cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java

@@ -31,6 +31,7 @@
 import com.google.inject.multibindings.OptionalBinder;
 import com.google.inject.util.Modules;
 import io.cdap.cdap.api.artifact.ArtifactManager;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisher;
 import io.cdap.cdap.api.metrics.MetricsCollectionService;
 import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule;
 import io.cdap.cdap.app.guice.AuthorizationModule;
@@ -84,6 +85,7 @@
 import io.cdap.cdap.security.auth.KeyManager;
 import io.cdap.cdap.security.auth.context.AuthenticationContextModules;
 import io.cdap.cdap.security.authorization.AuthorizationEnforcementModule;
+import io.cdap.cdap.security.authorization.RemoteAuditLogPublisher;
 import io.cdap.cdap.security.guice.CoreSecurityModule;
 import io.cdap.cdap.security.guice.FileBasedCoreSecurityModule;
 import io.cdap.cdap.security.guice.SecureStoreClientModule;
@@ -189,6 +191,7 @@ protected void configure() {
           protected void configure() {
             bind(MetadataPublisher.class).to(MessagingMetadataPublisher.class);
             bind(MetadataServiceClient.class).to(DefaultMetadataServiceClient.class);
+            bind(AuditLogPublisher.class).to(RemoteAuditLogPublisher.class).in(Scopes.SINGLETON);
           }
         }
     ));

cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/AbstractServiceMain.java

@@ -26,6 +26,9 @@
 import com.google.inject.Injector;
 import com.google.inject.Key;
 import com.google.inject.Module;
+import com.google.inject.Scopes;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisher;
+import io.cdap.cdap.api.auditlogging.AuditLogPublisherService;
 import io.cdap.cdap.api.metrics.MetricsCollectionService;
 import io.cdap.cdap.app.preview.PreviewConfigModule;
 import io.cdap.cdap.common.app.MainClassLoader;
@@ -55,6 +58,7 @@
 import io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule;
 import io.cdap.cdap.security.auth.TokenManager;
 import io.cdap.cdap.security.auth.context.AuthenticationContextModules;
+import io.cdap.cdap.security.authorization.RemoteAuditLogPublisher;
 import io.cdap.cdap.security.guice.CoreSecurityModule;
 import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule;
 import io.cdap.cdap.security.impersonation.SecurityUtil;
@@ -179,7 +183,12 @@ protected void configure() {
       }
     });
     modules.add(getLogAppenderModule());
-
+    modules.add(new AbstractModule() {
+      @Override
+      protected void configure() {
+        bind(AuditLogPublisher.class).to(RemoteAuditLogPublisher.class).in(Scopes.SINGLETON);
+      }
+    });
     CoreSecurityModule coreSecurityModule = CoreSecurityRuntimeModule.getDistributedModule(cConf);
     modules.add(coreSecurityModule);
     if (coreSecurityModule.requiresZKClient()) {
@@ -202,6 +211,7 @@ protected void configure() {
 
     // Add Services
     services.add(injector.getInstance(MetricsCollectionService.class));
+
     if (SecurityUtil.isInternalAuthEnabled(cConf)) {
       services.add(injector.getInstance(TokenManager.class));
     }

cdap-common/src/main/java/io/cdap/cdap/api/auditlogging/AuditLogPublisher.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright © 2024 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.cdap.api.auditlogging;
+
+
+import com.google.common.util.concurrent.Service;
+import io.cdap.cdap.security.spi.authorization.AuditLogContext;
+
+import java.util.Queue;
+
+
+/**
+ * Service to publish audit log to central audit log service in app fabric
+ */
+public interface AuditLogPublisher {
+
+  /**
+   * pushes the log entry to respective external service
+   */
+  void publish(Queue<AuditLogContext> auditLogContexts);
+}