From 7882f8cf0f385e9bb8c86f5b627e9e284d78d1cc Mon Sep 17 00:00:00 2001 From: Dennis Li <23002167+dli357@users.noreply.github.com> Date: Tue, 11 Jul 2023 20:35:13 -0700 Subject: [PATCH] [CDAP-20629] Create CredentialProvider API and SPI, implement provision handler methods, and add profile manager validation [CDAP-20629] Fix checkstyle warnings and reformat files [CDAP-20629] Refactor various ZK to lowercase Zk to comply with checkstyle [CDAP-20629] Refactor JVMResource, JMXMetricsCollector, and JMXMetricsCollectorFactory to lowercase Jvm and Jmx [CDAP-20629] Suppress warnings for SSL and HBase due to refactoring risk [CDAP-20629] Address comments [CDAP-20629] Moved CredentialProvider API classes to cdap-proto --- .../DistributedWorkflowProgramRunnerTest.java | 173 ++++++------- .../guice/AppFabricServiceRuntimeModule.java | 7 +- .../DistributedProgramContainerModule.java | 21 +- .../io/cdap/cdap/app/guice/TwillModule.java | 5 +- .../preview/PreviewRunnerTwillRunnable.java | 18 +- .../app/worker/TaskWorkerTwillRunnable.java | 12 +- .../ArtifactLocalizerTwillRunnable.java | 21 +- .../system/SystemWorkerTwillRunnable.java | 18 +- .../credential/CredentialIdentityManager.java | 15 +- .../credential/CredentialProfileManager.java | 38 ++- .../CredentialProviderExtensionLoader.java | 118 +++++++++ .../credential/CredentialProviderLoader.java | 33 +++ .../credential/CredentialProviderService.java | 27 +++ .../DefaultCredentialProviderContext.java | 53 ++++ .../DefaultCredentialProviderService.java | 136 +++++++++++ .../guice/MasterCredentialProviderModule.java | 39 +++ .../CredentialProviderHttpHandler.java | 56 ++++- ...CredentialProviderHttpHandlerInternal.java | 42 +++- .../InstantEpochSecondsTypeAdapter.java | 39 +++ .../store/CredentialIdentityStore.java | 21 +- .../store/CredentialProfileStore.java | 11 +- .../environment/k8s/AbstractServiceMain.java | 21 +- .../CredentialIdentityManagerTest.java | 59 +++-- .../credential/CredentialManagerTestBase.java | 79 ------ .../CredentialProfileManagerTest.java | 69 ++++-- .../CredentialProviderTestBase.java | 147 ++++++++++++ .../DefaultCredentialProviderServiceTest.java | 113 +++++++++ .../InstantEpochSecondsTypeAdapterTest.java | 47 ++++ .../io/cdap/cdap/common/conf/Constants.java | 168 ++++++------- .../cdap/common/guice/KafkaClientModule.java | 27 ++- ...KClientModule.java => ZkClientModule.java} | 11 +- ...veryModule.java => ZkDiscoveryModule.java} | 12 +- .../src/main/resources/cdap-default.xml | 10 + .../common/guice/KafkaClientModuleTest.java | 57 +++-- ...leTest.java => ZkDiscoveryModuleTest.java} | 51 ++-- .../coordination/ResourceCoordinatorTest.java | 46 ++-- .../table/hbase/HBaseMetricsTableTest.java | 59 ++--- .../TransactionServiceClientTest.java | 71 +++--- .../distributed/TransactionServiceTest.java | 124 +++++----- .../DatasetOpExecutorServiceTest.java | 86 ++++--- .../java/io/cdap/cdap/kafka/KafkaTester.java | 227 ++++++++++-------- .../cdap/cdap/gateway/router/RouterMain.java | 15 +- .../cdap/cdap/kafka/run/KafkaServerMain.java | 4 +- .../guice/SystemMetricsExporterModule.java | 12 +- .../DatasetOpExecutorServerTwillRunnable.java | 10 +- .../runtime/main/LogSaverTwillRunnable.java | 10 +- .../data/runtime/main/MasterServiceMain.java | 45 ++-- .../main/MessagingServiceTwillRunnable.java | 17 +- .../main/MetricsProcessorTwillRunnable.java | 10 +- .../runtime/main/MetricsTwillRunnable.java | 10 +- .../main/TransactionServiceTwillRunnable.java | 10 +- .../cdap/data/tools/HBaseTableExporter.java | 42 +++- .../cdap/data/tools/JobQueueDebugger.java | 35 ++- .../io/cdap/cdap/data/tools/UpgradeTool.java | 38 +-- .../environment/k8s/AppFabricServiceMain.java | 5 +- .../master/environment/k8s/StorageMain.java | 6 +- .../k8s/SystemMetricsExporterServiceMain.java | 8 +- .../master/startup/MasterStartupTool.java | 25 +- ...ollector.java => JmxMetricsCollector.java} | 43 ++-- ...y.java => JmxMetricsCollectorFactory.java} | 8 +- .../data/runtime/main/TwillRunnableTest.java | 17 +- .../k8s/AppFabricServiceMainTest.java | 35 +-- .../k8s/AuthenticationServiceMainTest.java | 15 +- .../k8s/MasterServiceMainTestBase.java | 105 ++++---- .../k8s/PreviewServiceMainTest.java | 155 +++++++----- ...erServiceMainWithSecurityDisabledTest.java | 7 +- ...terServiceMainWithSecurityEnabledTest.java | 7 +- .../k8s/SupportBundleServiceMainTest.java | 6 +- .../SystemMetricsExporterServiceMainTest.java | 13 +- ...Test.java => JmxMetricsCollectorTest.java} | 27 ++- .../proto/credential/CredentialIdentity.java | 25 +- .../proto/credential/CredentialProfile.java | 4 +- .../proto/credential/CredentialProvider.java | 49 ++++ .../CredentialProvisioningException.java | 32 +++ .../IdentityValidationException.java | 32 +++ .../proto/credential/NotFoundException.java | 32 +++ .../credential/ProvisionedCredential.java | 57 +++++ .../spi/credential/CredentialProvider.java | 65 +++++ .../credential/CredentialProviderContext.java | 37 +++ .../ProfileValidationException.java | 32 +++ .../runtime/AuthenticationServerMain.java | 10 +- .../auth/DistributedKeyManagerTest.java | 52 ++-- .../zookeeper/SharedResourceCacheTest.java | 23 +- .../java/io/cdap/cdap/StandaloneMain.java | 8 +- .../LeaderElectionMessagingServiceTest.java | 118 ++++----- .../LocalLogAppenderResilientTest.java | 85 ++++--- .../appender/kafka/TestKafkaLogging.java | 52 ++-- .../DistributedLogFrameworkTest.java | 96 ++++---- 88 files changed, 2665 insertions(+), 1271 deletions(-) create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderExtensionLoader.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderLoader.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderService.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderContext.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderService.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/guice/MasterCredentialProviderModule.java create mode 100644 cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/InstantEpochSecondsTypeAdapter.java delete mode 100644 cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialManagerTestBase.java create mode 100644 cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java create mode 100644 cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderServiceTest.java create mode 100644 cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/handler/InstantEpochSecondsTypeAdapterTest.java rename cdap-common/src/main/java/io/cdap/cdap/common/guice/{ZKClientModule.java => ZkClientModule.java} (91%) rename cdap-common/src/main/java/io/cdap/cdap/common/guice/{ZKDiscoveryModule.java => ZkDiscoveryModule.java} (92%) rename cdap-common/src/test/java/io/cdap/cdap/common/guice/{ZKDiscoveryModuleTest.java => ZkDiscoveryModuleTest.java} (77%) rename cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/{JMXMetricsCollector.java => JmxMetricsCollector.java} (82%) rename cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/{JMXMetricsCollectorFactory.java => JmxMetricsCollectorFactory.java} (75%) rename cdap-master/src/test/java/io/cdap/cdap/metrics/jmx/{JMXMetricsCollectorTest.java => JmxMetricsCollectorTest.java} (81%) create mode 100644 cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProvider.java create mode 100644 cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProvisioningException.java create mode 100644 cdap-proto/src/main/java/io/cdap/cdap/proto/credential/IdentityValidationException.java create mode 100644 cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NotFoundException.java create mode 100644 cdap-proto/src/main/java/io/cdap/cdap/proto/credential/ProvisionedCredential.java create mode 100644 cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/CredentialProvider.java create mode 100644 cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/CredentialProviderContext.java create mode 100644 cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/ProfileValidationException.java diff --git a/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java b/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java index 3f168698321b..9d1e4724c985 100644 --- a/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java +++ b/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2018-2022 Cask Data, Inc. + * Copyright © 2018-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -41,8 +41,8 @@ import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.LocalLocationModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.id.Id; import io.cdap.cdap.common.test.AppJarHelper; import io.cdap.cdap.data.runtime.DataFabricModules; @@ -82,8 +82,8 @@ import org.junit.rules.TemporaryFolder; /** - * Unit tests for the {@link DistributedWorkflowProgramRunner}. - * This test class uses {@link DistributedWorkflowTestApp} for testing various aspect of the program runner. + * Unit tests for the {@link DistributedWorkflowProgramRunner}. This test class uses {@link + * DistributedWorkflowTestApp} for testing various aspect of the program runner. */ public class DistributedWorkflowProgramRunnerTest { @@ -95,7 +95,7 @@ public class DistributedWorkflowProgramRunnerTest { @BeforeClass public static void init() throws IOException { - cConf = createCConf(); + cConf = createCconf(); programRunnerFactory = createProgramRunnerFactory(cConf); } @@ -104,7 +104,7 @@ public void testDefaultResources() throws IOException { // By default the workflow driver would have 768m if none of the children is setting it to higher // (default for programs are 512m) testDriverResources(DistributedWorkflowTestApp.SequentialWorkflow.class.getSimpleName(), - Collections.emptyMap(), new Resources(768)); + Collections.emptyMap(), new Resources(768)); } @Test @@ -112,19 +112,19 @@ public void testExplicitResources() throws IOException { // Explicitly set the resources for the workflow, it should always get honored. // The one prefixed with "task.workflow." should override the one without. testDriverResources(DistributedWorkflowTestApp.ComplexWorkflow.class.getSimpleName(), - ImmutableMap.of( - SystemArguments.MEMORY_KEY, "4096", - "task.workflow." + SystemArguments.MEMORY_KEY, "1024" - ), - new Resources(1024)); + ImmutableMap.of( + SystemArguments.MEMORY_KEY, "4096", + "task.workflow." + SystemArguments.MEMORY_KEY, "1024" + ), + new Resources(1024)); } @Test public void testInferredResources() throws IOException { // Inferred from the largest memory usage from children testDriverResources(DistributedWorkflowTestApp.SequentialWorkflow.class.getSimpleName(), - Collections.singletonMap("mapreduce.mr1." + SystemArguments.MEMORY_KEY, "2048"), - new Resources(2048)); + Collections.singletonMap("mapreduce.mr1." + SystemArguments.MEMORY_KEY, "2048"), + new Resources(2048)); } @Test @@ -132,27 +132,27 @@ public void testForkJoinInferredResources() throws IOException { // The complex workflow has 4 parallel executions at max, hence the memory setting should be summation of them // For vcores, it should pick the max testDriverResources(DistributedWorkflowTestApp.ComplexWorkflow.class.getSimpleName(), - ImmutableMap.of( - "spark.s2." + SystemArguments.MEMORY_KEY, "1024", - "mapreduce.mr3." + SystemArguments.CORES_KEY, "3" - ), - new Resources(512 + 512 + 1024 + 512, 3)); + ImmutableMap.of( + "spark.s2." + SystemArguments.MEMORY_KEY, "1024", + "mapreduce.mr3." + SystemArguments.CORES_KEY, "3" + ), + new Resources(512 + 512 + 1024 + 512, 3)); } @Test public void testConditionInferredResources() throws IOException { // Set one of the branch to have memory > fork memory testDriverResources(DistributedWorkflowTestApp.ComplexWorkflow.class.getSimpleName(), - Collections.singletonMap("spark.s4." + SystemArguments.MEMORY_KEY, "4096"), - new Resources(4096)); + Collections.singletonMap("spark.s4." + SystemArguments.MEMORY_KEY, "4096"), + new Resources(4096)); } @Test public void testInferredWithMaxResources() throws IOException { // Set to use large memory for the first node in the complex workflow to have it larger than the sum of all forks testDriverResources(DistributedWorkflowTestApp.ComplexWorkflow.class.getSimpleName(), - Collections.singletonMap("mapreduce.mr1." + SystemArguments.MEMORY_KEY, "4096"), - new Resources(4096)); + Collections.singletonMap("mapreduce.mr1." + SystemArguments.MEMORY_KEY, "4096"), + new Resources(4096)); } @Test @@ -160,24 +160,24 @@ public void testInferredScopedResources() throws IOException { // Inferred from the largest memory usage from children. // Make sure the children arguments have scope resolved correctly. testDriverResources(DistributedWorkflowTestApp.SequentialWorkflow.class.getSimpleName(), - ImmutableMap.of( - "mapreduce.mr1.task.mapper." + SystemArguments.MEMORY_KEY, "2048", - "spark.s1.task.client." + SystemArguments.MEMORY_KEY, "1024", - "spark.s1.task.driver." + SystemArguments.MEMORY_KEY, "4096" - ), - // Should pick the spark client memory - new Resources(1024)); + ImmutableMap.of( + "mapreduce.mr1.task.mapper." + SystemArguments.MEMORY_KEY, "2048", + "spark.s1.task.client." + SystemArguments.MEMORY_KEY, "1024", + "spark.s1.task.driver." + SystemArguments.MEMORY_KEY, "4096" + ), + // Should pick the spark client memory + new Resources(1024)); } @Test public void testOverrideInferredResources() throws IOException { // Explicitly setting memory always override what's inferred from children testDriverResources(DistributedWorkflowTestApp.SequentialWorkflow.class.getSimpleName(), - ImmutableMap.of( - "task.workflow." + SystemArguments.MEMORY_KEY, "512", - "mapreduce.mr1." + SystemArguments.MEMORY_KEY, "2048", - "spark.s1." + SystemArguments.MEMORY_KEY, "1024" - ), new Resources(512)); + ImmutableMap.of( + "task.workflow." + SystemArguments.MEMORY_KEY, "512", + "mapreduce.mr1." + SystemArguments.MEMORY_KEY, "2048", + "spark.s1." + SystemArguments.MEMORY_KEY, "1024" + ), new Resources(512)); } @Test @@ -185,10 +185,10 @@ public void testReservedMemoryOverride() throws IOException { // Sets the reserved memory override for the workflow String workflowName = DistributedWorkflowTestApp.SequentialWorkflow.class.getSimpleName(); ProgramLaunchConfig launchConfig = setupWorkflowRuntime(workflowName, - ImmutableMap.of( - SystemArguments.RESERVED_MEMORY_KEY_OVERRIDE, "400", - SystemArguments.MEMORY_KEY, "800" - )); + ImmutableMap.of( + SystemArguments.RESERVED_MEMORY_KEY_OVERRIDE, "400", + SystemArguments.MEMORY_KEY, "800" + )); RunnableDefinition runnableDefinition = launchConfig.getRunnables().get(workflowName); Assert.assertNotNull(runnableDefinition); @@ -200,14 +200,16 @@ public void testReservedMemoryOverride() throws IOException { /** - * Helper method to help testing workflow driver resources settings based on varying user arguments + * Helper method to help testing workflow driver resources settings based on varying user + * arguments * - * @param workflowName name of the workflow as defined in the {@link DistributedWorkflowTestApp}. - * @param runtimeArgs the runtime arguments for the workflow program + * @param workflowName name of the workflow as defined in the {@link + * DistributedWorkflowTestApp}. + * @param runtimeArgs the runtime arguments for the workflow program * @param expectedDriverResources the expected {@link Resources} setting for the workflow driver */ private void testDriverResources(String workflowName, Map runtimeArgs, - Resources expectedDriverResources) throws IOException { + Resources expectedDriverResources) throws IOException { ProgramLaunchConfig launchConfig = setupWorkflowRuntime(workflowName, runtimeArgs); @@ -224,7 +226,7 @@ private void testDriverResources(String workflowName, Map runtim * Setup the {@link ProgramLaunchConfig} for the given workflow. */ private ProgramLaunchConfig setupWorkflowRuntime(String workflowName, - Map runtimeArgs) throws IOException { + Map runtimeArgs) throws IOException { // Create the distributed workflow program runner ProgramRunner programRunner = programRunnerFactory.create(ProgramType.WORKFLOW); Assert.assertTrue(programRunner instanceof DistributedWorkflowProgramRunner); @@ -234,11 +236,11 @@ private ProgramLaunchConfig setupWorkflowRuntime(String workflowName, Program workflowProgram = createWorkflowProgram(cConf, programRunner, workflowName); ProgramLaunchConfig launchConfig = new ProgramLaunchConfig(); ProgramOptions programOpts = new SimpleProgramOptions(workflowProgram.getId(), - new BasicArguments(), new BasicArguments(runtimeArgs)); + new BasicArguments(), new BasicArguments(runtimeArgs)); // Setup the launching config workflowRunner.setupLaunchConfig(launchConfig, workflowProgram, programOpts, - cConf, new Configuration(), TEMP_FOLDER.newFolder()); + cConf, new Configuration(), TEMP_FOLDER.newFolder()); return launchConfig; } @@ -246,62 +248,63 @@ private ProgramLaunchConfig setupWorkflowRuntime(String workflowName, * Creates a workflow {@link Program}. */ private Program createWorkflowProgram(CConfiguration cConf, ProgramRunner programRunner, - String workflowName) throws IOException { - Location appJarLocation = AppJarHelper.createDeploymentJar(new LocalLocationFactory(TEMP_FOLDER.newFolder()), - DistributedWorkflowTestApp.class); + String workflowName) throws IOException { + Location appJarLocation = AppJarHelper + .createDeploymentJar(new LocalLocationFactory(TEMP_FOLDER.newFolder()), + DistributedWorkflowTestApp.class); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("test", "1.0.0"); DistributedWorkflowTestApp app = new DistributedWorkflowTestApp(); DefaultAppConfigurer configurer = new DefaultAppConfigurer(Id.Namespace.DEFAULT, - Id.Artifact.fromEntityId(artifactId), app); + Id.Artifact.fromEntityId(artifactId), app); app.configure(configurer, new DefaultApplicationContext<>()); ApplicationSpecification appSpec = configurer.createSpecification(null); ProgramId programId = NamespaceId.DEFAULT - .app(appSpec.getName()) - .program(ProgramType.WORKFLOW, workflowName); + .app(appSpec.getName()) + .program(ProgramType.WORKFLOW, workflowName); return Programs.create(cConf, programRunner, new ProgramDescriptor(programId, appSpec), - appJarLocation, TEMP_FOLDER.newFolder()); + appJarLocation, TEMP_FOLDER.newFolder()); } private static ProgramRunnerFactory createProgramRunnerFactory(CConfiguration cConf) { Injector injector = Guice.createInjector( - new ConfigModule(cConf), - RemoteAuthenticatorModules.getNoOpModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), - new LocalLogAppenderModule(), - new LocalLocationModule(), - new IOModule(), - new KafkaClientModule(), - new DataSetServiceModules().getDistributedModules(), - new DataFabricModules("cdap.master").getDistributedModules(), - new DataSetsModules().getDistributedModules(), - new MetricsClientRuntimeModule().getDistributedModules(), - new MetricsStoreModule(), - new MessagingClientModule(), - new AuditModule(), - CoreSecurityRuntimeModule.getDistributedModule(cConf), - new AuthenticationContextModules().getNoOpModule(), - new AuthorizationModule(), - new AuthorizationEnforcementModule().getMasterModule(), - new TwillModule(), - new AppFabricServiceRuntimeModule(cConf).getDistributedModules(), - new ProgramRunnerRuntimeModule().getDistributedModules(), - new SecureStoreServerModule(), - new OperationalStatsModule(), - new AbstractModule() { - @Override - protected void configure() { - // TODO (CDAP-14677): find a better way to inject metadata publisher - bind(MetadataServiceClient.class).to(DefaultMetadataServiceClient.class); - } - }); + new ConfigModule(cConf), + RemoteAuthenticatorModules.getNoOpModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), + new LocalLogAppenderModule(), + new LocalLocationModule(), + new IOModule(), + new KafkaClientModule(), + new DataSetServiceModules().getDistributedModules(), + new DataFabricModules("cdap.master").getDistributedModules(), + new DataSetsModules().getDistributedModules(), + new MetricsClientRuntimeModule().getDistributedModules(), + new MetricsStoreModule(), + new MessagingClientModule(), + new AuditModule(), + CoreSecurityRuntimeModule.getDistributedModule(cConf), + new AuthenticationContextModules().getNoOpModule(), + new AuthorizationModule(), + new AuthorizationEnforcementModule().getMasterModule(), + new TwillModule(), + new AppFabricServiceRuntimeModule(cConf).getDistributedModules(), + new ProgramRunnerRuntimeModule().getDistributedModules(), + new SecureStoreServerModule(), + new OperationalStatsModule(), + new AbstractModule() { + @Override + protected void configure() { + // TODO (CDAP-14677): find a better way to inject metadata publisher + bind(MetadataServiceClient.class).to(DefaultMetadataServiceClient.class); + } + }); return injector.getInstance(ProgramRunnerFactory.class); } - private static CConfiguration createCConf() throws IOException { + private static CConfiguration createCconf() throws IOException { CConfiguration cConf = CConfiguration.create(); cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath()); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java index 6442af24e92e..20eb3f8ed2ba 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java @@ -116,6 +116,7 @@ import io.cdap.cdap.internal.app.store.DefaultStore; import io.cdap.cdap.internal.bootstrap.guice.BootstrapModules; import io.cdap.cdap.internal.capability.CapabilityModule; +import io.cdap.cdap.internal.credential.guice.MasterCredentialProviderModule; import io.cdap.cdap.internal.credential.handler.CredentialProviderHttpHandler; import io.cdap.cdap.internal.credential.handler.CredentialProviderHttpHandlerInternal; import io.cdap.cdap.internal.events.EventPublishManager; @@ -185,6 +186,7 @@ public Module getInMemoryModules() { new ConfigStoreModule(), new SourceControlModule(), new EntityVerifierModule(), + new MasterCredentialProviderModule(), BootstrapModules.getInMemoryModule(), new AbstractModule() { @Override @@ -226,6 +228,7 @@ public Module getStandaloneModules() { new SourceControlModule(), new EntityVerifierModule(), new ProvisionerModule(), + new MasterCredentialProviderModule(), BootstrapModules.getFileBasedModule(), new AbstractModule() { @Override @@ -279,6 +282,7 @@ public Module getDistributedModules() { new SourceControlModule(), new EntityVerifierModule(), new ProvisionerModule(), + new MasterCredentialProviderModule(), BootstrapModules.getFileBasedModule(), new AbstractModule() { @Override @@ -525,7 +529,8 @@ private org.quartz.Scheduler getScheduler(JobStore store, /** * A Guice provider for the {@link UGIProvider} class based on the CDAP configuration. * - *

When Kerberos is enabled, it provides {@link DefaultUGIProvider} instance. Otherwise, an {@link + *

When Kerberos is enabled, it provides {@link DefaultUGIProvider} instance. Otherwise, an + * {@link * UnsupportedUGIProvider} will be used. */ private static final class UgiProviderProvider implements Provider { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/DistributedProgramContainerModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/DistributedProgramContainerModule.java index a2117c1eb71d..a5e818617cb9 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/DistributedProgramContainerModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/DistributedProgramContainerModule.java @@ -1,5 +1,5 @@ /* - * Copyright © 2018-2022 Cask Data, Inc. + * Copyright © 2018-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -33,8 +33,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.SupplierProviderBridge; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.internal.remote.InternalAuthenticator; import io.cdap.cdap.common.internal.remote.RemoteClientFactory; import io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule; @@ -111,6 +111,15 @@ public DistributedProgramContainerModule(CConfiguration cConf, Configuration hCo this(cConf, hConf, programRunId, programOpts, null); } + /** + * Creates a program container module for a program. + * + * @param cConf The CConf to use. + * @param hConf The HConf to use. + * @param programRunId The program run ID. + * @param programOpts The program options. + * @param serviceAnnouncer The service announcer to use. + */ public DistributedProgramContainerModule(CConfiguration cConf, Configuration hConf, ProgramRunId programRunId, ProgramOptions programOpts, @Nullable ServiceAnnouncer serviceAnnouncer) { @@ -235,15 +244,15 @@ private void addOnPremiseModules(List modules) { MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment(); if (masterEnv == null) { - modules.add(new ZKClientModule()); - modules.add(new ZKDiscoveryModule()); + modules.add(new ZkClientModule()); + modules.add(new ZkDiscoveryModule()); modules.add(new KafkaClientModule()); modules.add(new KafkaLogAppenderModule()); return; } if (coreSecurityModule.requiresZKClient()) { - modules.add(new ZKClientModule()); + modules.add(new ZkClientModule()); } modules.add(new AbstractModule() { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/TwillModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/TwillModule.java index 54f96f3b0e92..84a5cc38d8a0 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/TwillModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/TwillModule.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2017 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -13,6 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package io.cdap.cdap.app.guice; import com.google.inject.Inject; @@ -72,7 +73,7 @@ private static final class TwillRunnerServiceProvider implements Provider modules = new ArrayList<>(); - byte[] pollerInfoBytes = Bytes.toBytes(new Gson().toJson(pollerInfo)); SConfiguration sConf = SConfiguration.create(); modules.add(new ConfigModule(cConf, hConf, sConf)); @@ -217,8 +217,8 @@ static Injector createInjector(CConfiguration cConf, Configuration hConf, MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment(); if (masterEnv == null) { - modules.add(new ZKClientModule()); - modules.add(new ZKDiscoveryModule()); + modules.add(new ZkClientModule()); + modules.add(new ZkDiscoveryModule()); modules.add(new KafkaClientModule()); modules.add(new KafkaLogAppenderModule()); } else { @@ -247,6 +247,8 @@ protected void configure() { modules.add(new AuthenticationContextModules().getMasterWorkerModule()); modules.add(new AuthorizationEnforcementModule().getNoOpModules()); + + byte[] pollerInfoBytes = Bytes.toBytes(new Gson().toJson(pollerInfo)); modules.add(new AbstractModule() { @Override protected void configure() { diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java index d37b37034eff..e8728936f407 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/TaskWorkerTwillRunnable.java @@ -1,5 +1,5 @@ /* - * Copyright © 2021-2022 Cask Data, Inc. + * Copyright © 2021-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -34,8 +34,8 @@ import io.cdap.cdap.common.guice.LocalLocationModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; import io.cdap.cdap.common.guice.SupplierProviderBridge; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.logging.LoggingContext; import io.cdap.cdap.common.logging.LoggingContextAccessor; import io.cdap.cdap.common.logging.ServiceLoggingContext; @@ -102,8 +102,8 @@ static Injector createInjector(CConfiguration cConf, Configuration hConf) { MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment(); if (masterEnv == null) { - modules.add(new ZKClientModule()); - modules.add(new ZKDiscoveryModule()); + modules.add(new ZkClientModule()); + modules.add(new ZkDiscoveryModule()); modules.add(new KafkaClientModule()); modules.add(new KafkaLogAppenderModule()); } else { @@ -120,7 +120,7 @@ protected void configure() { modules.add(new RemoteLogAppenderModule()); if (coreSecurityModule.requiresZKClient()) { - modules.add(new ZKClientModule()); + modules.add(new ZkClientModule()); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerTwillRunnable.java index 037f50855917..0f81d52d62a4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/sidecar/ArtifactLocalizerTwillRunnable.java @@ -1,5 +1,5 @@ /* - * Copyright © 2021-2022 Cask Data, Inc. + * Copyright © 2021-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -35,8 +35,8 @@ import io.cdap.cdap.common.guice.LocalLocationModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; import io.cdap.cdap.common.guice.SupplierProviderBridge; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.logging.LoggingContext; import io.cdap.cdap.common.logging.LoggingContextAccessor; import io.cdap.cdap.common.logging.ServiceLoggingContext; @@ -71,7 +71,7 @@ /** * The {@link TwillRunnable} for running {@link ArtifactLocalizerService}. * - * This runnable will run as a sidecar container for {@link io.cdap.cdap.internal.app.worker.TaskWorkerTwillRunnable} + *

This runnable will run as a sidecar container for {@link io.cdap.cdap.internal.app.worker.TaskWorkerTwillRunnable} */ public class ArtifactLocalizerTwillRunnable extends AbstractTwillRunnable { @@ -85,6 +85,13 @@ public ArtifactLocalizerTwillRunnable(String cConfFileName, String hConfFileName super(ImmutableMap.of("cConf", cConfFileName, "hConf", hConfFileName)); } + /** + * Creates an injector for use in the task worker runnable. + * + * @param cConf The CConf to use. + * @param hConf The HConf to use. + * @return The injector for the task worker runnable. + */ @VisibleForTesting public static Injector createInjector(CConfiguration cConf, Configuration hConf) { List modules = new ArrayList<>(); @@ -103,8 +110,8 @@ public static Injector createInjector(CConfiguration cConf, Configuration hConf) MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment(); if (masterEnv == null) { - modules.add(new ZKClientModule()); - modules.add(new ZKDiscoveryModule()); + modules.add(new ZkClientModule()); + modules.add(new ZkDiscoveryModule()); modules.add(new KafkaClientModule()); modules.add(new KafkaLogAppenderModule()); modules.add(new DFSLocationModule()); @@ -123,7 +130,7 @@ protected void configure() { modules.add(new LocalLocationModule()); if (coreSecurityModule.requiresZKClient()) { - modules.add(new ZKClientModule()); + modules.add(new ZkClientModule()); } } modules.add(new DistributedArtifactManagerModule()); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java index 4927345aca3d..feb696e89c9c 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java @@ -1,5 +1,5 @@ /* - * Copyright © 2021 Cask Data, Inc. + * Copyright © 2021-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -45,8 +45,8 @@ import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; import io.cdap.cdap.common.guice.SupplierProviderBridge; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.logging.LoggingContext; import io.cdap.cdap.common.logging.LoggingContextAccessor; import io.cdap.cdap.common.logging.ServiceLoggingContext; @@ -197,8 +197,8 @@ protected void configure() { MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment(); if (masterEnv == null) { - modules.add(new ZKClientModule()); - modules.add(new ZKDiscoveryModule()); + modules.add(new ZkClientModule()); + modules.add(new ZkDiscoveryModule()); modules.add(new KafkaClientModule()); modules.add(new KafkaLogAppenderModule()); } else { @@ -217,14 +217,14 @@ protected void configure() { @Override protected void configure() { bind(TwillRunnerService.class).toProvider( - new SupplierProviderBridge<>(masterEnv.getTwillRunnerSupplier())) + new SupplierProviderBridge<>(masterEnv.getTwillRunnerSupplier())) .in(Scopes.SINGLETON); bind(TwillRunner.class).to(TwillRunnerService.class); } }); if (coreSecurityModule.requiresZKClient()) { - modules.add(new ZKClientModule()); + modules.add(new ZkClientModule()); } } @@ -238,12 +238,12 @@ private static Module getArtifactManagerModules(CConfiguration cConf) { protected void configure() { bind(PluginFinder.class).to(RemoteWorkerPluginFinder.class); bind(ArtifactRepositoryReader.class).to( - RemoteArtifactRepositoryReaderWithLocalization.class) + RemoteArtifactRepositoryReaderWithLocalization.class) .in(Scopes.SINGLETON); bind(ArtifactLocalizerClient.class).in(Scopes.SINGLETON); OptionalBinder.newOptionalBinder(binder(), ArtifactLocalizerClient.class); install(new FactoryModuleBuilder().implement(ArtifactManager.class, - RemoteArtifactManager.class) + RemoteArtifactManager.class) .build(ArtifactManagerFactory.class)); } }; diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityManager.java index 0cd5d4a02c7d..863370235cd6 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialIdentityManager.java @@ -77,11 +77,11 @@ public Optional get(CredentialIdentityId id) throws IOExcept /** * Creates a credential identity. * - * @param id The identity reference to create. + * @param id The identity reference to create. * @param identity The identity to create. * @throws AlreadyExistsException If the identity already exists. - * @throws IOException If any failure writing to storage occurs. - * @throws NotFoundException If the profile the identity is attached to does not exist. + * @throws IOException If any failure writing to storage occurs. + * @throws NotFoundException If the profile the identity is attached to does not exist. */ public void create(CredentialIdentityId id, CredentialIdentity identity) throws AlreadyExistsException, IOException, NotFoundException { @@ -97,9 +97,9 @@ public void create(CredentialIdentityId id, CredentialIdentity identity) /** * Updates a credential identity. * - * @param id The identity reference to update. + * @param id The identity reference to update. * @param identity The identity to update. - * @throws IOException If any failure writing to storage occurs. + * @throws IOException If any failure writing to storage occurs. * @throws NotFoundException If the identity does not exist or if the profile the identity is * attached to does not exist. */ @@ -118,7 +118,7 @@ public void update(CredentialIdentityId id, CredentialIdentity identity) * Deletes a credential identity. * * @param id The identity reference to update. - * @throws IOException If any failure writing to storage occurs. + * @throws IOException If any failure writing to storage occurs. * @throws NotFoundException If the identity does not exist. */ public void delete(CredentialIdentityId id) throws IOException, NotFoundException { @@ -134,7 +134,8 @@ public void delete(CredentialIdentityId id) throws IOException, NotFoundExceptio private void validateAndWriteIdentity(StructuredTableContext context, CredentialIdentityId id, CredentialIdentity identity) throws IOException, NotFoundException { // Validate the referenced profile exists. - CredentialProfileId profileId = identity.getCredentialProfile(); + CredentialProfileId profileId = new CredentialProfileId(identity.getProfileNamespace(), + identity.getProfileName()); if (!profileStore.get(context, profileId).isPresent()) { throw new NotFoundException(String.format("Credential profile '%s:%s' not found", profileId.getNamespace(), profileId.getName())); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileManager.java index 1781391a43f6..5681076f36f2 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProfileManager.java @@ -26,10 +26,13 @@ import io.cdap.cdap.proto.credential.CredentialProfile; import io.cdap.cdap.proto.id.CredentialIdentityId; import io.cdap.cdap.proto.id.CredentialProfileId; +import io.cdap.cdap.security.spi.credential.CredentialProvider; +import io.cdap.cdap.security.spi.credential.ProfileValidationException; import io.cdap.cdap.spi.data.transaction.TransactionRunner; import io.cdap.cdap.spi.data.transaction.TransactionRunners; import java.io.IOException; import java.util.Collection; +import java.util.Map; import java.util.Optional; import javax.inject.Inject; @@ -43,13 +46,16 @@ public class CredentialProfileManager { private final CredentialIdentityStore identityStore; private final CredentialProfileStore profileStore; private final TransactionRunner transactionRunner; + private final Map credentialProviders; @Inject CredentialProfileManager(CredentialIdentityStore identityStore, - CredentialProfileStore profileStore, TransactionRunner transactionRunner) { + CredentialProfileStore profileStore, TransactionRunner transactionRunner, + CredentialProviderLoader credentialProviderLoader) { this.identityStore = identityStore; this.profileStore = profileStore; this.transactionRunner = transactionRunner; + this.credentialProviders = credentialProviderLoader.loadCredentialProviders(); } /** @@ -81,11 +87,11 @@ public Optional get(CredentialProfileId id) throws IOExceptio /** * Creates a credential profile. * - * @param id The profile reference to create. + * @param id The profile reference to create. * @param profile The profile to create. - * @throws BadRequestException If the profile is invalid. + * @throws BadRequestException If the profile is invalid. * @throws AlreadyExistsException If the profile already exists. - * @throws IOException If any failure writing to storage occurs. + * @throws IOException If any failure writing to storage occurs. */ public void create(CredentialProfileId id, CredentialProfile profile) throws AlreadyExistsException, BadRequestException, IOException { @@ -102,11 +108,11 @@ public void create(CredentialProfileId id, CredentialProfile profile) /** * Updates a credential profile. * - * @param id The profile reference to update. + * @param id The profile reference to update. * @param profile The updated profile. * @throws BadRequestException If the profile is invalid. - * @throws IOException If any failure writing to storage occurs. - * @throws NotFoundException If the profile does not exist. + * @throws IOException If any failure writing to storage occurs. + * @throws NotFoundException If the profile does not exist. */ public void update(CredentialProfileId id, CredentialProfile profile) throws BadRequestException, IOException, NotFoundException { @@ -125,7 +131,7 @@ public void update(CredentialProfileId id, CredentialProfile profile) * * @param id The profile reference to delete. * @throws ConflictException If the profile still has attached identities. - * @throws IOException If any failure writing to storage occurs. + * @throws IOException If any failure writing to storage occurs. * @throws NotFoundException If the profile does not exist. */ public void delete(CredentialProfileId id) @@ -147,10 +153,20 @@ public void delete(CredentialProfileId id) } private void validateProfile(CredentialProfile profile) throws BadRequestException { - // Validate all fields are not null. - if (profile.getCredentialProviderType() == null - || profile.getCredentialProviderType().isEmpty()) { + // Ensure provider type is valid. + String providerType = profile.getCredentialProviderType(); + if (providerType == null || profile.getCredentialProviderType().isEmpty()) { throw new BadRequestException("Credential provider type cannot be null or empty."); } + if (!credentialProviders.containsKey(providerType)) { + throw new BadRequestException(String.format("Credential provider type '%s' is unsupported.", + providerType)); + } + try { + credentialProviders.get(providerType).validateProfile(profile); + } catch (ProfileValidationException e) { + throw new BadRequestException(String.format("Failed to validate profile with error: %s", + e.getMessage()), e); + } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderExtensionLoader.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderExtensionLoader.java new file mode 100644 index 000000000000..b98ecf16f0b7 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderExtensionLoader.java @@ -0,0 +1,118 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.credential; + +import com.google.inject.Inject; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.lang.ClassPathResources; +import io.cdap.cdap.common.lang.FilterClassLoader; +import io.cdap.cdap.extension.AbstractExtensionLoader; +import io.cdap.cdap.security.spi.credential.CredentialProvider; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +/** + * Extension loader for {@link CredentialProvider}. + */ +public class CredentialProviderExtensionLoader extends AbstractExtensionLoader implements CredentialProviderLoader { + + private volatile Set allowedResources; + private volatile Set allowedPackages; + + @Inject + CredentialProviderExtensionLoader(CConfiguration cConf) { + super(cConf.get(Constants.CredentialProvider.EXTENSIONS_DIR)); + } + + @Override + public Map loadCredentialProviders() { + return getAll(); + } + + @Override + protected Set getSupportedTypesForProvider(CredentialProvider credentialProvider) { + return Collections.singleton(credentialProvider.getName()); + } + + @Override + protected FilterClassLoader.Filter getExtensionParentClassLoaderFilter() { + // Only permit cdap-security-spi dependencies + return new FilterClassLoader.Filter() { + @Override + public boolean acceptResource(String resource) { + return getAllowedResources().contains(resource); + } + + @Override + public boolean acceptPackage(String packageName) { + return getAllowedPackages().contains(packageName); + } + }; + } + + /** + * Returns the set of resources that are visible to extensions. + */ + private Set getAllowedResources() { + Set resources = this.allowedResources; + if (resources != null) { + return resources; + } + + synchronized (this) { + resources = this.allowedResources; + if (resources != null) { + return resources; + } + try { + // All cdap-security-spi classes and its dependencies are visible to extensions + // The set of dependencies for cdap-security-spi should be kept at minimal to reduce + // dependency conflicts. + this.allowedResources = resources = ClassPathResources.getResourcesWithDependencies( + getClass().getClassLoader(), CredentialProvider.class); + return resources; + } catch (IOException e) { + throw new RuntimeException("Failed to find security SPI resources", e); + } + } + } + + /** + * Returns the set of package names that are visible to extensions. + */ + private Set getAllowedPackages() { + Set packages = this.allowedPackages; + if (packages != null) { + return packages; + } + synchronized (this) { + packages = this.allowedPackages; + if (packages != null) { + return packages; + } + + packages = createPackageSets(getAllowedResources()); + + this.allowedPackages = packages; + return packages; + } + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderLoader.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderLoader.java new file mode 100644 index 000000000000..78c779dced7b --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderLoader.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.credential; + +import io.cdap.cdap.security.spi.credential.CredentialProvider; +import java.util.Map; + +/** + * Provides {@link io.cdap.cdap.security.spi.credential.CredentialProvider}. + */ +public interface CredentialProviderLoader { + + /** + * Returns a map of credential providers type to credential providers. + * + * @return A map of credential providers type to credential providers. + */ + Map loadCredentialProviders(); +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderService.java new file mode 100644 index 000000000000..442710d7179a --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/CredentialProviderService.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.credential; + +import com.google.common.util.concurrent.Service; +import io.cdap.cdap.proto.credential.CredentialProvider; + +/** + * A service which provides credentials based on identity and profile. + */ +public interface CredentialProviderService extends CredentialProvider, Service { + +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderContext.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderContext.java new file mode 100644 index 000000000000..1820c2f506c7 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderContext.java @@ -0,0 +1,53 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.credential; + +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.Constants.CredentialProvider; +import io.cdap.cdap.security.spi.credential.CredentialProviderContext; +import java.util.Collections; +import java.util.Map; + +/** + * Default context used for credential provider initialization. + */ +public class DefaultCredentialProviderContext implements CredentialProviderContext { + + private final Map properties; + + /** + * Creates a new context. + * + * @param cConf The CConfiguration backing the context properties. + * @param providerName The credential provider name. + */ + protected DefaultCredentialProviderContext(CConfiguration cConf, String providerName) { + String prefix = String.format("%s%s.", CredentialProvider.SYSTEM_PROPERTY_PREFIX, + providerName); + this.properties = Collections.unmodifiableMap(cConf.getPropsWithPrefix(prefix)); + } + + /** + * Returns the properties for the provider. + * + * @return The properties for the provider. + */ + @Override + public Map getProperties() { + return properties; + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderService.java new file mode 100644 index 000000000000..749454a2cae2 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderService.java @@ -0,0 +1,136 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.credential; + +import com.google.common.util.concurrent.AbstractIdleService; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.proto.credential.CredentialIdentity; +import io.cdap.cdap.proto.credential.CredentialProfile; +import io.cdap.cdap.proto.credential.CredentialProvisioningException; +import io.cdap.cdap.proto.credential.IdentityValidationException; +import io.cdap.cdap.proto.credential.NotFoundException; +import io.cdap.cdap.proto.credential.ProvisionedCredential; +import io.cdap.cdap.proto.id.CredentialIdentityId; +import io.cdap.cdap.proto.id.CredentialProfileId; +import io.cdap.cdap.proto.security.StandardPermission; +import io.cdap.cdap.security.spi.authorization.ContextAccessEnforcer; +import io.cdap.cdap.security.spi.credential.CredentialProvider; +import java.io.IOException; +import java.util.Map; +import java.util.Optional; +import javax.inject.Inject; + +/** + * Default implementation for {@link CredentialProviderService} used in AppFabric. + */ +public class DefaultCredentialProviderService extends AbstractIdleService + implements CredentialProviderService { + + private final CConfiguration cConf; + private final ContextAccessEnforcer contextAccessEnforcer; + private final Map credentialProviders; + private final CredentialIdentityManager credentialIdentityManager; + private final CredentialProfileManager credentialProfileManager; + + @Inject + DefaultCredentialProviderService(CConfiguration cConf, + ContextAccessEnforcer contextAccessEnforcer, + CredentialProviderLoader credentialProviderLoader, + CredentialIdentityManager credentialIdentityManager, + CredentialProfileManager credentialProfileManager) { + this.cConf = cConf; + this.contextAccessEnforcer = contextAccessEnforcer; + this.credentialProviders = credentialProviderLoader.loadCredentialProviders(); + this.credentialIdentityManager = credentialIdentityManager; + this.credentialProfileManager = credentialProfileManager; + } + + @Override + protected void startUp() throws Exception { + for (CredentialProvider provider : credentialProviders.values()) { + provider.initialize(new DefaultCredentialProviderContext(cConf, provider.getName())); + } + } + + @Override + protected void shutDown() throws Exception { + + } + + /** + * Provisions a credential. + * + * @param namespace The identity namespace. + * @param identityName The identity name. + * @return A provisioned credential. + * @throws CredentialProvisioningException If provisioning fails in the extension. + * @throws IOException If any transport errors occur. + * @throws NotFoundException If the identity or profile are not found. + */ + @Override + public ProvisionedCredential provision(String namespace, String identityName) + throws CredentialProvisioningException, IOException, NotFoundException { + CredentialIdentityId identityId = new CredentialIdentityId(namespace, identityName); + contextAccessEnforcer.enforce(identityId, StandardPermission.USE); + Optional optIdentity = credentialIdentityManager.get(identityId); + if (!optIdentity.isPresent()) { + throw new NotFoundException(String.format("Credential identity '%s' was not found.", + identityId.toString())); + } + CredentialIdentity identity = optIdentity.get(); + return validateAndProvisionIdentity(identity); + } + + /** + * Validates an identity. + * + * @param identity The identity to validate. + * @throws IdentityValidationException If identity validation fails in the extension. + * @throws IOException If any transport errors occur. + * @throws NotFoundException If the identity or profile are not found. + */ + @Override + public void validateIdentity(CredentialIdentity identity) throws IdentityValidationException, + IOException, NotFoundException { + try { + validateAndProvisionIdentity(identity); + } catch (CredentialProvisioningException e) { + throw new IdentityValidationException(e); + } + } + + private ProvisionedCredential validateAndProvisionIdentity(CredentialIdentity identity) + throws CredentialProvisioningException, IOException, NotFoundException { + CredentialProfileId profileId = new CredentialProfileId(identity.getProfileNamespace(), + identity.getProfileName()); + contextAccessEnforcer.enforce(profileId, StandardPermission.USE); + Optional optProfile = credentialProfileManager.get(profileId); + if (!optProfile.isPresent()) { + throw new NotFoundException(String.format("Credential profile '%s' was not found.", + profileId.toString())); + } + CredentialProfile profile = optProfile.get(); + // This is a sanity check which should be impossible to fail. + String providerType = profile.getCredentialProviderType(); + if (!credentialProviders.containsKey(providerType)) { + throw new IllegalStateException(String.format("Unsupported credential provider type " + + "'%'", providerType)); + } + // Provision and return the credential. + return credentialProviders.get(providerType).provision(profile, identity); + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/guice/MasterCredentialProviderModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/guice/MasterCredentialProviderModule.java new file mode 100644 index 000000000000..3ab18f773fd4 --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/guice/MasterCredentialProviderModule.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.credential.guice; + +import com.google.inject.AbstractModule; +import com.google.inject.Scopes; +import io.cdap.cdap.internal.credential.CredentialProviderExtensionLoader; +import io.cdap.cdap.internal.credential.CredentialProviderLoader; +import io.cdap.cdap.internal.credential.CredentialProviderService; +import io.cdap.cdap.internal.credential.DefaultCredentialProviderService; +import io.cdap.cdap.proto.credential.CredentialProvider; + +/** + * Credential provider module for AppFabric. + */ +public class MasterCredentialProviderModule extends AbstractModule { + + @Override + protected void configure() { + bind(CredentialProvider.class).to(CredentialProviderService.class).in(Scopes.SINGLETON); + bind(CredentialProviderService.class).to(DefaultCredentialProviderService.class) + .in(Scopes.SINGLETON); + bind(CredentialProviderLoader.class).to(CredentialProviderExtensionLoader.class); + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/CredentialProviderHttpHandler.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/CredentialProviderHttpHandler.java index 02f1d995ad71..f254110d03a2 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/CredentialProviderHttpHandler.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/CredentialProviderHttpHandler.java @@ -33,6 +33,8 @@ import io.cdap.cdap.proto.credential.CreateCredentialProfileRequest; import io.cdap.cdap.proto.credential.CredentialIdentity; import io.cdap.cdap.proto.credential.CredentialProfile; +import io.cdap.cdap.proto.credential.CredentialProvider; +import io.cdap.cdap.proto.credential.IdentityValidationException; import io.cdap.cdap.proto.element.EntityType; import io.cdap.cdap.proto.id.CredentialIdentityId; import io.cdap.cdap.proto.id.CredentialProfileId; @@ -74,16 +76,19 @@ public class CredentialProviderHttpHandler extends AbstractHttpHandler { private final NamespaceQueryAdmin namespaceQueryAdmin; private final CredentialIdentityManager credentialIdentityManager; private final CredentialProfileManager credentialProfileManager; + private final CredentialProvider credentialProvider; @Inject CredentialProviderHttpHandler(ContextAccessEnforcer accessEnforcer, NamespaceQueryAdmin namespaceQueryAdmin, CredentialIdentityManager credentialIdentityManager, - CredentialProfileManager credentialProfileManager) { + CredentialProfileManager credentialProfileManager, + CredentialProvider credentialProvider) { this.accessEnforcer = accessEnforcer; this.namespaceQueryAdmin = namespaceQueryAdmin; this.credentialIdentityManager = credentialIdentityManager; this.credentialProfileManager = credentialProfileManager; + this.credentialProvider = credentialProvider; } /** @@ -99,6 +104,30 @@ public void listProviders(HttpRequest request, HttpResponder responder) { "Listing providers is unsupported."); } + /** + * Validates a credential identity. + * + * @param request The HTTP request. + * @param responder The HTTP responder. + * @throws BadRequestException If identity validation fails. + * @throws NotFoundException If the associated profile is not found. + * @throws IOException If transport errors occur. + */ + @POST + @Path("/credentials/identities/validate") + public void validateIdentity(FullHttpRequest request, HttpResponder responder) + throws BadRequestException, NotFoundException, IOException { + CredentialIdentity identity = deserializeRequestContent(request, CredentialIdentity.class); + try { + credentialProvider.validateIdentity(identity); + } catch (IdentityValidationException e) { + throw new BadRequestException(String.format("Identity failed validation with error: %s", + e.getMessage()), e); + } catch (io.cdap.cdap.proto.credential.NotFoundException e) { + throw new NotFoundException(e.getMessage()); + } + } + /** * Returns a list of credential profiles for the namespace. * @@ -299,8 +328,9 @@ public void createIdentity(FullHttpRequest request, HttpResponder responder, } accessEnforcer.enforce(identityId, StandardPermission.CREATE); ensureNamespaceExists(namespace); - validateIdentity(identity); - accessEnforcer.enforce(identity.getCredentialProfile(), StandardPermission.GET); + validateCredentialIdentity(identity); + accessEnforcer.enforce(new CredentialProfileId(identity.getProfileNamespace(), + identity.getProfileName()), StandardPermission.GET); credentialIdentityManager.create(identityId, identity); responder.sendStatus(HttpResponseStatus.OK); } @@ -328,8 +358,9 @@ public void updateIdentity(FullHttpRequest request, HttpResponder responder, ensureNamespaceExists(namespace); final CredentialIdentity identity = deserializeRequestContent(request, CredentialIdentity.class); - validateIdentity(identity); - accessEnforcer.enforce(identity.getCredentialProfile(), StandardPermission.GET); + validateCredentialIdentity(identity); + accessEnforcer.enforce(new CredentialProfileId(identity.getProfileNamespace(), + identity.getProfileName()), StandardPermission.GET); credentialIdentityManager.update(identityId, identity); responder.sendStatus(HttpResponseStatus.OK); } @@ -361,15 +392,18 @@ public void deleteIdentity(HttpRequest request, HttpResponder responder, private void ensureNamespaceExists(String namespace) throws IOException, NamespaceNotFoundException { + NamespaceId namespaceId; + Boolean namespaceFound; try { - NamespaceId namespaceId = new NamespaceId(namespace); - if (!namespaceQueryAdmin.exists(namespaceId)) { - throw new NamespaceNotFoundException(namespaceId); - } + namespaceId = new NamespaceId(namespace); + namespaceFound = namespaceQueryAdmin.exists(namespaceId); } catch (Exception e) { throw new IOException(String.format("Failed to check if namespace '%s' exists", namespace), e); } + if (!namespaceFound) { + throw new NamespaceNotFoundException(namespaceId); + } } private CredentialProfileId createProfileIdOrPropagate(String namespace, String name) @@ -390,8 +424,8 @@ private CredentialIdentityId createIdentityIdOrPropagate(String namespace, Strin } } - private void validateIdentity(CredentialIdentity identity) throws BadRequestException { - if (identity.getCredentialProfile() == null) { + private void validateCredentialIdentity(CredentialIdentity identity) throws BadRequestException { + if (identity.getProfileNamespace() == null || identity.getProfileName() == null) { throw new BadRequestException("Identity provided with no associated profile"); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/CredentialProviderHttpHandlerInternal.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/CredentialProviderHttpHandlerInternal.java index 16b4dec9014e..61f842f07b24 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/CredentialProviderHttpHandlerInternal.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/CredentialProviderHttpHandlerInternal.java @@ -16,12 +16,21 @@ package io.cdap.cdap.internal.credential.handler; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.inject.Singleton; +import io.cdap.cdap.common.NotFoundException; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.proto.credential.CredentialProvider; +import io.cdap.cdap.proto.credential.CredentialProvisioningException; import io.cdap.http.AbstractHttpHandler; import io.cdap.http.HttpHandler; import io.cdap.http.HttpResponder; import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponseStatus; +import java.io.IOException; +import java.time.Instant; +import javax.inject.Inject; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -33,10 +42,37 @@ @Path(Constants.Gateway.INTERNAL_API_VERSION_3) public class CredentialProviderHttpHandlerInternal extends AbstractHttpHandler { + private static final Gson GSON = new GsonBuilder().registerTypeAdapter(Instant.class, + new InstantEpochSecondsTypeAdapter()).create(); + + private final CredentialProvider credentialProvider; + + @Inject + CredentialProviderHttpHandlerInternal(CredentialProvider credentialProvider) { + this.credentialProvider = credentialProvider; + } + + /** + * Provisions a credential for a given identity. + * + * @param request The HTTP request. + * @param responder The HTTP responder. + * @param namespace The namespace of the identity for which to provision a credential. + * @param identityName The name of the identity for which to provision a credential. + * @throws CredentialProvisioningException If provisioning fails. + * @throws IOException If transport errors occur. + * @throws NotFoundException If the identity or associated profile are not found. + */ @POST - @Path("/namespaces/{namespace-id}/credentials/identities/{identity-id}/provision") + @Path("/namespaces/{namespace-id}/credentials/identities/{identity-name}/provision") public void provisionCredential(HttpRequest request, HttpResponder responder, - @PathParam("namespace-id") String namespace, @PathParam("identity-id") String identity) { - throw new UnsupportedOperationException("Credential provisioning is unsupported."); + @PathParam("namespace-id") String namespace, @PathParam("identity-name") String identityName) + throws CredentialProvisioningException, IOException, NotFoundException { + try { + responder.sendJson(HttpResponseStatus.OK, + GSON.toJson(credentialProvider.provision(namespace, identityName))); + } catch (io.cdap.cdap.proto.credential.NotFoundException e) { + throw new NotFoundException(e.getMessage()); + } } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/InstantEpochSecondsTypeAdapter.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/InstantEpochSecondsTypeAdapter.java new file mode 100644 index 000000000000..3aef2f886c5c --- /dev/null +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/handler/InstantEpochSecondsTypeAdapter.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.credential.handler; + +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import java.io.IOException; +import java.time.Instant; + +/** + * Type adapter which converts an {@link Instant} to a timestamp in seconds. + */ +public class InstantEpochSecondsTypeAdapter extends TypeAdapter { + + @Override + public void write(JsonWriter jsonWriter, Instant instant) throws IOException { + jsonWriter.value(instant.getEpochSecond()); + } + + @Override + public Instant read(JsonReader jsonReader) throws IOException { + return Instant.ofEpochSecond(jsonReader.nextLong()); + } +} diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialIdentityStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialIdentityStore.java index 3ecd32248bf0..3ca48f0ff1e8 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialIdentityStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialIdentityStore.java @@ -49,7 +49,7 @@ public class CredentialIdentityStore { /** * Lists entries in the credential identity table for a given namespace. * - * @param context The transaction context to use. + * @param context The transaction context to use. * @param namespace The namespace to list identities from. * @return A collection of identities in the namespace. * @throws IOException If any failure reading from storage occurs. @@ -68,7 +68,7 @@ public Collection list(StructuredTableContext context, Str /** * Lists entries in the credential identity table for a given profile. * - * @param context The transaction context to use. + * @param context The transaction context to use. * @param profileId The profile to list identities for. * @return A Collection of identities attached to the provided profile. * @throws IOException If any failure reading from storage occurs. @@ -76,7 +76,8 @@ public Collection list(StructuredTableContext context, Str public Collection listForProfile(StructuredTableContext context, CredentialProfileId profileId) throws IOException { Field indexKey = Fields.stringField( - CredentialProviderStore.IDENTITY_PROFILE_INDEX_FIELD, toProfileIndex(profileId)); + CredentialProviderStore.IDENTITY_PROFILE_INDEX_FIELD, + toProfileIndex(profileId.getNamespace(), profileId.getName())); try (CloseableIterator identityIterator = context.getTable( CredentialProviderStore.CREDENTIAL_IDENTITIES).scan(indexKey)) { return identitiesFromRowIterator(identityIterator); @@ -87,7 +88,7 @@ public Collection listForProfile(StructuredTableContext co * Fetch an entry from the identity table. * * @param context The transaction context to use. - * @param id The identity reference to fetch. + * @param id The identity reference to fetch. * @return The fetched credential identity. * @throws IOException If any failure reading from storage occurs. */ @@ -106,8 +107,8 @@ public Optional get(StructuredTableContext context, Credenti /** * Write an entry to the credential identity table. * - * @param context The transaction context to use. - * @param id The identity reference to write to. + * @param context The transaction context to use. + * @param id The identity reference to write to. * @param identity The identity to write. * @throws IOException If any failure reading from storage occurs. */ @@ -123,7 +124,7 @@ public void write(StructuredTableContext context, CredentialIdentityId id, Fields.stringField(CredentialProviderStore.IDENTITY_DATA_FIELD, GSON.toJson(identity)), Fields.stringField(CredentialProviderStore.IDENTITY_PROFILE_INDEX_FIELD, - toProfileIndex(identity.getCredentialProfile()))); + toProfileIndex(identity.getProfileNamespace(), identity.getProfileName()))); identityTable.upsert(row); } @@ -131,7 +132,7 @@ public void write(StructuredTableContext context, CredentialIdentityId id, * Deletes an entry from the credential identity table. * * @param context The transaction context to use. - * @param id The identity reference to delete. + * @param id The identity reference to delete. * @throws IOException If any failure reading from storage occurs. */ public void delete(StructuredTableContext context, CredentialIdentityId id) @@ -155,7 +156,7 @@ private static Collection identitiesFromRowIterator( .collect(Collectors.toList()); } - private static String toProfileIndex(CredentialProfileId profileId) { - return String.format("%s:%s", profileId.getNamespace(), profileId.getName()); + private static String toProfileIndex(String profileNamespace, String profileName) { + return String.format("%s:%s", profileNamespace, profileName); } } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialProfileStore.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialProfileStore.java index 9f9f9814b73f..fbb76048c000 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialProfileStore.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/credential/store/CredentialProfileStore.java @@ -49,7 +49,7 @@ public class CredentialProfileStore { /** * Lists entries in the credential profile table for a given namespace. * - * @param context The transaction context to use. + * @param context The transaction context to use. * @param namespace The namespace to list profiles from. * @return A collection of profiles in the namespace. * @throws IOException If any failure reading from storage occurs. @@ -69,7 +69,7 @@ public Collection list(StructuredTableContext context, Stri * Fetch an entry from the profile table. * * @param context The transaction context to use. - * @param id The profile reference to fetch. + * @param id The profile reference to fetch. * @return The fetched credential profile. * @throws IOException If any failure reading from storage occurs. */ @@ -89,7 +89,7 @@ public Optional get(StructuredTableContext context, Credentia * Write an entry to the credential profile table. * * @param context The transaction context to use. - * @param id The profile reference to write to. + * @param id The profile reference to write to. * @param profile The profile to write. * @throws IOException If any failure reading from storage occurs. */ @@ -110,7 +110,7 @@ public void write(StructuredTableContext context, CredentialProfileId id, * Deletes an entry from the credential profile table. * * @param context The transaction context to use. - * @param id The profile reference to delete. + * @param id The profile reference to delete. * @throws IOException If any failure reading from storage occurs. */ public void delete(StructuredTableContext context, CredentialProfileId id) @@ -124,7 +124,8 @@ public void delete(StructuredTableContext context, CredentialProfileId id) table.delete(key); } - private static Collection profilesFromRowIterator(Iterator iterator) { + private static Collection profilesFromRowIterator( + Iterator iterator) { return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false) .map(row -> new CredentialProfileId( diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/AbstractServiceMain.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/AbstractServiceMain.java index 6ee7c99a377e..33de125eac3d 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/AbstractServiceMain.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/master/environment/k8s/AbstractServiceMain.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019-2022 Cask Data, Inc. + * Copyright © 2019-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -35,7 +35,7 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; import io.cdap.cdap.common.guice.SupplierProviderBridge; -import io.cdap.cdap.common.guice.ZKClientModule; +import io.cdap.cdap.common.guice.ZkClientModule; import io.cdap.cdap.common.logging.LoggingContext; import io.cdap.cdap.common.logging.LoggingContextAccessor; import io.cdap.cdap.common.logging.common.UncaughtExceptionHandler; @@ -92,8 +92,8 @@ public abstract class AbstractServiceMain extends * Helper method for sub-class to call from static void main. * * @param mainClass the class of the master main class implementation - * @param args arguments to main - * @param type of the master main class + * @param args arguments to main + * @param type of the master main class * @throws Exception if execution failed */ protected static > @@ -183,7 +183,7 @@ protected void configure() { CoreSecurityModule coreSecurityModule = CoreSecurityRuntimeModule.getDistributedModule(cConf); modules.add(coreSecurityModule); if (coreSecurityModule.requiresZKClient()) { - modules.add(new ZKClientModule()); + modules.add(new ZkClientModule()); } modules.add(new AuthenticationContextModules().getMasterModule()); @@ -305,6 +305,7 @@ protected Module getLogAppenderModule() { * @param cConf the {@link CConfiguration} to be updated * @return the updated configuration */ + @SuppressWarnings("checkstyle:AbbreviationAsWordInName") protected CConfiguration updateCConf(CConfiguration cConf) { return cConf; } @@ -318,12 +319,12 @@ protected abstract List getServiceModules(MasterEnvironment masterEnv, T /** * Adds {@link Service} to run. * - * @param injector the Guice {@link Injector} for all the necessary bindings - * @param services the {@link List} to populate services to run + * @param injector the Guice {@link Injector} for all the necessary bindings + * @param services the {@link List} to populate services to run * @param closeableResources the {@link List} to populate {@link AutoCloseable} that will be - * closed on stopping - * @param masterEnv the {@link MasterEnvironment} created for this main service - * @param masterEnvContext the {@link MasterEnvironmentContext} created for this main service + * closed on stopping + * @param masterEnv the {@link MasterEnvironment} created for this main service + * @param masterEnvContext the {@link MasterEnvironmentContext} created for this main service */ protected abstract void addServices(Injector injector, List services, List closeableResources, diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialIdentityManagerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialIdentityManagerTest.java index 0deb7e794520..7b6dd7ef342c 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialIdentityManagerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialIdentityManagerTest.java @@ -19,12 +19,10 @@ import io.cdap.cdap.common.AlreadyExistsException; import io.cdap.cdap.common.NotFoundException; import io.cdap.cdap.proto.credential.CredentialIdentity; -import io.cdap.cdap.proto.credential.CredentialProfile; import io.cdap.cdap.proto.id.CredentialIdentityId; import io.cdap.cdap.proto.id.CredentialProfileId; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Optional; import org.junit.Assert; import org.junit.Test; @@ -32,35 +30,29 @@ /** * Tests for {@link CredentialIdentityManager}. */ -public class CredentialIdentityManagerTest extends CredentialManagerTestBase { +public class CredentialIdentityManagerTest extends CredentialProviderTestBase { private void assertCredentialIdentitiesEqual(CredentialIdentity expected, CredentialIdentity actual) { - Assert.assertEquals(expected.getCredentialProfile(), actual.getCredentialProfile()); + Assert.assertEquals(expected.getProfileNamespace(), actual.getProfileNamespace()); + Assert.assertEquals(expected.getProfileName(), actual.getProfileName()); Assert.assertEquals(expected.getIdentity(), actual.getIdentity()); Assert.assertEquals(expected.getSecureValue(), actual.getSecureValue()); } - private CredentialProfileId createDummyProfile(String namespace, String name) throws Exception { - CredentialProfile profile = new CredentialProfile("test", "some description", - Collections.singletonMap("some-key", "some-value")); - CredentialProfileId profileId = new CredentialProfileId(namespace, name); - credentialProfileManager.create(profileId, profile); - return profileId; - } - @Test public void testListIdentities() throws Exception { // Create a profile. String namespace = "testListIdentities"; - CredentialProfileId profileId = createDummyProfile(namespace, "list-identities-profile"); + CredentialProfileId profileId = createDummyProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + namespace, "list-identities-profile"); // Create 2 identities. CredentialIdentityId id1 = new CredentialIdentityId(namespace, "list1"); - CredentialIdentity identity1 = new CredentialIdentity(profileId, "some-identity", - "some-secure-value"); + CredentialIdentity identity1 = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-identity", "some-secure-value"); CredentialIdentityId id2 = new CredentialIdentityId(namespace, "list2"); - CredentialIdentity identity2 = new CredentialIdentity(profileId, "some-other-identity", - "some-other-secure-value"); + CredentialIdentity identity2 = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-other-identity", "some-other-secure-value"); credentialIdentityManager.create(id1, identity1); credentialIdentityManager.create(id2, identity2); Collection returnedIdentities = credentialIdentityManager @@ -72,20 +64,21 @@ public void testListIdentities() throws Exception { public void testCreateGetUpdateGetDelete() throws Exception { // Create a new profile. String namespace = "testCreateGetUpdateGetDelete"; - CredentialProfileId profileId = createDummyProfile(namespace, "test-profile"); + CredentialProfileId profileId = createDummyProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + namespace, "test-profile"); // Create a new identity. CredentialIdentityId id = new CredentialIdentityId(namespace, "test"); - CredentialIdentity identity = new CredentialIdentity(profileId, "some-identity", - "some-secure-value"); + CredentialIdentity identity = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-identity", "some-secure-value"); credentialIdentityManager.create(id, identity); Optional returnedIdentity = credentialIdentityManager.get(id); Assert.assertTrue(returnedIdentity.isPresent()); assertCredentialIdentitiesEqual(identity, returnedIdentity.get()); // Update the identity. - CredentialIdentity identity2 = new CredentialIdentity(profileId, "some-other-identity", - "some-other-secure-value"); + CredentialIdentity identity2 = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-other-identity", "some-other-secure-value"); credentialIdentityManager.update(id, identity2); returnedIdentity = credentialIdentityManager.get(id); Assert.assertTrue(returnedIdentity.isPresent()); @@ -100,12 +93,13 @@ public void testCreateGetUpdateGetDelete() throws Exception { @Test(expected = AlreadyExistsException.class) public void testCreateThrowsExceptionWhenAlreadyExists() throws Exception { String namespace = "testCreateThrowsExceptionWhenAlreadyExists"; - CredentialProfileId profileId = createDummyProfile(namespace, "test-profile"); + CredentialProfileId profileId = createDummyProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + namespace, "test-profile"); CredentialIdentityId id = new CredentialIdentityId(namespace, "test"); - CredentialIdentity identity1 = new CredentialIdentity(profileId, "some-identity", - "some-secure-value"); - CredentialIdentity identity2 = new CredentialIdentity(profileId, "some-other-identity", - "some-other-secure-value"); + CredentialIdentity identity1 = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-identity", "some-secure-value"); + CredentialIdentity identity2 = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-other-identity", "some-other-secure-value"); credentialIdentityManager.create(id, identity1); credentialIdentityManager.create(id, identity2); } @@ -116,19 +110,20 @@ public void testCreateThrowsExceptionWhenProfileDoesNotExist() throws Exception CredentialProfileId nonexistentProfile = new CredentialProfileId(namespace, "does-not-exist"); CredentialIdentityId id = new CredentialIdentityId(namespace, "test"); - CredentialIdentity identity = new CredentialIdentity(nonexistentProfile, "some-identity", - "some-secure-value"); + CredentialIdentity identity = new CredentialIdentity(nonexistentProfile.getNamespace(), + nonexistentProfile.getName(), "some-identity", "some-secure-value"); credentialIdentityManager.create(id, identity); } @Test(expected = NotFoundException.class) public void testUpdateThrowsExceptionWhenNotFound() throws Exception { String namespace = "testUpdateThrowsExceptionWhenNotFound"; - CredentialProfileId profileId = createDummyProfile(namespace, "test-profile"); + CredentialProfileId profileId = createDummyProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + namespace, "test-profile"); // Create a new identity. CredentialIdentityId id = new CredentialIdentityId(namespace, "does-not-exist"); - CredentialIdentity identity = new CredentialIdentity(profileId, "some-identity", - "some-secure-value"); + CredentialIdentity identity = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-identity", "some-secure-value"); credentialIdentityManager.update(id, identity); } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialManagerTestBase.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialManagerTestBase.java deleted file mode 100644 index 140270c2b2f3..000000000000 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialManagerTestBase.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright © 2023 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.cdap.internal.credential; - -import com.google.inject.AbstractModule; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Scopes; -import io.cdap.cdap.api.metrics.MetricsCollectionService; -import io.cdap.cdap.common.conf.CConfiguration; -import io.cdap.cdap.common.guice.ConfigModule; -import io.cdap.cdap.common.metrics.NoOpMetricsCollectionService; -import io.cdap.cdap.data.runtime.StorageModule; -import io.cdap.cdap.data.runtime.SystemDatasetRuntimeModule; -import io.cdap.cdap.internal.credential.store.CredentialIdentityStore; -import io.cdap.cdap.internal.credential.store.CredentialProfileStore; -import io.cdap.cdap.spi.data.StructuredTableAdmin; -import io.cdap.cdap.spi.data.transaction.TransactionRunner; -import io.cdap.cdap.store.StoreDefinition.CredentialProviderStore; -import java.io.IOException; -import org.apache.tephra.TransactionManager; -import org.apache.tephra.runtime.TransactionModules; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -public class CredentialManagerTestBase { - - private static TransactionManager txManager; - static CredentialProfileManager credentialProfileManager; - static CredentialIdentityManager credentialIdentityManager; - - @BeforeClass - public static void beforeClass() throws IOException { - CConfiguration cConf = CConfiguration.create(); - Injector injector = Guice.createInjector(new ConfigModule(cConf), - new SystemDatasetRuntimeModule().getInMemoryModules(), - new StorageModule(), - new TransactionModules().getInMemoryModules(), - new AbstractModule() { - @Override - protected void configure() { - bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class) - .in(Scopes.SINGLETON); - } - }); - txManager = injector.getInstance(TransactionManager.class); - txManager.startAndWait(); - TransactionRunner runner = injector.getInstance(TransactionRunner.class); - CredentialProviderStore.create(injector - .getInstance(StructuredTableAdmin.class)); - CredentialProfileStore profileStore = new CredentialProfileStore(); - CredentialIdentityStore identityStore = new CredentialIdentityStore(); - credentialProfileManager = new CredentialProfileManager(identityStore, profileStore, - runner); - credentialIdentityManager = new CredentialIdentityManager(identityStore, profileStore, - runner); - } - - @AfterClass - public static void teardown() throws Exception { - if (txManager != null) { - txManager.stopAndWait(); - } - } -} diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProfileManagerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProfileManagerTest.java index 33f04d2a6020..59c4a1ef01b0 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProfileManagerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProfileManagerTest.java @@ -17,6 +17,7 @@ package io.cdap.cdap.internal.credential; import io.cdap.cdap.common.AlreadyExistsException; +import io.cdap.cdap.common.BadRequestException; import io.cdap.cdap.common.ConflictException; import io.cdap.cdap.common.NotFoundException; import io.cdap.cdap.internal.credential.store.CredentialProfileStore; @@ -34,7 +35,7 @@ /** * Tests for {@link CredentialProfileStore}. */ -public class CredentialProfileManagerTest extends CredentialManagerTestBase { +public class CredentialProfileManagerTest extends CredentialProviderTestBase { private void assertCredentialProfilesEqual(CredentialProfile expected, CredentialProfile actual) { Assert.assertEquals(expected.getCredentialProviderType(), actual.getCredentialProviderType()); @@ -47,12 +48,12 @@ public void testListProfiles() throws Exception { String namespace = "testListProfiles"; // Create 2 profiles. CredentialProfileId id1 = new CredentialProfileId(namespace, "list1"); - CredentialProfile profile1 = new CredentialProfile("test", "some description", - Collections.singletonMap("some-key", "some-value")); + CredentialProfile profile1 = new CredentialProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + "some description", Collections.singletonMap("some-key", "some-value")); CredentialProfileId id2 = new CredentialProfileId(namespace, "list2"); - CredentialProfile profile2 = new CredentialProfile("other-test", - "some other description", - Collections.singletonMap("some-other-key", "some-other-value")); + CredentialProfile profile2 = new CredentialProfile( + CREDENTIAL_PROVIDER_TYPE_SUCCESS, + "some other description", Collections.singletonMap("some-other-key", "some-other-value")); credentialProfileManager.create(id1, profile1); credentialProfileManager.create(id2, profile2); Collection returnedProfiles = credentialProfileManager @@ -66,17 +67,16 @@ public void testCreateGetUpdateGetDelete() throws Exception { // Create a new profile. CredentialProfileId id = new CredentialProfileId(namespace, "test1"); - CredentialProfile profile = new CredentialProfile("test", "some description", - Collections.singletonMap("some-key", "some-value")); + CredentialProfile profile = new CredentialProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + "some description", Collections.singletonMap("some-key", "some-value")); credentialProfileManager.create(id, profile); Optional returnedProfile = credentialProfileManager.get(id); Assert.assertTrue(returnedProfile.isPresent()); assertCredentialProfilesEqual(profile, returnedProfile.get()); // Update the profile. - CredentialProfile profile2 = new CredentialProfile("other-test", - "some other description", - Collections.singletonMap("some-other-key", "some-other-value")); + CredentialProfile profile2 = new CredentialProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + "some other description", Collections.singletonMap("some-other-key", "some-other-value")); credentialProfileManager.update(id, profile2); returnedProfile = credentialProfileManager.get(id); Assert.assertTrue(returnedProfile.isPresent()); @@ -92,9 +92,11 @@ public void testCreateGetUpdateGetDelete() throws Exception { public void testCreateThrowsExceptionWhenAlreadyExists() throws Exception { String namespace = "testCreateThrowsExceptionWhenAlreadyExists"; CredentialProfileId id = new CredentialProfileId(namespace, "test2"); - CredentialProfile profile = new CredentialProfile("test", "some description", + CredentialProfile profile = new CredentialProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + "some description", Collections.singletonMap("some-key", "some-value")); - CredentialProfile profile2 = new CredentialProfile("test-other", + CredentialProfile profile2 = new CredentialProfile( + CREDENTIAL_PROVIDER_TYPE_SUCCESS, "some other description", Collections.singletonMap("some-other-key", "some-other-value")); credentialProfileManager.create(id, profile); @@ -105,7 +107,8 @@ public void testCreateThrowsExceptionWhenAlreadyExists() throws Exception { public void testUpdateThrowsExceptionWhenNotFound() throws Exception { String namespace = "testUpdateThrowsExceptionWhenNotFound"; CredentialProfileId id = new CredentialProfileId(namespace, "does-not-exist"); - CredentialProfile profile = new CredentialProfile("test", "some description", + CredentialProfile profile = new CredentialProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + "some description", Collections.singletonMap("some-key", "some-value")); credentialProfileManager.update(id, profile); } @@ -121,13 +124,45 @@ public void testDeleteThrowsExceptionWhenNotFound() throws Exception { public void testDeleteThrowsExceptionWhenIdentitiesStillExist() throws Exception { String namespace = "testDeleteThrowsExceptionWhenIdentitiesStillExist"; CredentialProfileId profileId = new CredentialProfileId(namespace, "has-identities"); - CredentialProfile profile = new CredentialProfile("test", "some description", + CredentialProfile profile = new CredentialProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + "some description", Collections.singletonMap("some-key", "some-value")); CredentialIdentityId identityId = new CredentialIdentityId(namespace, "test-identity"); - CredentialIdentity identity = new CredentialIdentity(profileId, "some-identity", - "some-secure-value"); + CredentialIdentity identity = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-identity", "some-secure-value"); credentialProfileManager.create(profileId, profile); credentialIdentityManager.create(identityId, identity); credentialProfileManager.delete(profileId); } + + @Test(expected = BadRequestException.class) + public void testCreateThrowsExceptionWhenUnsupportedProviderType() throws Exception { + String namespace = "testCreateThrowsExceptionWhenUnsupportedProviderType"; + CredentialProfileId id = new CredentialProfileId(namespace, "test"); + CredentialProfile profile = new CredentialProfile("invalid", + "some description", Collections.singletonMap("some-key", "some-value")); + credentialProfileManager.create(id, profile); + } + + @Test(expected = BadRequestException.class) + public void testUpdateThrowsExceptionWhenUnsupportedProviderType() throws Exception { + String namespace = "testUpdateThrowsExceptionWhenUnsupportedProviderType"; + CredentialProfileId id = new CredentialProfileId(namespace, "test"); + CredentialProfile profile = new CredentialProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + "some description", Collections.singletonMap("some-key", "some-value")); + credentialProfileManager.create(id, profile); + CredentialProfile profile2 = new CredentialProfile("invalid", + "some description", Collections.singletonMap("some-key", "some-value")); + credentialProfileManager.update(id, profile2); + } + + @Test(expected = BadRequestException.class) + public void testCreateValidationFailureThrowsException() throws Exception { + String namespace = "testCreateValidationFailureThrowsException"; + CredentialProfileId id = new CredentialProfileId(namespace, "test2"); + CredentialProfile profile = new CredentialProfile(CREDENTIAL_PROVIDER_TYPE_VALIDATION_FAILURE, + "some description", + Collections.singletonMap("some-key", "some-value")); + credentialProfileManager.create(id, profile); + } } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java new file mode 100644 index 000000000000..fba2625df286 --- /dev/null +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/CredentialProviderTestBase.java @@ -0,0 +1,147 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.credential; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Scopes; +import io.cdap.cdap.api.metrics.MetricsCollectionService; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.guice.ConfigModule; +import io.cdap.cdap.common.metrics.NoOpMetricsCollectionService; +import io.cdap.cdap.data.runtime.StorageModule; +import io.cdap.cdap.data.runtime.SystemDatasetRuntimeModule; +import io.cdap.cdap.internal.credential.store.CredentialIdentityStore; +import io.cdap.cdap.internal.credential.store.CredentialProfileStore; +import io.cdap.cdap.proto.credential.CredentialProfile; +import io.cdap.cdap.proto.credential.CredentialProvisioningException; +import io.cdap.cdap.proto.credential.ProvisionedCredential; +import io.cdap.cdap.proto.id.CredentialProfileId; +import io.cdap.cdap.security.authorization.AuthorizationEnforcementModule; +import io.cdap.cdap.security.spi.authorization.ContextAccessEnforcer; +import io.cdap.cdap.security.spi.credential.CredentialProvider; +import io.cdap.cdap.security.spi.credential.ProfileValidationException; +import io.cdap.cdap.spi.data.StructuredTableAdmin; +import io.cdap.cdap.spi.data.transaction.TransactionRunner; +import io.cdap.cdap.store.StoreDefinition.CredentialProviderStore; +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.tephra.TransactionManager; +import org.apache.tephra.runtime.TransactionModules; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +public class CredentialProviderTestBase { + + private static TransactionManager txManager; + static ContextAccessEnforcer contextAccessEnforcer; + static CredentialProfileManager credentialProfileManager; + static CredentialIdentityManager credentialIdentityManager; + static Map credentialProviders = new HashMap<>(); + + // Some pre-created mock credential providers which succeed, fail provisioning, or fail validation. + static String CREDENTIAL_PROVIDER_TYPE_SUCCESS = "success"; + static String CREDENTIAL_PROVIDER_TYPE_VALIDATION_FAILURE = "validationFailure"; + static String CREDENTIAL_PROVIDER_TYPE_PROVISION_FAILURE = "provisionFailure"; + + static ProvisionedCredential RETURNED_TOKEN = new ProvisionedCredential("returned_token", + Instant.ofEpochSecond(9999)); + + static class MockCredentialProviderLoader implements CredentialProviderLoader { + + @Override + public Map loadCredentialProviders() { + return credentialProviders; + } + } + + @BeforeClass + public static void beforeClass() throws CredentialProvisioningException, IOException, + ProfileValidationException { + CConfiguration cConf = CConfiguration.create(); + Injector injector = Guice.createInjector(new ConfigModule(cConf), + new SystemDatasetRuntimeModule().getInMemoryModules(), + new StorageModule(), + new TransactionModules().getInMemoryModules(), + new AuthorizationEnforcementModule().getNoOpModules(), + new AbstractModule() { + @Override + protected void configure() { + bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class) + .in(Scopes.SINGLETON); + } + }); + txManager = injector.getInstance(TransactionManager.class); + txManager.startAndWait(); + contextAccessEnforcer = injector.getInstance(ContextAccessEnforcer.class); + CredentialProviderStore.create(injector + .getInstance(StructuredTableAdmin.class)); + // Setup mock credential providers. + CredentialProvider mockCredentialProvider = mock(CredentialProvider.class); + when(mockCredentialProvider.provision(any(), any())).thenReturn(RETURNED_TOKEN); + CredentialProvider validationFailureMockCredentialProvider = mock(CredentialProvider.class); + when(validationFailureMockCredentialProvider.provision(any(), any())) + .thenReturn(RETURNED_TOKEN); + doThrow(new ProfileValidationException("profile validation always fails with this provider")) + .when(validationFailureMockCredentialProvider).validateProfile(any()); + CredentialProvider provisionFailureMockCredentialProvider = mock(CredentialProvider.class); + when(provisionFailureMockCredentialProvider.provision(any(), any())) + .thenThrow(new CredentialProvisioningException("provisioning always fails with this " + + "provider")); + credentialProviders.put(CREDENTIAL_PROVIDER_TYPE_SUCCESS, mockCredentialProvider); + credentialProviders.put(CREDENTIAL_PROVIDER_TYPE_VALIDATION_FAILURE, + validationFailureMockCredentialProvider); + credentialProviders.put(CREDENTIAL_PROVIDER_TYPE_PROVISION_FAILURE, + provisionFailureMockCredentialProvider); + CredentialProviderLoader mockCredentialProviderLoader + = new MockCredentialProviderLoader(); + + // Setup credential managers. + TransactionRunner runner = injector.getInstance(TransactionRunner.class); + CredentialProfileStore profileStore = new CredentialProfileStore(); + CredentialIdentityStore identityStore = new CredentialIdentityStore(); + credentialProfileManager = new CredentialProfileManager(identityStore, profileStore, + runner, mockCredentialProviderLoader); + credentialIdentityManager = new CredentialIdentityManager(identityStore, profileStore, + runner); + } + + @AfterClass + public static void teardown() throws Exception { + if (txManager != null) { + txManager.stopAndWait(); + } + } + + CredentialProfileId createDummyProfile(String type, String namespace, String name) + throws Exception { + CredentialProfile profile = new CredentialProfile(type, "some description", + Collections.singletonMap("some-key", "some-value")); + CredentialProfileId profileId = new CredentialProfileId(namespace, name); + credentialProfileManager.create(profileId, profile); + return profileId; + } +} diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderServiceTest.java new file mode 100644 index 000000000000..f4c2d6487450 --- /dev/null +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/DefaultCredentialProviderServiceTest.java @@ -0,0 +1,113 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.credential; + +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.proto.credential.CredentialIdentity; +import io.cdap.cdap.proto.credential.CredentialProvisioningException; +import io.cdap.cdap.proto.credential.IdentityValidationException; +import io.cdap.cdap.proto.credential.NotFoundException; +import io.cdap.cdap.proto.id.CredentialIdentityId; +import io.cdap.cdap.proto.id.CredentialProfileId; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Tests for {@link DefaultCredentialProviderService}. + */ +public class DefaultCredentialProviderServiceTest extends CredentialProviderTestBase { + + private static DefaultCredentialProviderService credentialProviderService; + + @BeforeClass + public static void startup() { + credentialProviderService = new DefaultCredentialProviderService(CConfiguration.create(), + contextAccessEnforcer, new MockCredentialProviderLoader(), credentialIdentityManager, + credentialProfileManager); + } + + @Test + public void testProvisionSuccess() throws Exception { + // Create a new profile. + String namespace = "testProvisionSuccess"; + CredentialProfileId profileId = createDummyProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, namespace, + "test-profile"); + + // Create a new identity. + CredentialIdentityId id = new CredentialIdentityId(namespace, "test"); + CredentialIdentity identity = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-identity", "some-secure-value"); + credentialIdentityManager.create(id, identity); + + Assert.assertEquals(RETURNED_TOKEN, credentialProviderService.provision(namespace, "test")); + } + + @Test(expected = NotFoundException.class) + public void testProvisionWithNotFoundIdentityThrowsException() throws Exception { + String namespace = "testProvisionWithNotFoundIdentityThrowsException"; + credentialProviderService.provision(namespace, "does-not-exist"); + } + + @Test(expected = CredentialProvisioningException.class) + public void testProvisionFailureThrowsException() throws Exception { + // Create a new profile. + String namespace = "testProvisionFailureThrowsException"; + CredentialProfileId profileId = createDummyProfile(CREDENTIAL_PROVIDER_TYPE_PROVISION_FAILURE, + namespace, "test-profile"); + + // Create a new identity. + CredentialIdentityId id = new CredentialIdentityId(namespace, "test"); + CredentialIdentity identity = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-identity", "some-secure-value"); + credentialIdentityManager.create(id, identity); + + Assert.assertEquals(RETURNED_TOKEN, credentialProviderService.provision(namespace, "test")); + } + + @Test + public void testIdentityValidationSuccess() throws Exception { + // Create a new profile. + String namespace = "testIdentityValidationSuccess"; + CredentialProfileId profileId = createDummyProfile(CREDENTIAL_PROVIDER_TYPE_SUCCESS, + namespace, "test-profile"); + + CredentialIdentity identity = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-identity", "some-secure-value"); + credentialProviderService.validateIdentity(identity); + } + + @Test(expected = IdentityValidationException.class) + public void testIdentityValidationOnProvisionFailureThrowsException() throws Exception { + // Create a new profile. + String namespace = "testIdentityValidationFailureThrowsException"; + CredentialProfileId profileId = createDummyProfile(CREDENTIAL_PROVIDER_TYPE_PROVISION_FAILURE, + namespace, "test-profile"); + + CredentialIdentity identity = new CredentialIdentity(profileId.getNamespace(), + profileId.getName(), "some-identity", "some-secure-value"); + credentialProviderService.validateIdentity(identity); + } + + @Test(expected = NotFoundException.class) + public void testIdentityValidationWithNotFoundProfileThrowsException() throws Exception { + String namespace = "testIdentityValidationWithNotFoundProfileThrowsException"; + CredentialIdentity identity = new CredentialIdentity(namespace, "does-not-exist", + "some-identity", "some-secure-value"); + credentialProviderService.validateIdentity(identity); + } +} diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/handler/InstantEpochSecondsTypeAdapterTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/handler/InstantEpochSecondsTypeAdapterTest.java new file mode 100644 index 000000000000..cbe22a9b8f6e --- /dev/null +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/credential/handler/InstantEpochSecondsTypeAdapterTest.java @@ -0,0 +1,47 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.credential.handler; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import java.time.Instant; +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests for {@link InstantEpochSecondsTypeAdapter}. + */ +public class InstantEpochSecondsTypeAdapterTest { + + @Test + public void testInstantProducesExpectedTimestamp() { + long expectedTime = 1689640167L; + Gson gson = new GsonBuilder().registerTypeAdapter(Instant.class, + new InstantEpochSecondsTypeAdapter()).create(); + String returnedTime = gson.toJson(Instant.ofEpochSecond(expectedTime)); + Assert.assertEquals(String.valueOf(expectedTime), returnedTime); + } + + @Test + public void testTimestampStringSerializesToExpectedInstant() { + long expectedTime = 1689640237L; + Gson gson = new GsonBuilder().registerTypeAdapter(Instant.class, + new InstantEpochSecondsTypeAdapter()).create(); + Instant returnedInstant = gson.fromJson(String.valueOf(expectedTime), Instant.class); + Assert.assertEquals(Instant.ofEpochSecond(expectedTime), returnedInstant); + } +} diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java b/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java index 7df591afb0c9..14cfe022274c 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java @@ -168,10 +168,9 @@ public static final class Zookeeper { public static final String TWILL_ZK_SERVER_LOCALHOST = "twill.zk.server.localhost"; /** - * Convenient method to get ZK quorum string from the configuration with - * proper default value. + * Convenient method to get ZK quorum string from the configuration with proper default value. */ - public static String getZKQuorum(CConfiguration cConf) { + public static String getZkQuorum(CConfiguration cConf) { String quorum = cConf.get(QUORUM); if (!Strings.isNullOrEmpty(quorum)) { return quorum; @@ -185,6 +184,7 @@ public static String getZKQuorum(CConfiguration cConf) { /** * HBase configurations. */ + @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public static final class HBase { public static final String AUTH_KEY_UPDATE_INTERVAL = "hbase.auth.key.update.interval"; @@ -192,21 +192,19 @@ public static final class HBase { public static final String CLIENT_RETRIES = "hbase.client.retries.number"; public static final String RPC_TIMEOUT = "hbase.rpc.timeout"; /** - * Determines how to behave when the HBase version is unsupported. - * cdap_set_hbase() method in cdap-common/bin/functions.sh must also be - * updated if this String is changed + * Determines how to behave when the HBase version is unsupported. cdap_set_hbase() method in + * cdap-common/bin/functions.sh must also be updated if this String is changed */ public static final String HBASE_VERSION_RESOLUTION_STRATEGY = "hbase.version.resolution.strategy"; /** - * Keep HBase version as it is when HBase version is unsupported. - * cdap_set_hbase() method in cdap-common/bin/functions.sh must also be - * updated if this String is changed + * Keep HBase version as it is when HBase version is unsupported. cdap_set_hbase() method in + * cdap-common/bin/functions.sh must also be updated if this String is changed */ public static final String HBASE_AUTO_STRICT_VERSION = "auto.strict"; /** - * Use latest HBase version available on the cluster when HBase version is - * unsupported. cdap_set_hbase() method in cdap-common/bin/functions.sh must - * also be updated if this String is changed + * Use latest HBase version available on the cluster when HBase version is unsupported. + * cdap_set_hbase() method in cdap-common/bin/functions.sh must also be updated if this String + * is changed */ public static final String HBASE_AUTO_LATEST_VERSION = "auto.latest"; } @@ -316,8 +314,7 @@ public static final class AppFabric { public static final String QUERY_PARAM_START_TIME = "start"; /** - * Query parameter to indicate status of a program {active, completed, - * failed}. + * Query parameter to indicate status of a program {active, completed, failed}. */ public static final String QUERY_PARAM_STATUS = "status"; @@ -339,8 +336,7 @@ public static final class AppFabric { public static final String WORKFLOW_TOKEN_MAX_SIZE_MB = "workflow.token.max.size.mb"; /** - * Name of the property used to identify whether the dataset is local or - * not. + * Name of the property used to identify whether the dataset is local or not. */ public static final String WORKFLOW_LOCAL_DATASET_PROPERTY = "workflow.local.dataset"; @@ -357,16 +353,16 @@ public static final class AppFabric { public static final String WORKFLOW_KEEP_LOCAL = "workflow.keep.local"; /** - * Configuration setting to localize extra jars to every program container - * and to be added to classpaths of CDAP programs. + * Configuration setting to localize extra jars to every program container and to be added to + * classpaths of CDAP programs. */ public static final String PROGRAM_CONTAINER_DIST_JARS = "program.container.dist.jars"; public static final String APP_UPDATE_SCHEDULES = "app.deploy.update.schedules"; /** - * Topic prefix for publishing status transitioning events of program runs - * to the messaging system. + * Topic prefix for publishing status transitioning events of program runs to the messaging + * system. */ public static final String PROGRAM_STATUS_EVENT_TOPIC = "program.status.event.topic"; @@ -376,18 +372,16 @@ public static final class AppFabric { public static final String PROGRAM_STATUS_RETRY_STRATEGY_PREFIX = "system.program.state."; /** - * Number of topics to use for program status events. All events related to - * same run should always go to same topic. If this value is 1, - * {@link #PROGRAM_STATUS_EVENT_TOPIC} is a topic name. If it's more than 1, - * {@link #PROGRAM_STATUS_EVENT_TOPIC} is a prefix, but bare name should - * still be subscribed to ensure any pending messages / active run events - * are processed properly. + * Number of topics to use for program status events. All events related to same run should + * always go to same topic. If this value is 1, {@link #PROGRAM_STATUS_EVENT_TOPIC} is a topic + * name. If it's more than 1, {@link #PROGRAM_STATUS_EVENT_TOPIC} is a prefix, but bare name + * should still be subscribed to ensure any pending messages / active run events are processed + * properly. */ public static final String PROGRAM_STATUS_EVENT_NUM_PARTITIONS = "program.status.event.topic.num.partitions"; /** - * Topic name for publishing program status recording events to the - * messaging system. + * Topic name for publishing program status recording events to the messaging system. */ public static final String PROGRAM_STATUS_RECORD_EVENT_TOPIC = "program.status.record.event.topic"; @@ -415,10 +409,9 @@ public static final class AppFabric { } /** - * A special annotation used in Guice bindings for ProgramRunner - * implementations. It is needed so that we can have different bindings in - * different private modules, without affecting/affected by unannotated - * bindings in the public space. + * A special annotation used in Guice bindings for ProgramRunner implementations. It is needed + * so that we can have different bindings in different private modules, without + * affecting/affected by unannotated bindings in the public space. */ @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) @@ -473,8 +466,7 @@ public static final class Preview { public static final class Environment { /** - * Configuration to decide if the master environment should be used for - * programs or not. + * Configuration to decide if the master environment should be used for programs or not. */ public static final String PROGRAM_SUBMISSION_MASTER_ENV_ENABLED = "program.submission.master.environment.enabled"; } @@ -603,8 +595,7 @@ public static final class Scheduler { public static final String CFG_SCHEDULER_MAX_THREAD_POOL_SIZE = "scheduler.max.thread.pool.size"; public static final String CFG_SCHEDULER_MISFIRE_THRESHOLD_MS = "scheduler.misfire.threshold.ms"; /** - * Topic name for publishing time events from time scheduler to the - * messaging system. + * Topic name for publishing time events from time scheduler to the messaging system. */ public static final String TIME_EVENT_TOPIC = "time.event.topic"; @@ -680,8 +671,8 @@ public static final class Container { public static final class DataJanitor { /** - * Whether or not the TransactionDataJanitor coprocessor should be enabled - * on tables. Disable for testing. + * Whether or not the TransactionDataJanitor coprocessor should be enabled on tables. Disable + * for testing. */ public static final String CFG_TX_JANITOR_ENABLE = "data.tx.janitor.enable"; public static final boolean DEFAULT_TX_JANITOR_ENABLE = true; @@ -772,8 +763,7 @@ public static final class Manager { public static final String OUTPUT_DIR = "dataset.service.output.dir"; /** - * Annotation for binding default dataset modules for the dataset - * service. + * Annotation for binding default dataset modules for the dataset service. */ @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) @@ -880,8 +870,7 @@ public static final class Router { // To block inbound requests through configuration, // Router will start responding to every inbound request with the response (status and message) declared in config /** - * Property to start/stop blocking requests to the router. Will be blocked - * if enabled + * Property to start/stop blocking requests to the router. Will be blocked if enabled */ public static final String BLOCK_REQUEST_ENABLED = "router.block.request.enabled"; /** @@ -1169,7 +1158,7 @@ public static final class Program { /** * JVM resource metrics. */ - public static final class JVMResource { + public static final class JvmResource { public static final String HEAP_USED_MB = "jvm.resource.heap.used.mb"; public static final String HEAP_MAX_MB = "jvm.resource.heap.max.mb"; @@ -1521,8 +1510,7 @@ public static final class Security { */ public static final String TOKEN_DIGEST_KEY_LENGTH = "security.token.digest.keylength"; /** - * Time duration in milliseconds after which an active secret key should be - * retired. + * Time duration in milliseconds after which an active secret key should be retired. */ public static final String TOKEN_DIGEST_KEY_EXPIRATION = "security.token.digest.key.expiration.ms"; /** @@ -1530,10 +1518,9 @@ public static final class Security { */ public static final String DIST_KEY_PARENT_ZNODE = "security.token.distributed.parent.znode"; /** - * Comma separated URL's that clients should use to communicate with the - * Authentication Server. Each URL should follow the format - * protocol://host:port. Leave empty to use the default URL generated by the - * Authentication Server. + * Comma separated URL's that clients should use to communicate with the Authentication Server. + * Each URL should follow the format protocol://host:port. Leave empty to use the default URL + * generated by the Authentication Server. */ public static final String AUTH_SERVER_ANNOUNCE_URLS = "security.auth.server.announce.urls"; @@ -1546,8 +1533,7 @@ public static final class Security { */ public static final String AUTH_SERVER_BIND_PORT = "security.auth.server.bind.port"; /** - * Maximum number of handler threads for the Authentication Server embedded - * Jetty instance. + * Maximum number of handler threads for the Authentication Server embedded Jetty instance. */ public static final String MAX_THREADS = "security.server.maxthreads"; /** @@ -1580,8 +1566,8 @@ public static final class Security { */ public static final String BASIC_REALM_FILE = "security.authentication.basic.realmfile"; /** - * Configuration for specifying keytab location. The location will contain - * ${name} which will be replaced by the user/owner of the entities name. + * Configuration for specifying keytab location. The location will contain ${name} which will be + * replaced by the user/owner of the entities name. */ public static final String KEYTAB_PATH = "security.keytab.path"; @@ -1605,6 +1591,7 @@ public static final class Security { /** * App Fabric. */ + @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public static final class SSL { /** @@ -1632,24 +1619,22 @@ public static final class SSL { public static final class Authentication { /** - * Determines which authentication mode to use. Should be chosen from the - * {@link io.cdap.cdap.security.auth.AuthenticationMode} enum. + * Determines which authentication mode to use. Should be chosen from the {@link + * io.cdap.cdap.security.auth.AuthenticationMode} enum. */ public static final String MODE = "security.authentication.mode"; /** - * The header from which CDAP should expect to receive the end user - * identity when using proxy auth mode. + * The header from which CDAP should expect to receive the end user identity when using proxy + * auth mode. */ public static final String PROXY_USER_ID_HEADER = "security.authentication.proxy.user.identity.header"; /** - * Determines whether to propagate the end user credential as part of the - * Principal. + * Determines whether to propagate the end user credential as part of the Principal. */ public static final String PROPAGATE_USER_CREDENTIAL = "security.authentication.propagate.user.credentials"; /** - * Enable encryption for user credential in http auth header. Set in - * cdap-security.xml. + * Enable encryption for user credential in http auth header. Set in cdap-security.xml. */ public static final String USER_CREDENTIAL_ENCRYPTION_ENABLED = "security.authentication.user.credential.encryption.enabled"; @@ -1659,14 +1644,13 @@ public static final class Authentication { public static final String USER_CREDENTIAL_ENCRYPTION_KEYSET = "security.authentication.user.credentials.encryption.keyset"; /** - * {@link CConfiguration} property to pass runtime token from driver to - * distributed jobs. + * {@link CConfiguration} property to pass runtime token from driver to distributed jobs. */ public static final String RUNTIME_TOKEN = "security.authentication.runtime.token"; /** - * File name to use to pass - * {@link Constants.Security.Headers#RUNTIME_TOKEN} to execution job. + * File name to use to pass {@link Constants.Security.Headers#RUNTIME_TOKEN} to execution + * job. */ public static final String RUNTIME_TOKEN_FILE = "cdap.runtime.token"; /** @@ -1710,8 +1694,8 @@ public static final class Authorization { */ public static final int VISIBLE_BATCH_SIZE = 500; /** - * Upper limit on extension operation time after which the time is logged - * as WARN rather than TRACE. + * Upper limit on extension operation time after which the time is logged as WARN rather than + * TRACE. */ public static final String EXTENSION_OPERATION_TIME_WARN_THRESHOLD = "security.authorization.extension.operation.time.warn.threshold.ms"; @@ -1762,8 +1746,7 @@ public static final class Headers { */ public static final String USER_IP = "CDAP-UserIP"; /** - * User principal passed from program container to cdap service - * containers. + * User principal passed from program container to cdap service containers. */ public static final String USER_PRINCIPAL = "CDAP-User-Principal"; /** @@ -1794,8 +1777,7 @@ public static final class Router { */ public static final String SSL_KEYSTORE_PASSWORD = "router.ssl.keystore.password"; /** - * Paths to exclude from authentication, given by a single regular - * expression. + * Paths to exclude from authentication, given by a single regular expression. */ public static final String BYPASS_AUTHENTICATION_REGEX = "router.bypass.auth.regex"; @@ -1913,8 +1895,8 @@ public enum InMemoryPersistenceType { public static final int DEFAULT_DATA_LEVELDB_COMPACTION_LEVEL_MAX = 4; /** - * LevelDB substracts 10 from maxOpenFiles configuration to calculate table - * cache size. This constant allows us to convert it back + * LevelDB substracts 10 from maxOpenFiles configuration to calculate table cache size. This + * constant allows us to convert it back * * @see org.iq80.leveldb.impl.DbImpl#DbImpl */ @@ -2147,6 +2129,8 @@ public static final class Retry { /** * Constants for HBase DDL executor. */ + + @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public static final class HBaseDDLExecutor { public static final String EXTENSIONS_DIR = "hbase.ddlexecutor.extension.dir"; @@ -2201,8 +2185,7 @@ public static final class Capability { */ public static final String CONFIG_DIR = "capability.config.dir"; /** - * Number of executor threads used to auto install resources when a - * capability is enabled. + * Number of executor threads used to auto install resources when a capability is enabled. */ public static final String AUTO_INSTALL_THREADS = "capability.autoinstall.threads"; } @@ -2258,14 +2241,12 @@ public static final class Security { public static final String IDENTITY_SYSTEM = "twill.security.identity.system"; /** - * The secret name for the cdap-security.xml disk mount for master - * services. + * The secret name for the cdap-security.xml disk mount for master services. */ public static final String MASTER_SECRET_DISK_NAME = "twill.security.master.secret.disk.name"; /** - * The secret path for the cdap-security.xml disk mount for master - * services. + * The secret path for the cdap-security.xml disk mount for master services. */ public static final String MASTER_SECRET_DISK_PATH = "twill.security.master.secret.disk.path"; @@ -2275,14 +2256,14 @@ public static final class Security { public static final String WORKER_MOUNT_SECRET = "twill.security.worker.mount.secret"; /** - * The secret name for the cdap-security.xml disk mount for worker - * services including preview and task workers. + * The secret name for the cdap-security.xml disk mount for worker services including preview + * and task workers. */ public static final String WORKER_SECRET_DISK_NAME = "twill.security.worker.secret.disk.name"; /** - * The secret path for the cdap-security.xml disk mount for worker - * services including preview and task workers. + * The secret path for the cdap-security.xml disk mount for worker services including preview + * and task workers. */ public static final String WORKER_SECRET_DISK_PATH = "twill.security.worker.secret.disk.path"; } @@ -2316,7 +2297,7 @@ public static final class SupportBundle { /** * JMX metrics collector config. */ - public static final class JMXMetricsCollector { + public static final class JmxMetricsCollector { public static final String POLL_INTERVAL_SECS = "jmx.metrics.collector.poll.interval.secs"; public static final String SERVER_PORT = "jmx.metrics.collector.server.port"; @@ -2343,20 +2324,18 @@ public static final class Tethering { public static final String CONNECTION_INTERVAL = "tethering.agent.connection.interval.secs"; /** - * Tethering connection is deemed down if we haven't heard from the peer - * within this timeout. + * Tethering connection is deemed down if we haven't heard from the peer within this timeout. */ public static final String CONNECTION_TIMEOUT_SECONDS = "tethering.connection.timeout.seconds"; public static final int DEFAULT_CONNECTION_TIMEOUT_SECONDS = 60; /** - * Specifies the timeout for establishing a connection with the tethering - * server. + * Specifies the timeout for establishing a connection with the tethering server. */ public static final String CLIENT_CONNECTION_TIMEOUT_MS = "tethering.client.connection.timeout.ms"; /** - * Specifies the timeout for reading data from the tethering server after - * the connection is established. + * Specifies the timeout for reading data from the tethering server after the connection is + * established. */ public static final String CLIENT_READ_TIMEOUT_MS = "tethering.client.read.timeout.ms"; @@ -2410,4 +2389,13 @@ public static final class SourceControlManagement { "source.control.repository.cleanup.interval.seconds"; public static final String REPOSITORY_TTL_SECONDS = "source.control.repository.ttl.seconds"; } + + /** + * Constants for credential provisioning. + */ + public static final class CredentialProvider { + + public static final String EXTENSIONS_DIR = "credential.provider.extensions.dir"; + public static final String SYSTEM_PROPERTY_PREFIX = "credential.provider.system.properties."; + } } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/guice/KafkaClientModule.java b/cdap-common/src/main/java/io/cdap/cdap/common/guice/KafkaClientModule.java index 656f178fc336..a90f9d361a51 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/guice/KafkaClientModule.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/guice/KafkaClientModule.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -13,6 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package io.cdap.cdap.common.guice; import com.google.common.util.concurrent.AbstractIdleService; @@ -50,7 +51,7 @@ /** * Guice module for {@link KafkaClient} and {@link KafkaClientService}. Requires bindings from - * {@link ConfigModule} and {@link ZKClientModule}. + * {@link ConfigModule} and {@link ZkClientModule}. */ public class KafkaClientModule extends PrivateModule { @@ -60,7 +61,7 @@ public class KafkaClientModule extends PrivateModule { protected void configure() { bind(ZKClientService.class) .annotatedWith(Names.named(KAFKA_ZK)) - .toProvider(ZKClientServiceProvider.class).in(Scopes.SINGLETON); + .toProvider(ZkClientServiceProvider.class).in(Scopes.SINGLETON); bind(KafkaClientService.class).to(DefaultKafkaClientService.class).in(Scopes.SINGLETON); bind(BrokerService.class).to(DefaultBrokerService.class).in(Scopes.SINGLETON); @@ -75,24 +76,24 @@ protected void configure() { * A {@link Provider} to provide {@link ZKClientService} used by {@link ZKKafkaClientService} and * {@link ZKBrokerService}. */ - private static final class ZKClientServiceProvider implements Provider { + private static final class ZkClientServiceProvider implements Provider { private final CConfiguration cConf; private final Injector injector; @Inject - ZKClientServiceProvider(CConfiguration cConf, Injector injector) { + ZkClientServiceProvider(CConfiguration cConf, Injector injector) { this.cConf = cConf; this.injector = injector; } @Override public ZKClientService get() { - String kafkaZKQuorum = cConf.get(KafkaConstants.ConfigKeys.ZOOKEEPER_QUORUM); + String kafkaZkQuorum = cConf.get(KafkaConstants.ConfigKeys.ZOOKEEPER_QUORUM); ZKClientService zkClientService; final AtomicInteger startedCount = new AtomicInteger(); - if (kafkaZKQuorum == null) { + if (kafkaZkQuorum == null) { // If there is no separate zookeeper quorum, use the shared ZKClientService. zkClientService = injector.getInstance(ZKClientService.class); @@ -113,7 +114,7 @@ public ZKClientService get() { zkClientService = ZKClientServices.delegate( ZKClients.reWatchOnExpire( ZKClients.retryOnFailure( - ZKClientService.Builder.of(kafkaZKQuorum) + ZKClientService.Builder.of(kafkaZkQuorum) .setSessionTimeout( cConf.getInt(Constants.Zookeeper.CFG_SESSION_TIMEOUT_MILLIS, Constants.Zookeeper.DEFAULT_SESSION_TIMEOUT_MILLIS)) @@ -152,13 +153,13 @@ public ListenableFuture stop() { * A {@link Service} wrapper that wraps a {@link Service} with a {@link ZKClientService} that will * get start/stop together. */ - private abstract static class AbstractServiceWithZKClient extends + private abstract static class AbstractServiceWithZkClient extends AbstractIdleService { private final ZKClientService zkClientService; private final T delegate; - AbstractServiceWithZKClient(ZKClientService zkClientService, T delegate) { + AbstractServiceWithZkClient(ZKClientService zkClientService, T delegate) { this.zkClientService = zkClientService; this.delegate = delegate; } @@ -202,7 +203,7 @@ protected T getDelegate() { * A {@link KafkaClientService} that bundles with a given {@link ZKClientService} for start/stop. */ private static final class DefaultKafkaClientService extends - AbstractServiceWithZKClient + AbstractServiceWithZkClient implements KafkaClientService { @Inject @@ -222,9 +223,9 @@ public KafkaConsumer getConsumer() { } /** - * A {@link BrokerService} that bundles with a given {@link ZKClientService} for start/stop + * A {@link BrokerService} that bundles with a given {@link ZKClientService} for start/stop. */ - private static final class DefaultBrokerService extends AbstractServiceWithZKClient + private static final class DefaultBrokerService extends AbstractServiceWithZkClient implements BrokerService { @Inject diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/guice/ZKClientModule.java b/cdap-common/src/main/java/io/cdap/cdap/common/guice/ZkClientModule.java similarity index 91% rename from cdap-common/src/main/java/io/cdap/cdap/common/guice/ZKClientModule.java rename to cdap-common/src/main/java/io/cdap/cdap/common/guice/ZkClientModule.java index 19501fd10c4b..9b2c7b2eb706 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/guice/ZKClientModule.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/guice/ZkClientModule.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -13,6 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package io.cdap.cdap.common.guice; import com.google.inject.AbstractModule; @@ -31,11 +32,11 @@ * Guice module for binding {@link ZKClient} and {@link ZKClientService}. Requires {@link * ConfigModule} bindings. */ -public class ZKClientModule extends AbstractModule { +public class ZkClientModule extends AbstractModule { @Override protected void configure() { - /** + /* * ZKClientService is provided by the provider method * {@link #provideZKClientService(io.cdap.cdap.common.conf.CConfiguration)}. */ @@ -44,11 +45,11 @@ protected void configure() { @Provides @Singleton - private ZKClientService provideZKClientService(CConfiguration cConf) { + private ZKClientService provideZkClientService(CConfiguration cConf) { return ZKClientServices.delegate( ZKClients.reWatchOnExpire( ZKClients.retryOnFailure( - ZKClientService.Builder.of(Constants.Zookeeper.getZKQuorum(cConf)) + ZKClientService.Builder.of(Constants.Zookeeper.getZkQuorum(cConf)) .setSessionTimeout(cConf.getInt(Constants.Zookeeper.CFG_SESSION_TIMEOUT_MILLIS, Constants.Zookeeper.DEFAULT_SESSION_TIMEOUT_MILLIS)) .build(), diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/guice/ZKDiscoveryModule.java b/cdap-common/src/main/java/io/cdap/cdap/common/guice/ZkDiscoveryModule.java similarity index 92% rename from cdap-common/src/main/java/io/cdap/cdap/common/guice/ZKDiscoveryModule.java rename to cdap-common/src/main/java/io/cdap/cdap/common/guice/ZkDiscoveryModule.java index 5b8e7109da13..2366c7120b38 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/guice/ZKDiscoveryModule.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/guice/ZkDiscoveryModule.java @@ -45,11 +45,11 @@ * The Guice module for providing bindings for {@link DiscoveryService} and {@link * DiscoveryServiceClient} that uses ZooKeeper as the service discovery mechanism. */ -public final class ZKDiscoveryModule extends PrivateModule { +public final class ZkDiscoveryModule extends PrivateModule { @Override protected void configure() { - bind(ZKDiscoveryService.class).toProvider(ZKDiscoveryServiceProvider.class) + bind(ZKDiscoveryService.class).toProvider(ZkDiscoveryServiceProvider.class) .in(Scopes.SINGLETON); bind(DiscoveryService.class).to(ZKDiscoveryService.class); @@ -62,12 +62,12 @@ protected void configure() { /** * A Guice Provider to provide instance of {@link ZKDiscoveryService}. */ - private static final class ZKDiscoveryServiceProvider implements Provider { + private static final class ZkDiscoveryServiceProvider implements Provider { private final ZKClient zkClient; @Inject - ZKDiscoveryServiceProvider(ZKClient zkClient) { + ZkDiscoveryServiceProvider(ZKClient zkClient) { this.zkClient = zkClient; } @@ -120,9 +120,9 @@ private CacheLoader createClientLoader() { return new CacheLoader() { @Override public ZKDiscoveryService load(String key) { - ProgramId programID = ServiceDiscoverable.getId(key); + ProgramId programId = ServiceDiscoverable.getId(key); String ns = String.format("%s/%s", twillNamespace, - TwillAppNames.toTwillAppName(programID)); + TwillAppNames.toTwillAppName(programId)); LOG.info("Create ZKDiscoveryClient for {}", ns); return new ZKDiscoveryService(ZKClients.namespace(zkClient, ns)); } diff --git a/cdap-common/src/main/resources/cdap-default.xml b/cdap-common/src/main/resources/cdap-default.xml index 57b7adecb104..39630b562987 100644 --- a/cdap-common/src/main/resources/cdap-default.xml +++ b/cdap-common/src/main/resources/cdap-default.xml @@ -5995,4 +5995,14 @@ + + + credential.provider.extensions.dir + /opt/cdap/master/ext/credentialproviders + + Semicolon-separated list of local directories that are scanned for CDAP + remote authenticator extensions. + + + diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/guice/KafkaClientModuleTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/guice/KafkaClientModuleTest.java index 80f14c6694d5..b41436c0b7bb 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/guice/KafkaClientModuleTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/guice/KafkaClientModuleTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017 Cask Data, Inc. + * Copyright © 2017-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -57,7 +57,7 @@ public class KafkaClientModuleTest { private InMemoryZKServer zkServer; private EmbeddedKafkaServer kafkaServer; - private String kafkaZKConnect; + private String kafkaZkConnect; @Before public void beforeTest() throws Exception { @@ -65,20 +65,20 @@ public void beforeTest() throws Exception { zkServer.startAndWait(); CConfiguration cConf = CConfiguration.create(); - String kafkaZKNamespace = cConf.get(KafkaConstants.ConfigKeys.ZOOKEEPER_NAMESPACE_CONFIG); - kafkaZKConnect = zkServer.getConnectionStr(); + String kafkaZkNamespace = cConf.get(KafkaConstants.ConfigKeys.ZOOKEEPER_NAMESPACE_CONFIG); + kafkaZkConnect = zkServer.getConnectionStr(); - if (kafkaZKNamespace != null) { + if (kafkaZkNamespace != null) { ZKClientService zkClient = new DefaultZKClientService(zkServer.getConnectionStr(), 2000, null, - ImmutableMultimap.of()); + ImmutableMultimap.of()); zkClient.startAndWait(); - zkClient.create("/" + kafkaZKNamespace, null, CreateMode.PERSISTENT); + zkClient.create("/" + kafkaZkNamespace, null, CreateMode.PERSISTENT); zkClient.stopAndWait(); - kafkaZKConnect += "/" + kafkaZKNamespace; + kafkaZkConnect += "/" + kafkaZkNamespace; } - kafkaServer = createKafkaServer(kafkaZKConnect, TEMP_FOLDER.newFolder()); + kafkaServer = createKafkaServer(kafkaZkConnect, TEMP_FOLDER.newFolder()); kafkaServer.startAndWait(); } @@ -89,21 +89,21 @@ public void afterTest() { } @Test - public void testWithSharedZKClient() throws Exception { + public void testWithSharedZkClient() throws Exception { CConfiguration cConf = CConfiguration.create(); cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr()); Injector injector = Guice.createInjector( - new ConfigModule(cConf), - new ZKClientModule(), - new KafkaClientModule() + new ConfigModule(cConf), + new ZkClientModule(), + new KafkaClientModule() ); // Get the shared zkclient and start it ZKClientService zkClientService = injector.getInstance(ZKClientService.class); zkClientService.startAndWait(); - int baseZKConns = getZKConnections(); + final int baseZkConns = getZkConnections(); KafkaClientService kafkaClientService = injector.getInstance(KafkaClientService.class); final BrokerService brokerService = injector.getInstance(BrokerService.class); @@ -116,7 +116,7 @@ public void testWithSharedZKClient() throws Exception { Assert.assertTrue(zkClientService.isRunning()); // It shouldn't increase the number of zk client connections - Assert.assertEquals(baseZKConns, getZKConnections()); + Assert.assertEquals(baseZkConns, getZkConnections()); // Make sure it is talking to Kafka. Tasks.waitFor(true, new Callable() { @@ -134,41 +134,41 @@ public Boolean call() throws Exception { Assert.assertTrue(zkClientService.isRunning()); // It still shouldn't increase the number of zk client connections - Assert.assertEquals(baseZKConns, getZKConnections()); + Assert.assertEquals(baseZkConns, getZkConnections()); zkClientService.stopAndWait(); } @Test - public void testWithDedicatedZKClient() throws Exception { + public void testWithDedicatedZkClient() throws Exception { CConfiguration cConf = CConfiguration.create(); cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr()); // Set the zk quorum for the kafka client, expects it to create and start/stop it's own zk client service - cConf.set(KafkaConstants.ConfigKeys.ZOOKEEPER_QUORUM, kafkaZKConnect); + cConf.set(KafkaConstants.ConfigKeys.ZOOKEEPER_QUORUM, kafkaZkConnect); Injector injector = Guice.createInjector( - new ConfigModule(cConf), - new ZKClientModule(), - new KafkaClientModule() + new ConfigModule(cConf), + new ZkClientModule(), + new KafkaClientModule() ); // Get the shared zkclient and start it ZKClientService zkClientService = injector.getInstance(ZKClientService.class); zkClientService.startAndWait(); - int baseZKConns = getZKConnections(); + int baseZkConns = getZkConnections(); KafkaClientService kafkaClientService = injector.getInstance(KafkaClientService.class); final BrokerService brokerService = injector.getInstance(BrokerService.class); // Start the kafka client, it should increase the zk connections by 1 kafkaClientService.startAndWait(); - Assert.assertEquals(baseZKConns + 1, getZKConnections()); + Assert.assertEquals(baseZkConns + 1, getZkConnections()); // Start the broker service, // it shouldn't affect the zk connections, as it share the same zk client with kafka client brokerService.startAndWait(); - Assert.assertEquals(baseZKConns + 1, getZKConnections()); + Assert.assertEquals(baseZkConns + 1, getZkConnections()); // Make sure it is talking to Kafka. Tasks.waitFor(true, new Callable() { @@ -178,17 +178,16 @@ public Boolean call() throws Exception { } }, 5L, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); - // Shouldn't affect the shared zk client state Assert.assertTrue(zkClientService.isRunning()); // Stop the broker service, it shouldn't affect the zk connections, as it is still used by the kafka client brokerService.stopAndWait(); - Assert.assertEquals(baseZKConns + 1, getZKConnections()); + Assert.assertEquals(baseZkConns + 1, getZkConnections()); // Stop the kafka client, the zk connections should be reduced by 1 kafkaClientService.stopAndWait(); - Assert.assertEquals(baseZKConns, getZKConnections()); + Assert.assertEquals(baseZkConns, getZkConnections()); // Still shouldn't affect the shared zk client Assert.assertTrue(zkClientService.isRunning()); @@ -199,12 +198,12 @@ public Boolean call() throws Exception { /** * Returns the number of client connections to the zk server */ - private int getZKConnections() throws IOException { + private int getZkConnections() throws IOException { InetSocketAddress zkAddr = zkServer.getLocalAddress(); try (Socket socket = new Socket(zkAddr.getAddress(), zkAddr.getPort())) { socket.getOutputStream().write("cons".getBytes(StandardCharsets.ISO_8859_1)); BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), - StandardCharsets.ISO_8859_1)); + StandardCharsets.ISO_8859_1)); String line = reader.readLine(); int count = 0; while (line != null) { diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/guice/ZKDiscoveryModuleTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/guice/ZkDiscoveryModuleTest.java similarity index 77% rename from cdap-common/src/test/java/io/cdap/cdap/common/guice/ZKDiscoveryModuleTest.java rename to cdap-common/src/test/java/io/cdap/cdap/common/guice/ZkDiscoveryModuleTest.java index 56fff8a5b054..c809a29c2824 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/guice/ZKDiscoveryModuleTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/guice/ZkDiscoveryModuleTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2018 Cask Data, Inc. + * Copyright © 2018-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -46,9 +46,9 @@ import org.junit.rules.TemporaryFolder; /** - * Unit test for the {@link ZKDiscoveryModule}. + * Unit test for the {@link ZkDiscoveryModule}. */ -public class ZKDiscoveryModuleTest { +public class ZkDiscoveryModuleTest { @ClassRule public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); @@ -74,26 +74,29 @@ public static void finish() { @Test public void testMasterDiscovery() { Injector injector = Guice.createInjector( - new ConfigModule(cConf), - new ZKClientModule(), - new ZKDiscoveryModule() + new ConfigModule(cConf), + new ZkClientModule(), + new ZkDiscoveryModule() ); ZKClientService zkClient = injector.getInstance(ZKClientService.class); zkClient.startAndWait(); try { DiscoveryService discoveryService = injector.getInstance(DiscoveryService.class); - DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); + DiscoveryServiceClient discoveryServiceClient = injector + .getInstance(DiscoveryServiceClient.class); // Register a master service InetSocketAddress socketAddr = new InetSocketAddress(InetAddress.getLoopbackAddress(), 43210); - Cancellable cancellable = discoveryService.register(new Discoverable(Constants.Service.APP_FABRIC_HTTP, - socketAddr)); + Cancellable cancellable = discoveryService + .register(new Discoverable(Constants.Service.APP_FABRIC_HTTP, + socketAddr)); try { // Discover the master service Discoverable discoverable = new RandomEndpointStrategy( - () -> discoveryServiceClient.discover(Constants.Service.APP_FABRIC_HTTP)).pick(10, TimeUnit.SECONDS); + () -> discoveryServiceClient.discover(Constants.Service.APP_FABRIC_HTTP)) + .pick(10, TimeUnit.SECONDS); Assert.assertNotNull(discoverable); Assert.assertEquals(Constants.Service.APP_FABRIC_HTTP, discoverable.getName()); @@ -110,9 +113,9 @@ public void testMasterDiscovery() { @Test public void testProgramDiscovery() { Injector injector = Guice.createInjector( - new ConfigModule(cConf), - new ZKClientModule(), - new ZKDiscoveryModule() + new ConfigModule(cConf), + new ZkClientModule(), + new ZkDiscoveryModule() ); ZKClientService zkClient = injector.getInstance(ZKClientService.class); @@ -120,20 +123,24 @@ public void testProgramDiscovery() { try { // Register a service using the twill ZKClient. This is to simulate how a user Service program register ProgramId programId = NamespaceId.DEFAULT.app("app").service("service"); - String twillNamespace = injector.getInstance(CConfiguration.class).get(Constants.CFG_TWILL_ZK_NAMESPACE); + String twillNamespace = injector.getInstance(CConfiguration.class) + .get(Constants.CFG_TWILL_ZK_NAMESPACE); String discoverableName = ServiceDiscoverable.getName(programId); - ZKClient twillZKClient = ZKClients.namespace(zkClient, - twillNamespace + "/" + TwillAppNames.toTwillAppName(programId)); - - try (ZKDiscoveryService twillDiscoveryService = new ZKDiscoveryService(twillZKClient)) { - InetSocketAddress socketAddr = new InetSocketAddress(InetAddress.getLoopbackAddress(), 43210); - Cancellable cancellable = twillDiscoveryService.register(new Discoverable(discoverableName, socketAddr)); + ZKClient twillZkClient = ZKClients.namespace(zkClient, + twillNamespace + "/" + TwillAppNames.toTwillAppName(programId)); + + try (ZKDiscoveryService twillDiscoveryService = new ZKDiscoveryService(twillZkClient)) { + InetSocketAddress socketAddr = new InetSocketAddress(InetAddress.getLoopbackAddress(), + 43210); + Cancellable cancellable = twillDiscoveryService + .register(new Discoverable(discoverableName, socketAddr)); try { // Discover the user service - DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); + DiscoveryServiceClient discoveryServiceClient = injector + .getInstance(DiscoveryServiceClient.class); Discoverable discoverable = new RandomEndpointStrategy( - () -> discoveryServiceClient.discover(discoverableName)).pick(10, TimeUnit.SECONDS); + () -> discoveryServiceClient.discover(discoverableName)).pick(10, TimeUnit.SECONDS); Assert.assertNotNull(discoverable); diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorTest.java index 037b2d5e954a..b7813fb679bc 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/zookeeper/coordination/ResourceCoordinatorTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2018 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -13,6 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package io.cdap.cdap.common.zookeeper.coordination; import com.google.inject.Guice; @@ -21,8 +22,8 @@ import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.discovery.ResolvingDiscoverable; import io.cdap.cdap.common.guice.ConfigModule; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -67,16 +68,16 @@ public void testAssignment() throws InterruptedException, ExecutionException { String serviceName = "test-assignment"; Injector injector = Guice.createInjector(new ConfigModule(cConf), - new ZKClientModule(), - new ZKDiscoveryModule()); + new ZkClientModule(), + new ZkDiscoveryModule()); ZKClientService zkClient = injector.getInstance(ZKClientService.class); zkClient.startAndWait(); DiscoveryService discoveryService = injector.getInstance(DiscoveryService.class); try { ResourceCoordinator coordinator = new ResourceCoordinator(zkClient, - injector.getInstance(DiscoveryServiceClient.class), - new BalancedAssignmentStrategy()); + injector.getInstance(DiscoveryServiceClient.class), + new BalancedAssignmentStrategy()); coordinator.startAndWait(); try { @@ -85,7 +86,8 @@ public void testAssignment() throws InterruptedException, ExecutionException { try { // Create a requirement - ResourceRequirement requirement = ResourceRequirement.builder(serviceName).addPartitions("p", 5, 1).build(); + ResourceRequirement requirement = ResourceRequirement.builder(serviceName) + .addPartitions("p", 5, 1).build(); client.submitRequirement(requirement).get(); // Fetch the requirement, just to verify it's the same as the one get submitted. @@ -93,13 +95,15 @@ public void testAssignment() throws InterruptedException, ExecutionException { // Register a discovery endpoint final Discoverable discoverable1 = createDiscoverable(serviceName, 10000); - Cancellable cancelDiscoverable1 = discoveryService.register(ResolvingDiscoverable.of(discoverable1)); + Cancellable cancelDiscoverable1 = discoveryService + .register(ResolvingDiscoverable.of(discoverable1)); // Add a change handler for this discoverable. final BlockingQueue> assignmentQueue = - new SynchronousQueue<>(); + new SynchronousQueue<>(); final Semaphore finishSemaphore = new Semaphore(0); - Cancellable cancelSubscribe1 = subscribe(client, discoverable1, assignmentQueue, finishSemaphore); + final Cancellable cancelSubscribe1 = subscribe(client, discoverable1, assignmentQueue, + finishSemaphore); // Assert that it received the changes. Collection assigned = assignmentQueue.poll(30, TimeUnit.SECONDS); @@ -118,7 +122,8 @@ public void testAssignment() throws InterruptedException, ExecutionException { // Register another discoverable final Discoverable discoverable2 = createDiscoverable(serviceName, 10001); - Cancellable cancelDiscoverable2 = discoveryService.register(ResolvingDiscoverable.of(discoverable2)); + final Cancellable cancelDiscoverable2 = discoveryService + .register(ResolvingDiscoverable.of(discoverable2)); // Changes should be received by the handler, with only 3 resources, // as 2 out of 5 should get moved to the new discoverable. @@ -138,7 +143,8 @@ public void testAssignment() throws InterruptedException, ExecutionException { // Subscribe to changes for the second discoverable, // it should see the latest assignment, even though no new fetch from ZK is triggered. - Cancellable cancelSubscribe2 = subscribe(client, discoverable2, assignmentQueue, finishSemaphore); + final Cancellable cancelSubscribe2 = subscribe(client, discoverable2, assignmentQueue, + finishSemaphore); assigned = assignmentQueue.poll(30, TimeUnit.SECONDS); Assert.assertNotNull(assigned); Assert.assertEquals(5, assigned.size()); @@ -148,7 +154,8 @@ public void testAssignment() throws InterruptedException, ExecutionException { Assert.assertTrue(assignmentQueue.poll(30, TimeUnit.SECONDS).isEmpty()); // Update the requirement to have one partition, the handler should receive one resource - client.submitRequirement(ResourceRequirement.builder(serviceName).addPartitions("p", 1, 1).build()); + client.submitRequirement( + ResourceRequirement.builder(serviceName).addPartitions("p", 1, 1).build()); assigned = assignmentQueue.poll(30, TimeUnit.SECONDS); Assert.assertNotNull(assigned); Assert.assertEquals(1, assigned.size()); @@ -187,15 +194,15 @@ public static void finish() { } private Cancellable subscribe(ResourceCoordinatorClient client, - final Discoverable discoverable, - final BlockingQueue> assignmentQueue, - final Semaphore finishSemaphore) { + final Discoverable discoverable, + final BlockingQueue> assignmentQueue, + final Semaphore finishSemaphore) { return client.subscribe(discoverable.getName(), new ResourceHandler(discoverable) { @Override public void onChange(Collection partitionReplicas) { try { LOG.debug("Discoverable {} Received: {}", - discoverable.getSocketAddress().getPort(), partitionReplicas); + discoverable.getSocketAddress().getPort(), partitionReplicas); assignmentQueue.put(partitionReplicas); } catch (InterruptedException e) { LOG.error("Interrupted.", e); @@ -208,7 +215,8 @@ public void finished(Throwable failureCause) { if (failureCause == null) { finishSemaphore.release(); } else { - LOG.error("Finished with failure for {}", discoverable.getSocketAddress().getPort(), failureCause); + LOG.error("Finished with failure for {}", discoverable.getSocketAddress().getPort(), + failureCause); } } }); diff --git a/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/hbase/HBaseMetricsTableTest.java b/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/hbase/HBaseMetricsTableTest.java index 994ccec40132..cdee6b61e188 100644 --- a/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/hbase/HBaseMetricsTableTest.java +++ b/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/dataset2/lib/table/hbase/HBaseMetricsTableTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2018 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -31,8 +31,8 @@ import io.cdap.cdap.common.guice.ConfigModule; import io.cdap.cdap.common.guice.DFSLocationModule; import io.cdap.cdap.common.guice.NamespaceAdminTestModule; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.data.hbase.HBaseTestBase; import io.cdap.cdap.data.hbase.HBaseTestFactory; import io.cdap.cdap.data.runtime.DataFabricModules; @@ -68,6 +68,7 @@ /** * metrics table test for HBase. */ +@SuppressWarnings("checkstyle:AbbreviationAsWordInName") @Category(SlowTests.class) public class HBaseMetricsTableTest extends MetricsTableTest { @@ -83,35 +84,36 @@ public static void setup() throws Exception { CConfiguration cConf = CConfiguration.create(); cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name")); Injector injector = Guice.createInjector(new DataFabricModules().getDistributedModules(), - new ConfigModule(cConf, TEST_HBASE.getConfiguration()), - new ZKClientModule(), - new ZKDiscoveryModule(), - new TransactionMetricsModule(), - new DFSLocationModule(), - new NamespaceAdminTestModule(), - new SystemDatasetRuntimeModule().getDistributedModules(), - new DataSetsModules().getInMemoryModules(), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getInMemoryModules(), - new AuthenticationContextModules().getNoOpModule(), - new AbstractModule() { - @Override - protected void configure() { - bind(UGIProvider.class).to(UnsupportedUGIProvider.class); - bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); - } - }); + new ConfigModule(cConf, TEST_HBASE.getConfiguration()), + new ZkClientModule(), + new ZkDiscoveryModule(), + new TransactionMetricsModule(), + new DFSLocationModule(), + new NamespaceAdminTestModule(), + new SystemDatasetRuntimeModule().getDistributedModules(), + new DataSetsModules().getInMemoryModules(), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getInMemoryModules(), + new AuthenticationContextModules().getNoOpModule(), + new AbstractModule() { + @Override + protected void configure() { + bind(UGIProvider.class).to(UnsupportedUGIProvider.class); + bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); + } + }); dsFramework = injector.getInstance(DatasetFramework.class); tableUtil = injector.getInstance(HBaseTableUtil.class); - ddlExecutor = new HBaseDDLExecutorFactory(cConf, TEST_HBASE.getHBaseAdmin().getConfiguration()).get(); + ddlExecutor = new HBaseDDLExecutorFactory(cConf, TEST_HBASE.getHBaseAdmin().getConfiguration()) + .get(); ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM)); } @AfterClass public static void tearDown() throws Exception { tableUtil.deleteAllInNamespace(ddlExecutor, tableUtil.getHBaseNamespace(NamespaceId.SYSTEM), - TEST_HBASE.getHBaseAdmin().getConfiguration()); + TEST_HBASE.getHBaseAdmin().getConfiguration()); ddlExecutor.deleteNamespaceIfExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM)); } @@ -127,10 +129,10 @@ public void testConcurrentIncrement() throws Exception { // HBaseMetricsTable does not support mixed increment and incrementAndGet so the // updates and assertions here are different from MetricsTableTest.testConcurrentIncrement() Collection threads = - ImmutableList.of(new IncThread(getTable(testConcurrentIncrement), A, inc1, rounds), - new IncThread(getTable(testConcurrentIncrement), A, inc2, rounds), - new IncAndGetThread(getTable(testConcurrentIncrement), A, R, 5, rounds), - new IncAndGetThread(getTable(testConcurrentIncrement), A, R, 2, rounds)); + ImmutableList.of(new IncThread(getTable(testConcurrentIncrement), A, inc1, rounds), + new IncThread(getTable(testConcurrentIncrement), A, inc2, rounds), + new IncAndGetThread(getTable(testConcurrentIncrement), A, R, 5, rounds), + new IncAndGetThread(getTable(testConcurrentIncrement), A, R, 2, rounds)); for (Thread t : threads) { t.start(); } @@ -149,12 +151,13 @@ public void testConcurrentIncrement() throws Exception { private DatasetId getDatasetId(String tableNamePrefix) { return NamespaceId.SYSTEM.dataset(tableNamePrefix + "v3"); } + @Override protected MetricsTable getTable(String name) throws Exception { // add v3 so that all the tests are performed for v3 table DatasetId metricsDatasetInstanceId = getDatasetId(name); DatasetProperties props = TableProperties.builder().setReadlessIncrementSupport(true).build(); return DatasetsUtil.getOrCreateDataset(dsFramework, metricsDatasetInstanceId, - MetricsTable.class.getName(), props, null); + MetricsTable.class.getName(), props, null); } } diff --git a/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceClientTest.java b/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceClientTest.java index f24a3ca9b0a5..027a57f135d5 100644 --- a/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceClientTest.java +++ b/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceClientTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2022 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -25,8 +25,8 @@ import io.cdap.cdap.common.guice.ConfigModule; import io.cdap.cdap.common.guice.NonCustomLocationUnitTestModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; import io.cdap.cdap.common.namespace.SimpleNamespaceQueryAdmin; import io.cdap.cdap.common.utils.Networks; @@ -68,6 +68,7 @@ * HBase queue tests. */ public class TransactionServiceClientTest extends TransactionSystemTest { + @ClassRule public static TemporaryFolder tmpFolder = new TemporaryFolder(); @@ -109,41 +110,42 @@ public static void beforeClass() throws Exception { // getCommonConfiguration() sets up an hConf with tx service configuration. // however, createTxService() will override these with defaults from the CConf. // hence, we must pass in these settings when creating the tx service. - Configuration extraCConf = new Configuration(); - extraCConf.clear(); - extraCConf = getCommonConfiguration(extraCConf); - for (Map.Entry entry : extraCConf) { + Configuration extraCconf = new Configuration(); + extraCconf.clear(); + extraCconf = getCommonConfiguration(extraCconf); + for (Map.Entry entry : extraCconf) { cConf.set(entry.getKey(), entry.getValue()); } - server = TransactionServiceTest.createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), - hConf, tmpFolder.newFolder(), cConf); + server = TransactionServiceTest + .createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), + hConf, tmpFolder.newFolder(), cConf); server.startAndWait(); injector = Guice.createInjector( - new ConfigModule(cConf, hConf), - RemoteAuthenticatorModules.getNoOpModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), - new NonCustomLocationUnitTestModule(), - new TransactionMetricsModule(), - new DataFabricModules().getDistributedModules(), - new AbstractModule() { - @Override - protected void configure() { - bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class); - bind(UGIProvider.class).to(UnsupportedUGIProvider.class); - bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); - } - }, - Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() { - @Override - protected void configure() { - bind(MetadataStorage.class).to(NoopMetadataStorage.class); - } - }), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getInMemoryModules(), - new AuthenticationContextModules().getNoOpModule()); + new ConfigModule(cConf, hConf), + RemoteAuthenticatorModules.getNoOpModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), + new NonCustomLocationUnitTestModule(), + new TransactionMetricsModule(), + new DataFabricModules().getDistributedModules(), + new AbstractModule() { + @Override + protected void configure() { + bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class); + bind(UGIProvider.class).to(UnsupportedUGIProvider.class); + bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); + } + }, + Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() { + @Override + protected void configure() { + bind(MetadataStorage.class).to(NoopMetadataStorage.class); + } + }), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getInMemoryModules(), + new AuthenticationContextModules().getNoOpModule()); zkClient = injector.getInstance(ZKClientService.class); zkClient.startAndWait(); @@ -176,7 +178,8 @@ public void resetState() { @Test public void testGetSnapshot() throws Exception { TransactionSystemClient client = getClient(); - SnapshotCodecProvider codecProvider = new SnapshotCodecProvider(injector.getInstance(Configuration.class)); + SnapshotCodecProvider codecProvider = new SnapshotCodecProvider( + injector.getInstance(Configuration.class)); Transaction tx1 = client.startShort(); long currentTime = System.currentTimeMillis(); diff --git a/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceTest.java b/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceTest.java index 9a24101e2fe4..ee712eefb298 100644 --- a/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceTest.java +++ b/cdap-data-fabric-tests/src/test/java/io/cdap/cdap/data2/transaction/distributed/TransactionServiceTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2022 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -28,8 +28,8 @@ import io.cdap.cdap.common.guice.ConfigModule; import io.cdap.cdap.common.guice.NonCustomLocationUnitTestModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; import io.cdap.cdap.common.namespace.SimpleNamespaceQueryAdmin; import io.cdap.cdap.common.utils.Networks; @@ -74,6 +74,7 @@ * Testing HA for {@link TransactionService}. */ public class TransactionServiceTest { + @ClassRule public static TemporaryFolder tmpFolder = new TemporaryFolder(); @@ -107,7 +108,7 @@ public void after() throws Exception { } @Test(timeout = 30000) - public void testHA() throws Exception { + public void testHighAvailability() throws Exception { // NOTE: we play with blocking/nonblocking a lot below // as until we integrate with "leader election" stuff, service blocks on start if it is not a leader @@ -120,30 +121,30 @@ public void testHA() throws Exception { cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath()); Injector injector = Guice.createInjector( - new ConfigModule(cConf), - RemoteAuthenticatorModules.getNoOpModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), - new NonCustomLocationUnitTestModule(), - new TransactionMetricsModule(), - new AbstractModule() { - @Override - protected void configure() { - bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class); - bind(UGIProvider.class).to(UnsupportedUGIProvider.class); - bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); - } - }, - new DataFabricModules().getDistributedModules(), - Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() { - @Override - protected void configure() { - bind(MetadataStorage.class).to(NoopMetadataStorage.class); - } - }), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getInMemoryModules(), - new AuthenticationContextModules().getNoOpModule() + new ConfigModule(cConf), + RemoteAuthenticatorModules.getNoOpModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), + new NonCustomLocationUnitTestModule(), + new TransactionMetricsModule(), + new AbstractModule() { + @Override + protected void configure() { + bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class); + bind(UGIProvider.class).to(UnsupportedUGIProvider.class); + bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); + } + }, + new DataFabricModules().getDistributedModules(), + Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() { + @Override + protected void configure() { + bind(MetadataStorage.class).to(NoopMetadataStorage.class); + } + }), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getInMemoryModules(), + new AuthenticationContextModules().getNoOpModule() ); ZKClientService zkClient = injector.getInstance(ZKClientService.class); @@ -156,18 +157,20 @@ protected void configure() { TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class); TransactionExecutor txExecutor = new DefaultTransactionExecutor(txClient, - ImmutableList.of((TransactionAware) table)); + ImmutableList.of((TransactionAware) table)); // starting tx service, tx client can pick it up - TransactionService first = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), - hConf, tmpFolder.newFolder()); + TransactionService first = createTxService(zkServer.getConnectionStr(), + Networks.getRandomPort(), + hConf, tmpFolder.newFolder()); first.startAndWait(); Assert.assertNotNull(txClient.startShort()); verifyGetAndPut(table, txExecutor, null, "val1"); // starting another tx service should not hurt - TransactionService second = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), - hConf, tmpFolder.newFolder()); + TransactionService second = createTxService(zkServer.getConnectionStr(), + Networks.getRandomPort(), + hConf, tmpFolder.newFolder()); // NOTE: we don't have to wait for start as client should pick it up anyways, but we do wait to ensure // the case with two active is handled well second.startAndWait(); @@ -184,8 +187,9 @@ protected void configure() { verifyGetAndPut(table, txExecutor, "val2", "val3"); // doing same trick again to failover to the third one - TransactionService third = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), - hConf, tmpFolder.newFolder()); + TransactionService third = createTxService(zkServer.getConnectionStr(), + Networks.getRandomPort(), + hConf, tmpFolder.newFolder()); // NOTE: we don't have to wait for start as client should pick it up anyways third.start(); // stopping second one @@ -207,8 +211,8 @@ protected void configure() { } private void verifyGetAndPut(final Table table, TransactionExecutor txExecutor, - final String verifyGet, final String toPut) - throws TransactionFailureException, InterruptedException { + final String verifyGet, final String toPut) + throws TransactionFailureException, InterruptedException { txExecutor.execute(() -> { byte[] existing = table.get(Bytes.toBytes("row"), Bytes.toBytes("col")); @@ -228,44 +232,44 @@ private void dropTable(String tableName) { } private static TransactionService createTxService(String zkConnectionString, int txServicePort, - Configuration hConf, final File outPath) { + Configuration hConf, final File outPath) { return createTxService(zkConnectionString, txServicePort, hConf, outPath, null); } static TransactionService createTxService(String zkConnectionString, int txServicePort, - Configuration hConf, final File outPath, - @Nullable CConfiguration cConfig) { + Configuration hConf, final File outPath, + @Nullable CConfiguration cConfig) { final CConfiguration cConf = cConfig == null ? CConfiguration.create() : cConfig; // tests should use the current user for HDFS cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name")); cConf.set(Constants.Zookeeper.QUORUM, zkConnectionString); cConf.set(Constants.CFG_LOCAL_DATA_DIR, outPath.getAbsolutePath()); cConf.set(TxConstants.Service.CFG_DATA_TX_BIND_PORT, - Integer.toString(txServicePort)); + Integer.toString(txServicePort)); // we want persisting for this test cConf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, true); cConf.setBoolean(TxConstants.TransactionPruning.PRUNE_ENABLE, false); final Injector injector = - Guice.createInjector(new ConfigModule(cConf, hConf), - new NonCustomLocationUnitTestModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), - new TransactionMetricsModule(), - new AbstractModule() { - @Override - protected void configure() { - bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class); - bind(UGIProvider.class).to(UnsupportedUGIProvider.class); - bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); - } - }, - new DataFabricModules().getDistributedModules(), - new SystemDatasetRuntimeModule().getInMemoryModules(), - new DataSetsModules().getInMemoryModules(), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getInMemoryModules(), - new AuthenticationContextModules().getNoOpModule()); + Guice.createInjector(new ConfigModule(cConf, hConf), + new NonCustomLocationUnitTestModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), + new TransactionMetricsModule(), + new AbstractModule() { + @Override + protected void configure() { + bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class); + bind(UGIProvider.class).to(UnsupportedUGIProvider.class); + bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); + } + }, + new DataFabricModules().getDistributedModules(), + new SystemDatasetRuntimeModule().getInMemoryModules(), + new DataSetsModules().getInMemoryModules(), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getInMemoryModules(), + new AuthenticationContextModules().getNoOpModule()); injector.getInstance(ZKClientService.class).startAndWait(); return injector.getInstance(TransactionService.class); diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetOpExecutorServiceTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetOpExecutorServiceTest.java index b615a2089273..7558ede43a0b 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetOpExecutorServiceTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/datafabric/dataset/service/executor/DatasetOpExecutorServiceTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2022 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -40,7 +40,7 @@ import io.cdap.cdap.common.guice.NamespaceAdminTestModule; import io.cdap.cdap.common.guice.NonCustomLocationUnitTestModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; +import io.cdap.cdap.common.guice.ZkClientModule; import io.cdap.cdap.common.http.DefaultHttpRequestConfig; import io.cdap.cdap.common.namespace.NamespaceAdmin; import io.cdap.cdap.common.utils.Networks; @@ -108,7 +108,6 @@ public class DatasetOpExecutorServiceTest { @Before public void setUp() throws Exception { - Configuration hConf = new Configuration(); CConfiguration cConf = CConfiguration.create(); File datasetDir = new File(TMP_FOLDER.newFolder(), "datasetUser"); @@ -120,30 +119,32 @@ public void setUp() throws Exception { cConf.set(Constants.Dataset.Executor.ADDRESS, "localhost"); cConf.setInt(Constants.Dataset.Executor.PORT, Networks.getRandomPort()); + Configuration hConf = new Configuration(); + Injector injector = Guice.createInjector( - new ConfigModule(cConf, hConf), - RemoteAuthenticatorModules.getNoOpModule(), - new IOModule(), - new ZKClientModule(), - new KafkaClientModule(), - new InMemoryDiscoveryModule(), - new NonCustomLocationUnitTestModule(), - new DataFabricModules().getInMemoryModules(), - new DataSetsModules().getStandaloneModules(), - new DataSetServiceModules().getInMemoryModules(), - new TransactionMetricsModule(), - new NamespaceAdminTestModule(), - new AuthenticationContextModules().getMasterModule(), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getInMemoryModules(), - new AbstractModule() { - @Override - protected void configure() { - bind(UGIProvider.class).to(UnsupportedUGIProvider.class); - bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); - bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class); - } - }); + new ConfigModule(cConf, hConf), + RemoteAuthenticatorModules.getNoOpModule(), + new IOModule(), + new ZkClientModule(), + new KafkaClientModule(), + new InMemoryDiscoveryModule(), + new NonCustomLocationUnitTestModule(), + new DataFabricModules().getInMemoryModules(), + new DataSetsModules().getStandaloneModules(), + new DataSetServiceModules().getInMemoryModules(), + new TransactionMetricsModule(), + new NamespaceAdminTestModule(), + new AuthenticationContextModules().getMasterModule(), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getInMemoryModules(), + new AbstractModule() { + @Override + protected void configure() { + bind(UGIProvider.class).to(UnsupportedUGIProvider.class); + bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); + bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class); + } + }); txManager = injector.getInstance(TransactionManager.class); txManager.startAndWait(); @@ -160,7 +161,8 @@ protected void configure() { // find host DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class); - endpointStrategy = new RandomEndpointStrategy(() -> discoveryClient.discover(Constants.Service.DATASET_MANAGER)); + endpointStrategy = new RandomEndpointStrategy( + () -> discoveryClient.discover(Constants.Service.DATASET_MANAGER)); namespaceAdmin = injector.getInstance(NamespaceAdmin.class); namespaceAdmin.create(NamespaceMeta.DEFAULT); @@ -196,14 +198,15 @@ public void testRest() throws Exception { final Table table = dsFramework.getDataset(bob, DatasetDefinition.NO_ARGUMENTS, null); Assert.assertNotNull(table); TransactionExecutor txExecutor = - new DefaultTransactionExecutor(new InMemoryTxSystemClient(txManager), - ImmutableList.of((TransactionAware) table)); + new DefaultTransactionExecutor(new InMemoryTxSystemClient(txManager), + ImmutableList.of((TransactionAware) table)); // writing smth to table txExecutor.execute(() -> table.put(new Put("key1", "col1", "val1"))); // verify that we can read the data - txExecutor.execute(() -> Assert.assertEquals("val1", table.get(new Get("key1", "col1")).getString("col1"))); + txExecutor.execute( + () -> Assert.assertEquals("val1", table.get(new Get("key1", "col1")).getString("col1"))); testAdminOp(bob, "truncate", 200, null); @@ -227,7 +230,8 @@ public void testUpdate() throws Exception { dsFramework.addInstance("table", bob, DatasetProperties.EMPTY); testAdminOp(bob, "exists", 200, true); - dsFramework.updateInstance(bob, DatasetProperties.builder().add("dataset.table.ttl", "10000").build()); + dsFramework + .updateInstance(bob, DatasetProperties.builder().add("dataset.table.ttl", "10000").build()); // check upgrade testAdminOp(bob, "upgrade", 200, null); @@ -237,21 +241,22 @@ public void testUpdate() throws Exception { } @SuppressWarnings("SameParameterValue") - private void testAdminOp(String instanceName, String opName, int expectedStatus, Object expectedResult) - throws IOException { + private void testAdminOp(String instanceName, String opName, int expectedStatus, + Object expectedResult) + throws IOException { testAdminOp(NamespaceId.DEFAULT.dataset(instanceName), opName, expectedStatus, - expectedResult); + expectedResult); } private void testAdminOp(DatasetId datasetInstanceId, String opName, int expectedStatus, - Object expectedResult) - throws IOException { + Object expectedResult) + throws IOException { String path = String.format("/namespaces/%s/data/datasets/%s/admin/%s", - datasetInstanceId.getNamespace(), datasetInstanceId.getEntityName(), opName); + datasetInstanceId.getNamespace(), datasetInstanceId.getEntityName(), opName); URL targetUrl = resolve(path); HttpResponse response = HttpRequests.execute(HttpRequest.post(targetUrl).build(), - new DefaultHttpRequestConfig(false)); + new DefaultHttpRequestConfig(false)); DatasetAdminOpResponse body = getResponse(response.getResponseBody()); Assert.assertEquals(expectedStatus, response.getResponseCode()); Assert.assertEquals(expectedResult, body.getResult()); @@ -259,11 +264,12 @@ private void testAdminOp(DatasetId datasetInstanceId, String opName, int expecte private URL resolve(String path) throws MalformedURLException { Discoverable discoverable = endpointStrategy.pick(1, TimeUnit.SECONDS); - return URIScheme.createURI(discoverable, "%s%s", Constants.Gateway.API_VERSION_3_TOKEN, path).toURL(); + return URIScheme.createURI(discoverable, "%s%s", Constants.Gateway.API_VERSION_3_TOKEN, path) + .toURL(); } private DatasetAdminOpResponse getResponse(byte[] body) { return Objects.firstNonNull(GSON.fromJson(Bytes.toString(body), DatasetAdminOpResponse.class), - new DatasetAdminOpResponse(null, null)); + new DatasetAdminOpResponse(null, null)); } } diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/kafka/KafkaTester.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/kafka/KafkaTester.java index 6995eefe6473..dfef63007771 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/kafka/KafkaTester.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/kafka/KafkaTester.java @@ -1,5 +1,5 @@ /* - * Copyright © 2016-2018 Cask Data, Inc. + * Copyright © 2016-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -30,7 +30,7 @@ import io.cdap.cdap.common.conf.KafkaConstants; import io.cdap.cdap.common.guice.ConfigModule; import io.cdap.cdap.common.guice.KafkaClientModule; -import io.cdap.cdap.common.guice.ZKClientModule; +import io.cdap.cdap.common.guice.ZkClientModule; import io.cdap.cdap.common.service.Retries; import io.cdap.cdap.common.service.RetryStrategies; import io.cdap.cdap.common.utils.Tasks; @@ -75,6 +75,7 @@ * {@link ExternalResource} to be used in tests that require in-memory Kafka. */ public class KafkaTester extends ExternalResource { + private static final Logger LOG = LoggerFactory.getLogger(KafkaTester.class); private static final Gson GSON = new Gson(); @@ -88,31 +89,33 @@ public class KafkaTester extends ExternalResource { private final Map extraConfigs; private final Iterable extraModules; private final CConfiguration cConf; - private final String [] kafkaBrokerListParams; + private final String[] kafkaBrokerListParams; private final TemporaryFolder tmpFolder; private final int numPartitions; /** - * Create a {@link KafkaTester} with default configurations and {@link Module Guice modules} and a single - * Kafka partition. + * Create a {@link KafkaTester} with default configurations and {@link Module Guice modules} and a + * single Kafka partition. */ public KafkaTester() { this(Collections.emptyMap(), Collections.emptyList(), 1); } /** - * Create a {@link KafkaTester} with the specified set of extra configurations, {@link Module Guice modules} and - * Kafka partitions. + * Create a {@link KafkaTester} with the specified set of extra configurations, {@link Module + * Guice modules} and Kafka partitions. * - * @param extraConfigs the specified extra configurations to provide while creating the - * {@link Injector Guice injector} - * @param extraModules the specified extra {@link Module Guice modules} to include in the - * {@link Injector Guice injector} - * @param numPartitions the specified number of Kafka partitions - * @param kafkaBrokerListParams a list of configuration parameters to which the kafka broker list should be set + * @param extraConfigs the specified extra configurations to provide while creating the + * {@link Injector Guice injector} + * @param extraModules the specified extra {@link Module Guice modules} to include in the + * {@link Injector Guice injector} + * @param numPartitions the specified number of Kafka partitions + * @param kafkaBrokerListParams a list of configuration parameters to which the kafka broker list + * should be set */ - public KafkaTester(Map extraConfigs, Iterable extraModules, int numPartitions, - String ... kafkaBrokerListParams) { + public KafkaTester(Map extraConfigs, Iterable extraModules, + int numPartitions, + String... kafkaBrokerListParams) { this.extraConfigs = extraConfigs; this.extraModules = extraModules; this.cConf = CConfiguration.create(); @@ -130,7 +133,7 @@ protected void before() throws Throwable { kafkaServer = new EmbeddedKafkaServer(generateKafkaConfig()); kafkaServer.startAndWait(); - initializeCConf(); + initializeCconf(); injector = createInjector(); zkClient = injector.getInstance(ZKClientService.class); zkClient.startAndWait(); @@ -139,7 +142,8 @@ protected void before() throws Throwable { brokerService = injector.getInstance(BrokerService.class); brokerService.startAndWait(); - String brokerList = updateKafkaBrokerList(injector.getInstance(CConfiguration.class), brokerService); + String brokerList = updateKafkaBrokerList(injector.getInstance(CConfiguration.class), + brokerService); LOG.info("Waiting for Kafka server to startup..."); waitForKafkaStartup(); LOG.info("Kafka server started with broker list {}", brokerList); @@ -154,7 +158,7 @@ protected void after() { zkServer.stopAndWait(); } - private void initializeCConf() throws IOException { + private void initializeCconf() throws IOException { cConf.unset(KafkaConstants.ConfigKeys.ZOOKEEPER_NAMESPACE_CONFIG); cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath()); cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr()); @@ -165,9 +169,9 @@ private void initializeCConf() throws IOException { private String updateKafkaBrokerList(CConfiguration cConf, BrokerService brokerService) { String brokerList = Retries.callWithRetries( - brokerService::getBrokerList, - RetryStrategies.timeLimit(10, TimeUnit.SECONDS, - RetryStrategies.exponentialDelay(100, 2000, TimeUnit.MILLISECONDS)) + brokerService::getBrokerList, + RetryStrategies.timeLimit(10, TimeUnit.SECONDS, + RetryStrategies.exponentialDelay(100, 2000, TimeUnit.MILLISECONDS)) ); for (String param : kafkaBrokerListParams) { @@ -179,11 +183,11 @@ private String updateKafkaBrokerList(CConfiguration cConf, BrokerService brokerS private Injector createInjector() { List modules = ImmutableList.builder() - .add(new ConfigModule(cConf)) - .add(new ZKClientModule()) - .add(new KafkaClientModule()) - .addAll(extraModules) - .build(); + .add(new ConfigModule(cConf)) + .add(new ZkClientModule()) + .add(new KafkaClientModule()) + .addAll(extraModules) + .build(); return Guice.createInjector(modules); } @@ -214,20 +218,21 @@ private void waitForKafkaStartup() throws Exception { final AtomicBoolean isKafkaStarted = new AtomicBoolean(false); try { KafkaPublisher kafkaPublisher = kafkaClient.getPublisher(KafkaPublisher.Ack.LEADER_RECEIVED, - Compression.NONE); + Compression.NONE); final String testTopic = "kafkatester.test.topic"; final String testMessage = "Test Message"; kafkaPublisher.prepare(testTopic).add(Charsets.UTF_8.encode(testMessage), 0).send().get(); - getPublishedMessages(testTopic, ImmutableSet.of(0), 1, 0, new Function() { - @Override - public String apply(FetchedMessage input) { - String fetchedMessage = Charsets.UTF_8.decode(input.getPayload()).toString(); - if (fetchedMessage.equalsIgnoreCase(testMessage)) { - isKafkaStarted.set(true); - } - return ""; - } - }); + getPublishedMessages(testTopic, ImmutableSet.of(0), 1, 0, + new Function() { + @Override + public String apply(FetchedMessage input) { + String fetchedMessage = Charsets.UTF_8.decode(input.getPayload()).toString(); + if (fetchedMessage.equalsIgnoreCase(testMessage)) { + isKafkaStarted.set(true); + } + return ""; + } + }); } catch (Exception e) { // nothing to do as waiting for kafka startup } @@ -239,72 +244,75 @@ public Injector getInjector() { return injector; } - public CConfiguration getCConf() { + public CConfiguration getCconf() { return cConf; } /** * Return a list of messages from the specified Kafka topic. * - * @param topic the specified Kafka topic + * @param topic the specified Kafka topic * @param expectedNumMsgs the expected number of messages - * @param typeOfT the {@link Type} of each message - * @param the type of each message + * @param typeOfT the {@link Type} of each message + * @param the type of each message * @return a list of messages from the specified Kafka topic */ @SuppressWarnings("unused") - public List getPublishedMessages(String topic, int expectedNumMsgs, Type typeOfT) throws InterruptedException { + public List getPublishedMessages(String topic, int expectedNumMsgs, Type typeOfT) + throws InterruptedException { return getPublishedMessages(topic, expectedNumMsgs, typeOfT, GSON); } /** * Return a list of messages from the specified Kafka topic. * - * @param topic the specified Kafka topic + * @param topic the specified Kafka topic * @param expectedNumMsgs the expected number of messages - * @param typeOfT the {@link Type} of each message - * @param gson the {@link Gson} object to use for deserializing messages - * @param the type of each message + * @param typeOfT the {@link Type} of each message + * @param gson the {@link Gson} object to use for deserializing messages + * @param the type of each message * @return a list of messages from the specified Kafka topic */ public List getPublishedMessages(String topic, int expectedNumMsgs, Type typeOfT, - Gson gson) throws InterruptedException { + Gson gson) throws InterruptedException { return getPublishedMessages(topic, expectedNumMsgs, typeOfT, gson, 0); } /** * Return a list of messages from the specified Kafka topic. * - * @param topic the specified Kafka topic + * @param topic the specified Kafka topic * @param expectedNumMsgs the expected number of messages - * @param typeOfT the {@link Type} of each message - * @param gson the {@link Gson} object to use for deserializing messages - * @param offset the Kafka offset - * @param the type of each message + * @param typeOfT the {@link Type} of each message + * @param gson the {@link Gson} object to use for deserializing messages + * @param offset the Kafka offset + * @param the type of each message * @return a list of messages from the specified Kafka topic */ public List getPublishedMessages(String topic, int expectedNumMsgs, final Type typeOfT, - final Gson gson, int offset) throws InterruptedException { - return getPublishedMessages(topic, ImmutableSet.of(0), expectedNumMsgs, offset, new Function() { - @Override - public T apply(FetchedMessage input) { - return gson.fromJson(Bytes.toString(input.getPayload()), typeOfT); - } - }); + final Gson gson, int offset) throws InterruptedException { + return getPublishedMessages(topic, ImmutableSet.of(0), expectedNumMsgs, offset, + new Function() { + @Override + public T apply(FetchedMessage input) { + return gson.fromJson(Bytes.toString(input.getPayload()), typeOfT); + } + }); } /** * Return a list of messages from the specified Kafka topic. * - * @param topic the specified Kafka topic + * @param topic the specified Kafka topic * @param expectedNumMsgs the expected number of messages - * @param offset the Kafka offset - * @param converter converter function to convert payload bytebuffer into type T - * @param the type of each message + * @param offset the Kafka offset + * @param converter converter function to convert payload bytebuffer into type T + * @param the type of each message * @return a list of messages from the specified Kafka topic */ - public List getPublishedMessages(String topic, Set partitions, int expectedNumMsgs, int offset, - final Function converter) throws InterruptedException { + public List getPublishedMessages(String topic, Set partitions, + int expectedNumMsgs, int offset, + final Function converter) throws InterruptedException { final CountDownLatch latch = new CountDownLatch(expectedNumMsgs); final CountDownLatch stopLatch = new CountDownLatch(1); final List actual = new ArrayList<>(expectedNumMsgs); @@ -313,35 +321,37 @@ public List getPublishedMessages(String topic, Set partitions, i preparer.add(topic, partition, offset); } Cancellable cancellable = preparer.consume( - new KafkaConsumer.MessageCallback() { - @Override - public long onReceived(Iterator messages) { - long offset = 0L; - while (messages.hasNext()) { - FetchedMessage message = messages.next(); - actual.add(converter.apply(message)); - latch.countDown(); - offset = message.getNextOffset(); + new KafkaConsumer.MessageCallback() { + @Override + public long onReceived(Iterator messages) { + long offset = 0L; + while (messages.hasNext()) { + FetchedMessage message = messages.next(); + actual.add(converter.apply(message)); + latch.countDown(); + offset = message.getNextOffset(); + } + return offset; } - return offset; - } - @Override - public void finished() { - stopLatch.countDown(); + @Override + public void finished() { + stopLatch.countDown(); + } } - } ); - Assert.assertTrue(String.format("Expected %d messages but found %d messages", expectedNumMsgs, actual.size()), - latch.await(15, TimeUnit.SECONDS)); + Assert.assertTrue( + String.format("Expected %d messages but found %d messages", expectedNumMsgs, actual.size()), + latch.await(15, TimeUnit.SECONDS)); cancellable.cancel(); Assert.assertTrue(stopLatch.await(15, TimeUnit.SECONDS)); return actual; } - public Map getPublishedMessages(String topic, Set partitions, int expectedNumMsgs, - final Function converter) - throws InterruptedException { + public Map getPublishedMessages(String topic, Set partitions, + int expectedNumMsgs, + final Function converter) + throws InterruptedException { final CountDownLatch latch = new CountDownLatch(expectedNumMsgs); final CountDownLatch stopLatch = new CountDownLatch(1); final Map actual = new HashMap<>(); @@ -350,42 +360,45 @@ public Map getPublishedMessages(String topic, Set parti preparer.addFromBeginning(topic, partition); } Cancellable cancellable = preparer.consume( - new KafkaConsumer.MessageCallback() { - @Override - public long onReceived(Iterator messages) { - long offset = 0L; - while (messages.hasNext()) { - FetchedMessage message = messages.next(); - actual.put(message.getTopicPartition().getPartition(), converter.apply(message)); - latch.countDown(); - offset = message.getNextOffset(); + new KafkaConsumer.MessageCallback() { + @Override + public long onReceived(Iterator messages) { + long offset = 0L; + while (messages.hasNext()) { + FetchedMessage message = messages.next(); + actual.put(message.getTopicPartition().getPartition(), converter.apply(message)); + latch.countDown(); + offset = message.getNextOffset(); + } + return offset; } - return offset; - } - @Override - public void finished() { - stopLatch.countDown(); + @Override + public void finished() { + stopLatch.countDown(); + } } - } ); - Assert.assertTrue(String.format("Expected %d messages but found %d messages", expectedNumMsgs, actual.size()), - latch.await(15, TimeUnit.SECONDS)); + Assert.assertTrue( + String.format("Expected %d messages but found %d messages", expectedNumMsgs, actual.size()), + latch.await(15, TimeUnit.SECONDS)); cancellable.cancel(); Assert.assertTrue(stopLatch.await(15, TimeUnit.SECONDS)); return actual; } /** - * Returns the {@link KafkaClient} that can be used to talk to the Kafka server started in this class. + * Returns the {@link KafkaClient} that can be used to talk to the Kafka server started in this + * class. */ public KafkaClient getKafkaClient() { return kafkaClient; } /** - * Returns the {@link BrokerService} that provides information about the Kafka server started in this class. + * Returns the {@link BrokerService} that provides information about the Kafka server started in + * this class. */ public BrokerService getBrokerService() { return brokerService; @@ -395,8 +408,10 @@ public BrokerService getBrokerService() { * Creates a topic with the given number of partitions. */ public void createTopic(String topic, int partitions) { - ZkClient zkClient = new ZkClient(zkServer.getConnectionStr(), 20000, 2000, ZKStringSerializer$.MODULE$); + ZkClient zkClient = new ZkClient(zkServer.getConnectionStr(), 20000, 2000, + ZKStringSerializer$.MODULE$); ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zkServer.getConnectionStr()), false); - AdminUtils.createTopic(zkUtils, topic, partitions, 1, new Properties(), RackAwareMode.Disabled$.MODULE$); + AdminUtils.createTopic(zkUtils, topic, partitions, 1, new Properties(), + RackAwareMode.Disabled$.MODULE$); } } diff --git a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java index 3f14473bfb98..d92f185e6c3f 100644 --- a/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java +++ b/cdap-gateway/src/main/java/io/cdap/cdap/gateway/router/RouterMain.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2022 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -26,8 +26,8 @@ import io.cdap.cdap.common.guice.ConfigModule; import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.runtime.DaemonMain; import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule; import io.cdap.cdap.security.guice.ExternalAuthenticationModule; @@ -49,6 +49,11 @@ public class RouterMain extends DaemonMain { private ZKClientService zkClientService; private NettyRouter router; + /** + * Entry point for the router. + * + * @param args Arguments to the router. + */ public static void main(String[] args) { try { new RouterMain().doMain(args); @@ -130,8 +135,8 @@ static Injector createGuiceInjector(CConfiguration cConf) { return Guice.createInjector( new ConfigModule(cConf), RemoteAuthenticatorModules.getDefaultModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new RouterModules().getDistributedModules(), CoreSecurityRuntimeModule.getDistributedModule(cConf), new ExternalAuthenticationModule(), diff --git a/cdap-kafka/src/main/java/io/cdap/cdap/kafka/run/KafkaServerMain.java b/cdap-kafka/src/main/java/io/cdap/cdap/kafka/run/KafkaServerMain.java index a0ec0f423503..ccf6c3bd4174 100644 --- a/cdap-kafka/src/main/java/io/cdap/cdap/kafka/run/KafkaServerMain.java +++ b/cdap-kafka/src/main/java/io/cdap/cdap/kafka/run/KafkaServerMain.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -57,7 +57,7 @@ public static void main(String[] args) throws Exception { public void init(String[] args) { CConfiguration cConf = CConfiguration.create(); - String zkConnectStr = Constants.Zookeeper.getZKQuorum(cConf); + String zkConnectStr = Constants.Zookeeper.getZkQuorum(cConf); String zkNamespace = cConf.get(KafkaConstants.ConfigKeys.ZOOKEEPER_NAMESPACE_CONFIG); if (zkNamespace != null) { diff --git a/cdap-master/src/main/java/io/cdap/cdap/app/guice/SystemMetricsExporterModule.java b/cdap-master/src/main/java/io/cdap/cdap/app/guice/SystemMetricsExporterModule.java index 8023b54bc826..9437c39e94ea 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/app/guice/SystemMetricsExporterModule.java +++ b/cdap-master/src/main/java/io/cdap/cdap/app/guice/SystemMetricsExporterModule.java @@ -1,5 +1,5 @@ /* - * Copyright © 2022 Cask Data, Inc. + * Copyright © 2022-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -19,22 +19,22 @@ import com.google.inject.AbstractModule; import com.google.inject.assistedinject.FactoryModuleBuilder; import io.cdap.cdap.api.metrics.MetricsPublisher; -import io.cdap.cdap.metrics.jmx.JMXMetricsCollector; -import io.cdap.cdap.metrics.jmx.JMXMetricsCollectorFactory; +import io.cdap.cdap.metrics.jmx.JmxMetricsCollector; +import io.cdap.cdap.metrics.jmx.JmxMetricsCollectorFactory; import io.cdap.cdap.metrics.process.loader.MetricsWriterExtensionLoader; import io.cdap.cdap.metrics.process.loader.MetricsWriterProvider; import io.cdap.cdap.metrics.publisher.MetricsWritersMetricsPublisher; /** - * Guice module that provides mappings for SystemMetricsExporterServiceMain + * Guice module that provides mappings for SystemMetricsExporterServiceMain. */ public class SystemMetricsExporterModule extends AbstractModule { @Override protected void configure() { install(new FactoryModuleBuilder() - .implement(JMXMetricsCollector.class, JMXMetricsCollector.class) - .build(JMXMetricsCollectorFactory.class)); + .implement(JmxMetricsCollector.class, JmxMetricsCollector.class) + .build(JmxMetricsCollectorFactory.class)); bind(MetricsPublisher.class).to(MetricsWritersMetricsPublisher.class); bind(MetricsWriterProvider.class).to(MetricsWriterExtensionLoader.class); } diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/DatasetOpExecutorServerTwillRunnable.java b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/DatasetOpExecutorServerTwillRunnable.java index 185578f51842..f43aff07efac 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/DatasetOpExecutorServerTwillRunnable.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/DatasetOpExecutorServerTwillRunnable.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2022 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -31,8 +31,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.logging.LoggingContextAccessor; import io.cdap.cdap.common.logging.ServiceLoggingContext; import io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule; @@ -107,8 +107,8 @@ static Injector createInjector(CConfiguration cConf, Configuration hConf, String new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new KafkaClientModule(), new MessagingClientModule(), new MetricsClientRuntimeModule().getDistributedModules(), diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/LogSaverTwillRunnable.java b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/LogSaverTwillRunnable.java index 7b9292bc8c19..a3f947c811da 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/LogSaverTwillRunnable.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/LogSaverTwillRunnable.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2022 Cask Data, Inc. + * Copyright © 2017-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -29,8 +29,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule; import io.cdap.cdap.common.twill.AbstractMasterTwillRunnable; import io.cdap.cdap.data.runtime.DataFabricModules; @@ -102,8 +102,8 @@ static Injector createGuiceInjector(CConfiguration cConf, Configuration hConf, new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new KafkaClientModule(), new MetricsClientRuntimeModule().getDistributedModules(), new DFSLocationModule(), diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java index 137cb262574d..f84dc6a8c6d4 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2022 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -46,8 +46,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.io.URLConnections; import io.cdap.cdap.common.logging.LoggerLogHandler; import io.cdap.cdap.common.runtime.DaemonMain; @@ -191,6 +191,12 @@ public class MasterServiceMain extends DaemonMain { } } + /** + * Entry point for master. + * + * @param args The master arguments. + * @throws Exception If an error occurs. + */ public static void main(final String[] args) throws Exception { ClassLoader classLoader = MainClassLoader.createFromContext(); if (classLoader == null) { @@ -207,6 +213,9 @@ public static void main(final String[] args) throws Exception { } } + /** + * Creates a master service main. + */ @SuppressWarnings("WeakerAccess") public MasterServiceMain() { CConfiguration cConf = CConfiguration.create(); @@ -261,7 +270,7 @@ public void start() throws Exception { logAppenderInitializer.initialize(); resetShutdownTime(); createDirectory("twill"); - createSystemHBaseNamespace(); + createSystemHbaseNamespace(); updateConfigurationTable(); Services.startAndWait(zkClient, cConf.getLong(Constants.Zookeeper.CLIENT_STARTUP_TIMEOUT_MILLIS), @@ -468,7 +477,7 @@ private void login(CConfiguration cConf) { /** * Creates HBase namespace for the cdap system namespace. */ - private void createSystemHBaseNamespace() { + private void createSystemHbaseNamespace() { HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get(); try (HBaseDDLExecutor ddlExecutor = new HBaseDDLExecutorFactory(cConf, hConf).get()) { ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM)); @@ -527,7 +536,7 @@ private void resetShutdownTime() { static Injector createProcessInjector(CConfiguration cConf, Configuration hConf) { return Guice.createInjector( new ConfigModule(cConf, hConf), - new ZKClientModule(), + new ZkClientModule(), new KafkaLogAppenderModule() ); } @@ -545,7 +554,7 @@ static Injector createLeaderInjector(CConfiguration cConf, Configuration hConf, new AbstractModule() { @Override protected void configure() { - // Instead of using ZKClientModule that will create new instance of ZKClient, we create instance + // Instead of using ZkClientModule that will create new instance of ZKClient, we create instance // binding to reuse the same ZKClient used for leader election bind(ZKClient.class).toInstance(zkClientService); bind(ZKClientService.class).toInstance(zkClientService); @@ -555,7 +564,7 @@ protected void configure() { new KafkaLogAppenderModule(), new DFSLocationModule(), new IOModule(), - new ZKDiscoveryModule(), + new ZkDiscoveryModule(), new KafkaClientModule(), new DataSetServiceModules().getDistributedModules(), new DataFabricModules("cdap.master").getDistributedModules(), @@ -752,7 +761,8 @@ void postStop() { return; } - SortedMap participants = ImmutableSortedMap.of(); + SortedMap participants = ImmutableSortedMap + .of(); try { participants = electionInfoService.fetchCurrentParticipants(); @@ -790,10 +800,9 @@ void postStop() { /** * Monitors the twill application for master services running through Twill. * - * @param executor executor for re-running the application if it gets terminated - * @param failures number of failures in starting the application - * @param serviceController the reference to be updated with the active {@link - * TwillController} + * @param executor executor for re-running the application if it gets terminated + * @param failures number of failures in starting the application + * @param serviceController the reference to be updated with the active {@link TwillController} */ private void monitorTwillApplication(final ScheduledExecutorService executor, final int failures, @@ -925,10 +934,8 @@ private TwillController startTwillApplication(TwillRunnerService twillRunner, cConf.get(Constants.AppFabric.TEMP_DIR)).toAbsolutePath()); final Path runDir = Files.createTempDirectory(tempPath, "master"); try { - Path logbackFile = saveLogbackConf(runDir.resolve("logback.xml")); MasterTwillApplication masterTwillApp = new MasterTwillApplication(cConf, getServiceInstances(serviceStore, cConf)); - List extraClassPath = masterTwillApp.prepareLocalizeResource(runDir, hConf); TwillPreparer preparer = twillRunner.prepare(masterTwillApp); Map twillConfigs = new HashMap<>(); @@ -939,6 +946,8 @@ private TwillController startTwillApplication(TwillRunnerService twillRunner, cConf.get(Constants.AppFabric.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL)); preparer.withConfiguration(twillConfigs); + Path logbackFile = saveLogbackConf(runDir.resolve("logback.xml")); + // Add logback xml if (Files.exists(logbackFile)) { preparer @@ -973,6 +982,7 @@ private TwillController startTwillApplication(TwillRunnerService twillRunner, // Setup extra classpath. Currently twill doesn't support different classpath per runnable, // hence we just set it for all containers. The actual jars are localized via the MasterTwillApplication, // and having missing jars as specified in the classpath is ok. + List extraClassPath = masterTwillApp.prepareLocalizeResource(runDir, hConf); preparer = preparer.withClassPaths(extraClassPath); // Set the container to use MasterServiceMainClassLoader for class rewriting @@ -1022,8 +1032,9 @@ public void run() { private Map getServiceInstances(ServiceStore serviceStore, CConfiguration cConf) { Map instanceCountMap = new HashMap<>(); - Set serviceResourceKeysSet = MasterUtils.createSystemServicesResourceKeysSet( - cConf); + Set serviceResourceKeysSet = MasterUtils + .createSystemServicesResourceKeysSet( + cConf); for (ServiceResourceKeys serviceResourceKeys : serviceResourceKeysSet) { String service = serviceResourceKeys.getServiceName(); try { diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MessagingServiceTwillRunnable.java b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MessagingServiceTwillRunnable.java index 97cffb86de6c..3b9b99b1b2f5 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MessagingServiceTwillRunnable.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MessagingServiceTwillRunnable.java @@ -1,5 +1,5 @@ /* - * Copyright © 2016-2022 Cask Data, Inc. + * Copyright © 2016-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -27,8 +27,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.logging.LoggingContextAccessor; import io.cdap.cdap.common.logging.ServiceLoggingContext; import io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule; @@ -80,14 +80,21 @@ protected Injector doInit(TwillContext context) { return injector; } + /** + * Creates an injector for the messaging service. + * + * @param cConf The CConf to use. + * @param hConf The HConf to use. + * @return An injector for the messaging service. + */ @VisibleForTesting public static Injector createInjector(CConfiguration cConf, Configuration hConf) { return Guice.createInjector( new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new KafkaClientModule(), new MetricsClientRuntimeModule().getDistributedModules(), new KafkaLogAppenderModule(), diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MetricsProcessorTwillRunnable.java b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MetricsProcessorTwillRunnable.java index 8255f10ad78c..41bc8f986a7c 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MetricsProcessorTwillRunnable.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MetricsProcessorTwillRunnable.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2022 Cask Data, Inc. + * Copyright © 2017-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -32,8 +32,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.logging.LoggingContextAccessor; import io.cdap.cdap.common.logging.ServiceLoggingContext; import io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule; @@ -117,8 +117,8 @@ static Injector createGuiceInjector(CConfiguration cConf, Configuration hConf, S new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new KafkaClientModule(), new MessagingClientModule(), new MetricsClientRuntimeModule().getDistributedModules(), diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MetricsTwillRunnable.java b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MetricsTwillRunnable.java index 56f74e0eee53..c1386d23515d 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MetricsTwillRunnable.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MetricsTwillRunnable.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2022 Cask Data, Inc. + * Copyright © 2017-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -29,8 +29,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.logging.LoggingContextAccessor; import io.cdap.cdap.common.logging.ServiceLoggingContext; import io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule; @@ -106,8 +106,8 @@ static Injector createGuiceInjector(CConfiguration cConf, Configuration hConf, new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new KafkaClientModule(), new MessagingClientModule(), new DataFabricModules(txClientId).getDistributedModules(), diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/TransactionServiceTwillRunnable.java b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/TransactionServiceTwillRunnable.java index a2cc320dd793..29b1959412bc 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/TransactionServiceTwillRunnable.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/TransactionServiceTwillRunnable.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2022 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -29,8 +29,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.logging.LoggingContextAccessor; import io.cdap.cdap.common.logging.ServiceLoggingContext; import io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule; @@ -101,8 +101,8 @@ static Injector createGuiceInjector(CConfiguration cConf, Configuration hConf, new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new KafkaClientModule(), new MessagingClientModule(), new DataFabricModules(txClientId).getDistributedModules(), diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/tools/HBaseTableExporter.java b/cdap-master/src/main/java/io/cdap/cdap/data/tools/HBaseTableExporter.java index c88c8200b69e..a2ad3e468d1e 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/tools/HBaseTableExporter.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/tools/HBaseTableExporter.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2022 Cask Data, Inc. + * Copyright © 2015-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -13,6 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package io.cdap.cdap.data.tools; import com.google.common.annotations.VisibleForTesting; @@ -27,8 +28,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule; import io.cdap.cdap.data.runtime.DataFabricModules; import io.cdap.cdap.data.runtime.DataSetsModules; @@ -72,6 +73,7 @@ * Tool to export the HBase table to HFiles. Tool accepts the HBase table name as input parameter * and outputs the HDFS path where the corresponding HFiles are exported. */ +@SuppressWarnings("checkstyle:AbbreviationAsWordInName") public class HBaseTableExporter { private static final Logger LOG = LoggerFactory.getLogger(HBaseTableExporter.class); @@ -82,6 +84,11 @@ public class HBaseTableExporter { private final TransactionSystemClient txClient; private Path bulkloadDir; + /** + * Creates an HBase table exporter. + * + * @throws Exception If creation fails. + */ public HBaseTableExporter() throws Exception { this.hConf = HBaseConfiguration.create(); Injector injector = createInjector(CConfiguration.create(), hConf); @@ -102,14 +109,21 @@ public void run() { } + /** + * Creates an injector for the exporter. + * + * @param cConf The CConf to use. + * @param hConf The HConf to use. + * @return The injector. + */ @VisibleForTesting public static Injector createInjector(CConfiguration cConf, Configuration hConf) { return Guice.createInjector( new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new KafkaClientModule(), new DFSLocationModule(), new DataFabricModules(HBaseTableExporter.class.getName()).getDistributedModules(), @@ -136,9 +150,9 @@ protected void configure() { /** * Sets up the actual MapReduce job. * - * @param tx The transaction which needs to be passed to the Scan instance. This transaction - * is be used by coprocessors to filter out the data corresonding to the invalid transactions - * . + * @param tx The transaction which needs to be passed to the Scan instance. This + * transaction is be used by coprocessors to filter out the data corresonding to + * the invalid transactions . * @param tableName Name of the table which need to be exported as HFiles. * @return the configured job */ @@ -207,6 +221,12 @@ private void printHelp() { System.out.println(" tablename Name of the table to copy"); } + /** + * The entrypoint for the exporter. + * + * @param args Arguments for the exporter. + * @throws Exception If something fails. + */ public void doMain(String[] args) throws Exception { if (args.length < 1) { printHelp(); @@ -234,6 +254,12 @@ public void doMain(String[] args) throws Exception { } } + /** + * The entrypoint for the exporter. + * + * @param args Arguments for the exporter. + * @throws Exception If something fails. + */ public static void main(String[] args) throws Exception { try { HBaseTableExporter hBaseTableExporter = new HBaseTableExporter(); diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java b/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java index 08ef0864afc0..1fbdba189312 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2022 Cask Data, Inc. + * Copyright © 2017-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -44,8 +44,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.data.runtime.DataFabricModules; import io.cdap.cdap.data.runtime.DataSetsModules; import io.cdap.cdap.data.runtime.SystemDatasetRuntimeModule; @@ -87,11 +87,11 @@ /** * Debugging tool for {@link JobQueue}. * - * Because the JobQueue is scanned over multiple transactions, it will be an inconsistent view. The - * same Job will not be counted multiple times, but some Jobs may be missed if they were deleted or - * added during the scan. The count of the Job State may also be inconsistent. + *

Because the JobQueue is scanned over multiple transactions, it will be an inconsistent view. + * The same Job will not be counted multiple times, but some Jobs may be missed if they were deleted + * or added during the scan. The count of the Job State may also be inconsistent. * - * The publish timestamp of the last message processed from the topics will also be inconsistent + *

The publish timestamp of the last message processed from the topics will also be inconsistent * from the Jobs in the JobQueue. */ public class JobQueueDebugger extends AbstractIdleService { @@ -108,6 +108,13 @@ public class JobQueueDebugger extends AbstractIdleService { private JobQueueScanner jobQueueScanner; + /** + * Debugger for the job queue. + * + * @param cConf The CConf to use. + * @param zkClientService The ZK client service. + * @param transactionRunner The transaction runner. + */ @Inject public JobQueueDebugger(CConfiguration cConf, ZKClientService zkClientService, TransactionRunner transactionRunner) { @@ -267,6 +274,10 @@ void updateWithJob(Job job) { case PENDING_LAUNCH: pendingLaunch++; break; + default: + throw new IllegalStateException(String.format("Job '%s' in unexpected state %s", + job.getJobKey(), + job.getState().toString())); } updateOldestNewest(job); @@ -329,8 +340,8 @@ private static Injector createInjector() throws Exception { new ConfigModule(cConf, HBaseConfiguration.create()), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new DFSLocationModule(), new TwillModule(), new DataFabricModules().getDistributedModules(), @@ -370,6 +381,12 @@ static JobQueueDebugger createDebugger() throws Exception { return createInjector().getInstance(JobQueueDebugger.class); } + /** + * Entry point for the job queue debugger. + * + * @param args The arguments to the debugger. + * @throws Exception If something fails. + */ public static void main(String[] args) throws Exception { Options options = new Options() .addOption(new Option("h", "help", false, "Print this usage message.")) diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/tools/UpgradeTool.java b/cdap-master/src/main/java/io/cdap/cdap/data/tools/UpgradeTool.java index 50ee7a5f732c..b9b05870dc60 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/tools/UpgradeTool.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/tools/UpgradeTool.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015-2022 Cask Data, Inc. + * Copyright © 2015-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -43,8 +43,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.metrics.NoOpMetricsCollectionService; import io.cdap.cdap.common.metrics.NoOpMetricsSystemClient; import io.cdap.cdap.common.service.Services; @@ -95,7 +95,7 @@ import org.slf4j.LoggerFactory; /** - * Command line tool for the Upgrade tool + * Command line tool for the Upgrade tool. */ public class UpgradeTool { @@ -173,8 +173,8 @@ Injector createInjector() { new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new DFSLocationModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new MessagingClientModule(), Modules.override(new DataSetsModules().getDistributedModules()).with( new AbstractModule() { @@ -257,8 +257,8 @@ protected void configure() { * these new tables to be added so that the co processor of these tables can be upgraded when the * user runs CDAP's Hbase Upgrade after upgrading to a newer version of Hbase. * - * @param includeNewDatasets boolean which specifies whether to add new datasets in ds - * framework or not + * @param includeNewDatasets boolean which specifies whether to add new datasets in ds framework + * or not */ private void startUp(boolean includeNewDatasets) throws Exception { // Start all the services. @@ -272,13 +272,13 @@ private void startUp(boolean includeNewDatasets) throws Exception { LOG.info("Starting Transaction Service..."); txService.startAndWait(); LOG.info("Initializing Dataset Framework..."); - initializeDSFramework(dsFramework, includeNewDatasets); + initializeDsFramework(dsFramework, includeNewDatasets); LOG.info("Building and uploading new HBase coprocessors..."); coprocessorManager.ensureCoprocessorExists(); } /** - * Stop services and + * Stops services. */ private void stop() { try { @@ -324,7 +324,7 @@ private void doMain(String[] args) throws Exception { System.out.println("Starting upgrade ..."); try { startUp(false); - ensureCDAPMasterStopped(); + ensureCdapMasterStopped(); performUpgrade(); System.out.println("\nUpgrade completed successfully.\n"); } finally { @@ -343,7 +343,7 @@ private void doMain(String[] args) throws Exception { System.out.println("Starting upgrade ..."); try { startUp(true); - performHBaseUpgrade(); + performHbaseUpgrade(); System.out.println("\nUpgrade completed successfully.\n"); } finally { stop(); @@ -356,6 +356,9 @@ private void doMain(String[] args) throws Exception { case HELP: printHelp(); break; + default: + throw new IllegalArgumentException(String.format("Invalid action '%s'", + action.toString())); } } catch (Exception e) { System.out.println( @@ -370,7 +373,7 @@ private void doMain(String[] args) throws Exception { * * @throws Exception if at least one master is running */ - private void ensureCDAPMasterStopped() throws Exception { + private void ensureCdapMasterStopped() throws Exception { String appFabricPath = String.format("/discoverable/%s", Constants.Service.APP_FABRIC_HTTP); NodeChildren nodeChildren = zkClientService.getChildren(appFabricPath).get(); List runningNodes = new ArrayList<>(); @@ -432,7 +435,7 @@ private void performUpgrade() throws Exception { performCoprocessorUpgrade(); } - private void performHBaseUpgrade() throws Exception { + private void performHbaseUpgrade() throws Exception { System.setProperty(AbstractHBaseDataSetAdmin.SYSTEM_PROPERTY_FORCE_HBASE_UPGRADE, Boolean.TRUE.toString()); performCoprocessorUpgrade(); @@ -447,6 +450,11 @@ private void performCoprocessorUpgrade() throws Exception { dsUpgrade.upgrade(); } + /** + * Entry point for the upgrade tool. + * + * @param args Arguments for the tool. + */ public static void main(String[] args) { try { UpgradeTool upgradeTool = new UpgradeTool(); @@ -468,7 +476,7 @@ public static void main(String[] args) { * these new tables to be added so that the co processor of these tables can be upgraded when the * user runs CDAP's Hbase Upgrade after upgrading to a newer version of Hbase. */ - private void initializeDSFramework(DatasetFramework datasetFramework, boolean includeNewDatasets) + private void initializeDsFramework(DatasetFramework datasetFramework, boolean includeNewDatasets) throws IOException, DatasetManagementException, UnauthorizedException { // Note: do no remove this block even if it's empty. Read the comment below and function doc above //noinspection StatementWithEmptyBody diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java index aa8b391294d1..026acbd48747 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java @@ -52,6 +52,7 @@ import io.cdap.cdap.internal.app.services.AppFabricServer; import io.cdap.cdap.internal.app.worker.TaskWorkerServiceLauncher; import io.cdap.cdap.internal.app.worker.system.SystemWorkerServiceLauncher; +import io.cdap.cdap.internal.credential.CredentialProviderService; import io.cdap.cdap.internal.events.EventPublishManager; import io.cdap.cdap.master.spi.environment.MasterEnvironment; import io.cdap.cdap.master.spi.environment.MasterEnvironmentContext; @@ -116,7 +117,7 @@ protected void configure() { @Override protected void configure() { bind(TwillRunnerService.class).toProvider( - new SupplierProviderBridge<>(masterEnv.getTwillRunnerSupplier())) + new SupplierProviderBridge<>(masterEnv.getTwillRunnerSupplier())) .in(Scopes.SINGLETON); bind(TwillRunner.class).to(TwillRunnerService.class); @@ -169,6 +170,8 @@ protected void addServices(Injector injector, List services, // Event publisher could rely on task workers for token generated for security enabled deployments services.add(injector.getInstance(EventPublishManager.class)); + services.add(injector.getInstance(CredentialProviderService.class)); + // Adds the master environment tasks masterEnv.getTasks() .forEach(task -> services.add(new MasterTaskExecutorService(task, masterEnvContext))); diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/StorageMain.java b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/StorageMain.java index e0a0e5e0b6eb..5543b2450206 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/StorageMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/StorageMain.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019-2022 Cask Data, Inc. + * Copyright © 2019-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -29,7 +29,7 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.InMemoryDiscoveryModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; +import io.cdap.cdap.common.guice.ZkClientModule; import io.cdap.cdap.common.metrics.NoOpMetricsCollectionService; import io.cdap.cdap.data.runtime.ConstantTransactionSystemClient; import io.cdap.cdap.data.runtime.DataSetsModules; @@ -96,7 +96,7 @@ protected void configure() { )); if (coreSecurityModule.requiresZKClient()) { - modules.add(new ZKClientModule()); + modules.add(new ZkClientModule()); } Injector injector = Guice.createInjector(modules); diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMain.java index b3a3160f8d35..c00043325d59 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMain.java @@ -1,5 +1,5 @@ /* - * Copyright © 2021 Cask Data, Inc. + * Copyright © 2021-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -28,7 +28,7 @@ import io.cdap.cdap.master.spi.environment.MasterEnvironment; import io.cdap.cdap.master.spi.environment.MasterEnvironmentContext; import io.cdap.cdap.messaging.guice.MessagingClientModule; -import io.cdap.cdap.metrics.jmx.JMXMetricsCollectorFactory; +import io.cdap.cdap.metrics.jmx.JmxMetricsCollectorFactory; import io.cdap.cdap.proto.id.NamespaceId; import java.util.Arrays; import java.util.HashMap; @@ -41,7 +41,7 @@ public class SystemMetricsExporterServiceMain extends AbstractServiceMain { /** - * Main entry point + * Main entry point. */ public static void main(String[] args) throws Exception { main(SystemMetricsExporterServiceMain.class, args); @@ -80,7 +80,7 @@ protected void addServices(Injector injector, List services, Map conf = masterEnvContext.getConfigurations(); Map metricTags = filterByKeyPrefix( conf, MasterEnvironmentContext.ENVIRONMENT_PROPERTY_PREFIX); - services.add(injector.getInstance(JMXMetricsCollectorFactory.class).create(metricTags)); + services.add(injector.getInstance(JmxMetricsCollectorFactory.class).create(metricTags)); } @Override diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/startup/MasterStartupTool.java b/cdap-master/src/main/java/io/cdap/cdap/master/startup/MasterStartupTool.java index cc4b70f5ed38..b05a50e2fb61 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/startup/MasterStartupTool.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/startup/MasterStartupTool.java @@ -1,5 +1,5 @@ /* - * Copyright © 2016-2022 Cask Data, Inc. + * Copyright © 2016-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -29,8 +29,8 @@ import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.startup.CheckRunner; import io.cdap.cdap.common.startup.ConfigurationLogger; import io.cdap.cdap.data.runtime.main.ClientVersions; @@ -52,6 +52,11 @@ public class MasterStartupTool { private static final Logger LOG = LoggerFactory.getLogger(MasterStartupTool.class); private final CheckRunner checkRunner; + /** + * Entry point for the master startup tool. + * + * @param args Arguments for the tool. + */ public static void main(String[] args) { CConfiguration cConf = CConfiguration.create(); @@ -86,10 +91,20 @@ public static void main(String[] args) { } } + /** + * Constructs an instance of a {@link MasterStartupTool}. + * + * @param injector The injector to use. + */ public MasterStartupTool(Injector injector) { this.checkRunner = createCheckRunner(injector); } + /** + * Returns whether the master will startup successfully. + * + * @return Whether the master will startup successfully. + */ public boolean canStartMaster() { List failures = checkRunner.runChecks(); if (!failures.isEmpty()) { @@ -153,8 +168,8 @@ static Injector createInjector(CConfiguration cConf, Configuration hConf) { return Guice.createInjector( new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new IOModule(), new KafkaClientModule(), new DFSLocationModule() diff --git a/cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JMXMetricsCollector.java b/cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollector.java similarity index 82% rename from cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JMXMetricsCollector.java rename to cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollector.java index f9dcbe0d13b0..fe7f02e781a1 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JMXMetricsCollector.java +++ b/cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollector.java @@ -1,5 +1,5 @@ /* - * Copyright © 2021 Cask Data, Inc. + * Copyright © 2021-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -24,6 +24,7 @@ import io.cdap.cdap.api.metrics.MetricsPublisher; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.conf.Constants.Metrics.JvmResource; import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; @@ -53,9 +54,9 @@ * -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false} The * property for setting the JMX server port number should be present in {@code cdap-site.xml}. */ -public class JMXMetricsCollector extends AbstractScheduledService { +public class JmxMetricsCollector extends AbstractScheduledService { - private static final Logger LOG = LoggerFactory.getLogger(JMXMetricsCollector.class); + private static final Logger LOG = LoggerFactory.getLogger(JmxMetricsCollector.class); private static final long MEGA_BYTE = 1024 * 1024; private static final long MAX_PORT = (1 << 16) - 1; private static final long SYSTEM_LOAD_SCALING_FACTOR = 100; @@ -66,16 +67,24 @@ public class JMXMetricsCollector extends AbstractScheduledService { private final JMXServiceURL serviceUrl; private ScheduledExecutorService executor; + /** + * Constructs a new {@link JmxMetricsCollector}. + * + * @param cConf The CConf to use. + * @param metricsPublisher The metrics publisher to use. + * @param metricTags The tags to use. + * @throws MalformedURLException If the server URL is invalid. + */ @Inject - public JMXMetricsCollector(CConfiguration cConf, + public JmxMetricsCollector(CConfiguration cConf, MetricsPublisher metricsPublisher, @Assisted Map metricTags) throws MalformedURLException { this.cConf = cConf; - int serverPort = cConf.getInt(Constants.JMXMetricsCollector.SERVER_PORT); + int serverPort = cConf.getInt(Constants.JmxMetricsCollector.SERVER_PORT); if (serverPort < 0 || serverPort > MAX_PORT) { throw new IllegalArgumentException(String.format( "%s variable (%d) is not a valid port number.", - Constants.JMXMetricsCollector.SERVER_PORT, serverPort)); + Constants.JmxMetricsCollector.SERVER_PORT, serverPort)); } String serverUrl = String.format(SERVICE_URL_FORMAT, "localhost", serverPort); this.metricTags = metricTags; @@ -85,9 +94,9 @@ public JMXMetricsCollector(CConfiguration cConf, @Override protected void startUp() { - LOG.info("Starting JMXMetricsCollector with polling frequency: {}, JMX server port: {}", - this.cConf.getInt(Constants.JMXMetricsCollector.POLL_INTERVAL_SECS), - this.cConf.getInt(Constants.JMXMetricsCollector.SERVER_PORT)); + LOG.info("Starting JmxMetricsCollector with polling frequency: {}, JMX server port: {}", + this.cConf.getInt(Constants.JmxMetricsCollector.POLL_INTERVAL_SECS), + this.cConf.getInt(Constants.JmxMetricsCollector.SERVER_PORT)); this.metricsPublisher.initialize(); } @@ -97,7 +106,7 @@ protected void shutDown() throws IOException { executor.shutdownNow(); } this.metricsPublisher.close(); - LOG.info("Shutting down JMXMetricsCollector has completed."); + LOG.info("Shutting down JmxMetricsCollector has completed."); } @Override @@ -107,7 +116,7 @@ protected void runOneIteration() { try (JMXConnector jmxConnector = JMXConnectorFactory.connect(serviceUrl, null)) { MBeanServerConnection mBeanConn = jmxConnector.getMBeanServerConnection(); metrics.addAll(getMemoryMetrics(mBeanConn)); - metrics.addAll(getCPUMetrics(mBeanConn)); + metrics.addAll(getCpuMetrics(mBeanConn)); metrics.addAll(getThreadMetrics(mBeanConn)); } catch (IOException e) { LOG.error("Error occurred while connecting to JMX server.", e); @@ -126,16 +135,16 @@ Collection getMemoryMetrics(MBeanServerConnection mBeanConn) throws MemoryMXBean.class); MemoryUsage heapMemoryUsage = mxBean.getHeapMemoryUsage(); Collection metrics = new ArrayList<>(); - metrics.add(new MetricValue(Constants.Metrics.JVMResource.HEAP_USED_MB, + metrics.add(new MetricValue(JvmResource.HEAP_USED_MB, MetricType.GAUGE, heapMemoryUsage.getUsed() / MEGA_BYTE)); if (heapMemoryUsage.getMax() >= 0) { - metrics.add(new MetricValue(Constants.Metrics.JVMResource.HEAP_MAX_MB, + metrics.add(new MetricValue(JvmResource.HEAP_MAX_MB, MetricType.GAUGE, heapMemoryUsage.getMax() / MEGA_BYTE)); } return metrics; } - Collection getCPUMetrics(MBeanServerConnection conn) throws IOException { + Collection getCpuMetrics(MBeanServerConnection conn) throws IOException { OperatingSystemMXBean mxBean = ManagementFactory .newPlatformMXBeanProxy(conn, ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME, OperatingSystemMXBean.class); @@ -147,7 +156,7 @@ Collection getCPUMetrics(MBeanServerConnection conn) throws IOExcep double processorCount = mxBean.getAvailableProcessors(); double systemLoadPerProcessorScaled = (systemLoad * SYSTEM_LOAD_SCALING_FACTOR) / processorCount; - metrics.add(new MetricValue(Constants.Metrics.JVMResource.SYSTEM_LOAD_PER_PROCESSOR_SCALED, + metrics.add(new MetricValue(JvmResource.SYSTEM_LOAD_PER_PROCESSOR_SCALED, MetricType.GAUGE, (long) systemLoadPerProcessorScaled)); } return metrics; @@ -157,7 +166,7 @@ Collection getThreadMetrics(MBeanServerConnection conn) throws IOEx ThreadMXBean mxBean = ManagementFactory .newPlatformMXBeanProxy(conn, ManagementFactory.THREAD_MXBEAN_NAME, ThreadMXBean.class); Collection metrics = new ArrayList<>(); - metrics.add(new MetricValue(Constants.Metrics.JVMResource.THREAD_COUNT, + metrics.add(new MetricValue(JvmResource.THREAD_COUNT, MetricType.GAUGE, mxBean.getThreadCount())); return metrics; } @@ -165,7 +174,7 @@ Collection getThreadMetrics(MBeanServerConnection conn) throws IOEx @Override protected Scheduler scheduler() { return Scheduler.newFixedRateSchedule( - 0, cConf.getInt(Constants.JMXMetricsCollector.POLL_INTERVAL_SECS), TimeUnit.SECONDS); + 0, cConf.getInt(Constants.JmxMetricsCollector.POLL_INTERVAL_SECS), TimeUnit.SECONDS); } @Override diff --git a/cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JMXMetricsCollectorFactory.java b/cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollectorFactory.java similarity index 75% rename from cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JMXMetricsCollectorFactory.java rename to cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollectorFactory.java index f506ee21f476..b233f85bfd81 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JMXMetricsCollectorFactory.java +++ b/cdap-master/src/main/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollectorFactory.java @@ -1,5 +1,5 @@ /* - * Copyright © 2022 Cask Data, Inc. + * Copyright © 2022-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -19,9 +19,9 @@ import java.util.Map; /** - * Factory for creating {@link JMXMetricsCollector}. + * Factory for creating {@link JmxMetricsCollector}. */ -public interface JMXMetricsCollectorFactory { +public interface JmxMetricsCollectorFactory { - JMXMetricsCollector create(Map metricTags); + JmxMetricsCollector create(Map metricTags); } diff --git a/cdap-master/src/test/java/io/cdap/cdap/data/runtime/main/TwillRunnableTest.java b/cdap-master/src/test/java/io/cdap/cdap/data/runtime/main/TwillRunnableTest.java index 6a5f31e1a7a4..6c6024d3d8bd 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/data/runtime/main/TwillRunnableTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/data/runtime/main/TwillRunnableTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2016-2019 Cask Data, Inc. + * Copyright © 2016-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -35,7 +35,7 @@ public class TwillRunnableTest { @Test public void testDatasetOpExecutorTwillRunnableInjector() { Injector injector = DatasetOpExecutorServerTwillRunnable.createInjector(CConfiguration.create(), - HBaseConfiguration.create(), ""); + HBaseConfiguration.create(), ""); Store store = injector.getInstance(Store.class); Assert.assertNotNull(store); NamespaceQueryAdmin namespaceQueryAdmin = injector.getInstance(NamespaceQueryAdmin.class); @@ -43,7 +43,7 @@ public void testDatasetOpExecutorTwillRunnableInjector() { } @Test - public void testHBaseTableExporterInjector() { + public void testHbaseTableExporterInjector() { HBaseTableExporter.createInjector(CConfiguration.create(), new Configuration()); } @@ -54,17 +54,20 @@ public void testMessagingServiceTwillRunnableInjector() { @Test public void testMetricsTwillRunnableInjector() { - MetricsTwillRunnable.createGuiceInjector(CConfiguration.create(), HBaseConfiguration.create(), ""); + MetricsTwillRunnable + .createGuiceInjector(CConfiguration.create(), HBaseConfiguration.create(), ""); } @Test public void testMetricsProcessorTwillRunnableInjector() { - MetricsProcessorTwillRunnable.createGuiceInjector(CConfiguration.create(), new Configuration(), "", - new MockTwillContext()); + MetricsProcessorTwillRunnable + .createGuiceInjector(CConfiguration.create(), new Configuration(), "", + new MockTwillContext()); } @Test public void testLogSaverTwillRunnableInjector() { - LogSaverTwillRunnable.createGuiceInjector(CConfiguration.create(), new Configuration(), new MockTwillContext()); + LogSaverTwillRunnable + .createGuiceInjector(CConfiguration.create(), new Configuration(), new MockTwillContext()); } } diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMainTest.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMainTest.java index f39a143225e2..f8711d3f013c 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMainTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMainTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019 Cask Data, Inc. + * Copyright © 2019-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -46,39 +46,42 @@ public class AppFabricServiceMainTest extends MasterServiceMainTestBase { public void testAppFabricService() throws Exception { // Query the system services endpoint - URL url = getRouterBaseURI().resolve("/v3/system/services").toURL(); - HttpResponse response = HttpRequests.execute(HttpRequest.get(url).build(), new DefaultHttpRequestConfig(false)); + URL url = getRouterBaseUri().resolve("/v3/system/services").toURL(); + HttpResponse response = HttpRequests + .execute(HttpRequest.get(url).build(), new DefaultHttpRequestConfig(false)); Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode()); // Deploy an app LocationFactory locationFactory = new LocalLocationFactory(TEMP_FOLDER.newFolder()); - Location deploymentJar = AppJarHelper.createDeploymentJar(locationFactory, AllProgramsApp.class); + Location deploymentJar = AppJarHelper + .createDeploymentJar(locationFactory, AllProgramsApp.class); - URI baseURI = getRouterBaseURI().resolve("/v3/namespaces/default/"); - url = baseURI.resolve("apps").toURL(); + URI baseUri = getRouterBaseUri().resolve("/v3/namespaces/default/"); + url = baseUri.resolve("apps").toURL(); HttpRequestConfig requestConfig = new HttpRequestConfig(0, 0, false); response = HttpRequests.execute( - HttpRequest - .post(url) - .withBody((ContentProvider) deploymentJar::getInputStream) - .addHeader("X-Archive-Name", AllProgramsApp.class.getSimpleName() + "-1.0-SNAPSHOT.jar") - .build(), requestConfig); + HttpRequest + .post(url) + .withBody((ContentProvider) deploymentJar::getInputStream) + .addHeader("X-Archive-Name", AllProgramsApp.class.getSimpleName() + "-1.0-SNAPSHOT.jar") + .build(), requestConfig); Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode()); // Get the application - url = baseURI.resolve("apps/" + AllProgramsApp.NAME).toURL(); + url = baseUri.resolve("apps/" + AllProgramsApp.NAME).toURL(); response = HttpRequests.execute(HttpRequest.get(url).build(), requestConfig); Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode()); - ApplicationDetail appDetail = new Gson().fromJson(response.getResponseBodyAsString(), ApplicationDetail.class); + ApplicationDetail appDetail = new Gson() + .fromJson(response.getResponseBodyAsString(), ApplicationDetail.class); // Do some basic validation only. Assert.assertEquals(AllProgramsApp.NAME, appDetail.getName()); Assert.assertTrue(appDetail.getPrograms() - .stream() - .filter(r -> r.getType() == ProgramType.WORKFLOW) - .anyMatch(r -> AllProgramsApp.NoOpWorkflow.NAME.equals(r.getName()))); + .stream() + .filter(r -> r.getType() == ProgramType.WORKFLOW) + .anyMatch(r -> AllProgramsApp.NoOpWorkflow.NAME.equals(r.getName()))); } } diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/AuthenticationServiceMainTest.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/AuthenticationServiceMainTest.java index c4c92f93cc93..b1d96909833e 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/AuthenticationServiceMainTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/AuthenticationServiceMainTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019 Cask Data, Inc. + * Copyright © 2019-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -70,17 +70,20 @@ private static String realmFile() throws IOException { @Test public void testBasicAuthenticationEnabled() throws IOException { - HttpResponse response = HttpRequests.execute(HttpRequest.get(getAuthenticationBaseURI().toURL()).build() - , new HttpRequestConfig(0, 0, false)); + HttpResponse response = HttpRequests + .execute(HttpRequest.get(getAuthenticationBaseUri().toURL()).build(), + new HttpRequestConfig(0, 0, false)); Assert.assertEquals("basic realm=\"null\"", - response.getHeaders().get("WWW-Authenticate").stream().findFirst().orElse(null)); + response.getHeaders().get("WWW-Authenticate").stream().findFirst().orElse(null)); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, response.getResponseCode()); Injector injector = getServiceMainInstance(AuthenticationServiceMain.class).getInjector(); - DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class); + DiscoveryServiceClient discoveryServiceClient = injector + .getInstance(DiscoveryServiceClient.class); Discoverable authenticationEndpoint = new RandomEndpointStrategy( - () -> discoveryServiceClient.discover(Service.EXTERNAL_AUTHENTICATION)).pick(5, TimeUnit.SECONDS); + () -> discoveryServiceClient.discover(Service.EXTERNAL_AUTHENTICATION)) + .pick(5, TimeUnit.SECONDS); Assert.assertNotNull(authenticationEndpoint); } diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/MasterServiceMainTestBase.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/MasterServiceMainTestBase.java index a924a308a06f..874f5b26254f 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/MasterServiceMainTestBase.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/MasterServiceMainTestBase.java @@ -57,7 +57,7 @@ public class MasterServiceMainTestBase { private static InMemoryZKServer zkServer; private static final Map, ServiceMainManager> SERVICE_MANAGERS = - new LinkedHashMap<>(); + new LinkedHashMap<>(); protected static String[] initArgs; protected static CConfiguration cConf = CConfiguration.create(); @@ -65,7 +65,8 @@ public class MasterServiceMainTestBase { @BeforeClass public static void init() throws Exception { - zkServer = InMemoryZKServer.builder().setAutoCleanDataDir(false).setDataDir(TEMP_FOLDER.newFolder()).build(); + zkServer = InMemoryZKServer.builder().setAutoCleanDataDir(false) + .setDataDir(TEMP_FOLDER.newFolder()).build(); zkServer.startAndWait(); // Set the HDFS directory as well as we are using DFSLocationModule in the master services @@ -76,7 +77,7 @@ public static void init() throws Exception { String keyPass = "testing"; KeyStore keyStore = KeyStores.generatedCertKeyStore(1, keyPass); File pemFile = KeyStoresTest.writePEMFile(TEMP_FOLDER.newFile(), - keyStore, keyStore.aliases().nextElement(), keyPass); + keyStore, keyStore.aliases().nextElement(), keyPass); cConf.setBoolean(Constants.Security.SSL.INTERNAL_ENABLED, true); cConf.set(Constants.Security.SSL.INTERNAL_CERT_PATH, pemFile.getAbsolutePath()); sConf.set(Constants.Security.SSL.INTERNAL_CERT_PASSWORD, keyPass); @@ -84,7 +85,7 @@ public static void init() throws Exception { // Generate a self-signed cert for router SSL keyStore = KeyStores.generatedCertKeyStore(1, keyPass); pemFile = KeyStoresTest.writePEMFile(TEMP_FOLDER.newFile(), keyStore, - keyStore.aliases().nextElement(), keyPass); + keyStore.aliases().nextElement(), keyPass); cConf.setBoolean(Constants.Security.SSL.EXTERNAL_ENABLED, true); cConf.set(Constants.Security.Router.SSL_CERT_PATH, pemFile.getAbsolutePath()); sConf.set(Constants.Security.Router.SSL_CERT_PASSWORD, keyPass); @@ -92,9 +93,9 @@ public static void init() throws Exception { // Set all bind address to localhost String localhost = InetAddress.getLoopbackAddress().getHostName(); StreamSupport.stream(CConfiguration.create().spliterator(), false) - .map(Map.Entry::getKey) - .filter(s -> s.endsWith(".bind.address")) - .forEach(key -> cConf.set(key, localhost)); + .map(Map.Entry::getKey) + .filter(s -> s.endsWith(".bind.address")) + .forEach(key -> cConf.set(key, localhost)); // Set router to bind to random port cConf.setInt(Constants.Router.ROUTER_PORT, 0); @@ -104,22 +105,22 @@ public static void init() throws Exception { // Use remote fetcher for runtime server cConf.setClass(Constants.RuntimeMonitor.RUN_RECORD_FETCHER_CLASS, - RemoteProgramRunRecordFetcher.class, ProgramRunRecordFetcher.class); + RemoteProgramRunRecordFetcher.class, ProgramRunRecordFetcher.class); // Set JMX server port for JMXMetricsCollector - cConf.setInt(Constants.JMXMetricsCollector.SERVER_PORT, 11022); + cConf.setInt(Constants.JmxMetricsCollector.SERVER_PORT, 11022); // Starting all master service mains List>> serviceMainClasses = new ArrayList<>( - Arrays.asList(RouterServiceMain.class, - MessagingServiceMain.class, - MetricsServiceMain.class, - LogsServiceMain.class, - MetadataServiceMain.class, - RuntimeServiceMain.class, - AppFabricServiceMain.class, - SupportBundleServiceMain.class, - SystemMetricsExporterServiceMain.class, - ArtifactCacheServiceMain.class, - TetheringAgentServiceMain.class)); + Arrays.asList(RouterServiceMain.class, + MessagingServiceMain.class, + MetricsServiceMain.class, + LogsServiceMain.class, + MetadataServiceMain.class, + RuntimeServiceMain.class, + AppFabricServiceMain.class, + SupportBundleServiceMain.class, + SystemMetricsExporterServiceMain.class, + ArtifactCacheServiceMain.class, + TetheringAgentServiceMain.class)); if (SecurityUtil.isManagedSecurity(cConf)) { serviceMainClasses.add(AuthenticationServiceMain.class); @@ -132,28 +133,30 @@ public static void init() throws Exception { @AfterClass public static void finish() { // Reverse stop services - Lists.reverse(new ArrayList<>(SERVICE_MANAGERS.keySet())).forEach(MasterServiceMainTestBase::stopService); + Lists.reverse(new ArrayList<>(SERVICE_MANAGERS.keySet())) + .forEach(MasterServiceMainTestBase::stopService); zkServer.stopAndWait(); } /** - * Instantiate and start up the given service main class and add it to the map from {@link AbstractServiceMain} - * to {@link ServiceMainManager} + * Instantiate and start up the given service main class and add it to the map from {@link + * AbstractServiceMain} to {@link ServiceMainManager} * * @param serviceMainClass the service main class to start - * @param the type of service main class (e.g. {@link AppFabricServiceMain}) + * @param the type of service main class (e.g. {@link AppFabricServiceMain}) * @throws Exception if failed to start service main */ - protected static void startService(Class serviceMainClass) throws Exception { + protected static void startService(Class serviceMainClass) + throws Exception { SERVICE_MANAGERS.put(serviceMainClass, runMain(cConf, sConf, serviceMainClass)); } /** - * Stop the given service main and remove it from the map from {@link AbstractServiceMain} - * to {@link ServiceMainManager} + * Stop the given service main and remove it from the map from {@link AbstractServiceMain} to + * {@link ServiceMainManager} * * @param serviceMainClass the service main class to stop - * @param the type of service main class (e.g. {@link AppFabricServiceMain}) + * @param the type of service main class (e.g. {@link AppFabricServiceMain}) */ protected static void stopService(Class serviceMainClass) { final ServiceMainManager serviceMainManager = SERVICE_MANAGERS.remove(serviceMainClass); @@ -163,8 +166,9 @@ protected static void stopService(Class servi /** * Returns the base URI for the router. */ - static URI getRouterBaseURI() { - NettyRouter router = getServiceMainInstance(RouterServiceMain.class).getInjector().getInstance(NettyRouter.class); + static URI getRouterBaseUri() { + NettyRouter router = getServiceMainInstance(RouterServiceMain.class).getInjector() + .getInstance(NettyRouter.class); InetSocketAddress addr = router.getBoundAddress().orElseThrow(IllegalStateException::new); return URI.create(String.format("https://%s:%d/", addr.getHostName(), addr.getPort())); } @@ -172,8 +176,9 @@ static URI getRouterBaseURI() { /** * Returns the base URI for the authentication. */ - static URI getAuthenticationBaseURI() { - ExternalAuthenticationServer externalAuthenticationServer = getServiceMainInstance(AuthenticationServiceMain.class) + static URI getAuthenticationBaseUri() { + ExternalAuthenticationServer externalAuthenticationServer = getServiceMainInstance( + AuthenticationServiceMain.class) .getInjector().getInstance(ExternalAuthenticationServer.class); InetSocketAddress addr = externalAuthenticationServer.getSocketAddress(); return URI.create(String.format("https://%s:%d/", addr.getHostName(), addr.getPort())); @@ -186,17 +191,19 @@ static T getServiceMainInstance(Class service ServiceMainManager manager = SERVICE_MANAGERS.get(serviceMainClass); AbstractServiceMain instance = manager.getInstance(); if (!serviceMainClass.isInstance(instance)) { - throw new IllegalArgumentException("Mismatch manager class." + serviceMainClass + " != " + instance); + throw new IllegalArgumentException( + "Mismatch manager class." + serviceMainClass + " != " + instance); } //noinspection unchecked return (T) instance; } - protected static ServiceMainManager runMain(CConfiguration cConf, - SConfiguration sConf, - Class serviceMainClass) - throws Exception { + protected static ServiceMainManager runMain( + CConfiguration cConf, + SConfiguration sConf, + Class serviceMainClass) + throws Exception { return runMain(cConf, sConf, serviceMainClass, serviceMainClass.getSimpleName()); } @@ -204,43 +211,44 @@ protected static ServiceMainManager runMain(C * Instantiate and start the given service main class. * * @param serviceMainClass the service main class to start - * @param dataDir the directory to use for data. - * @param type of the service main class + * @param dataDir the directory to use for data. + * @param type of the service main class * @return A {@link ServiceMainManager} to interface with the service instance * @throws Exception if failed to start the service */ - protected static ServiceMainManager runMain(CConfiguration cConf, - SConfiguration sConf, - Class serviceMainClass, - String dataDir) throws Exception { + protected static ServiceMainManager runMain( + CConfiguration cConf, + SConfiguration sConf, + Class serviceMainClass, + String dataDir) throws Exception { // Set a unique local data directory for each service - CConfiguration serviceCConf = CConfiguration.copy(cConf); + CConfiguration serviceCconf = CConfiguration.copy(cConf); File dataDirFolder = new File(TEMP_FOLDER.getRoot(), dataDir); boolean dataAlreadyExists = true; if (!dataDirFolder.exists()) { dataDirFolder = TEMP_FOLDER.newFolder(dataDir); dataAlreadyExists = false; } - serviceCConf.set(Constants.CFG_LOCAL_DATA_DIR, dataDirFolder.getAbsolutePath()); + serviceCconf.set(Constants.CFG_LOCAL_DATA_DIR, dataDirFolder.getAbsolutePath()); // Create StructuredTable stores before starting the main. // The registry will be preserved and pick by the main class. // Also try to create metadata tables. if (!dataAlreadyExists) { - new StorageMain().createStorage(serviceCConf); + new StorageMain().createStorage(serviceCconf); } // Write the "cdap-site.xml" and pass the directory to the main service File confDir = TEMP_FOLDER.newFolder(); try (Writer writer = Files.newBufferedWriter(new File(confDir, "cdap-site.xml").toPath())) { - serviceCConf.writeXml(writer); + serviceCconf.writeXml(writer); } try (Writer writer = Files.newBufferedWriter(new File(confDir, "cdap-security.xml").toPath())) { sConf.writeXml(writer); } - initArgs = new String[] { "--env=mock", "--conf=" + confDir.getAbsolutePath() }; + initArgs = new String[]{"--env=mock", "--conf=" + confDir.getAbsolutePath()}; T service = serviceMainClass.newInstance(); service.init(initArgs); service.start(); @@ -265,6 +273,7 @@ public void cancel() { * @param type of the service main class */ private interface ServiceMainManager extends Cancellable { + T getInstance(); } } diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMainTest.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMainTest.java index 8d945fe6fd9c..1b9a0287a086 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMainTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMainTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019 Cask Data, Inc. + * Copyright © 2019-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -76,9 +76,11 @@ * Unit test for {@link PreviewServiceMain}. */ public class PreviewServiceMainTest extends MasterServiceMainTestBase { + private static final Gson GSON = new Gson(); - private static final Type ARTIFACT_SUMMARY_LIST = new TypeToken>() { }.getType(); + private static final Type ARTIFACT_SUMMARY_LIST = new TypeToken>() { + }.getType(); @ClassRule public static TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -122,8 +124,9 @@ public void testPreviewSimpleApp() throws Exception { // Run a preview ArtifactSummary artifactSummary = new ArtifactSummary(artifactName, artifactVersion); - PreviewConfig previewConfig = new PreviewConfig(PreviewTestApp.TestWorkflow.NAME, ProgramType.WORKFLOW, - Collections.emptyMap(), 2); + PreviewConfig previewConfig = new PreviewConfig(PreviewTestApp.TestWorkflow.NAME, + ProgramType.WORKFLOW, + Collections.emptyMap(), 2); AppRequest appRequest = new AppRequest<>(artifactSummary, null, previewConfig); ApplicationId previewId = runPreview(appRequest); @@ -131,17 +134,18 @@ public void testPreviewSimpleApp() throws Exception { waitForPreview(previewId); // Verify the result of preview run - URL url = getRouterBaseURI() - .resolve(String.format("/v3/namespaces/default/previews/%s/tracers/%s", - previewId.getApplication(), PreviewTestApp.TRACER_NAME)).toURL(); - HttpResponse response = HttpRequests.execute(HttpRequest.get(url).build(), getHttpRequestConfig()); + URL url = getRouterBaseUri() + .resolve(String.format("/v3/namespaces/default/previews/%s/tracers/%s", + previewId.getApplication(), PreviewTestApp.TRACER_NAME)).toURL(); + HttpResponse response = HttpRequests + .execute(HttpRequest.get(url).build(), getHttpRequestConfig()); Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode()); Map> tracerData = GSON.fromJson(response.getResponseBodyAsString(), - new TypeToken>>() { - }.getType()); + new TypeToken>>() { + }.getType()); Assert.assertEquals(Collections.singletonMap(PreviewTestApp.TRACER_KEY, - Collections.singletonList(PreviewTestApp.TRACER_VAL)), - tracerData); + Collections.singletonList(PreviewTestApp.TRACER_VAL)), + tracerData); // Clean up deleteArtfiact(artifactName, artifactVersion); @@ -151,7 +155,8 @@ public void testPreviewSimpleApp() throws Exception { public void testPreviewAppWithPlugin() throws Exception { // Build the app LocationFactory locationFactory = new LocalLocationFactory(TEMP_FOLDER.newFolder()); - Location appJar = AppJarHelper.createDeploymentJar(locationFactory, PreviewTestAppWithPlugin.class); + Location appJar = AppJarHelper + .createDeploymentJar(locationFactory, PreviewTestAppWithPlugin.class); String appArtifactName = PreviewTestAppWithPlugin.class.getSimpleName() + "_artifact"; String artifactVersion = "1.0.0-SNAPSHOT"; @@ -161,32 +166,38 @@ public void testPreviewAppWithPlugin() throws Exception { // Build plugin artifact Manifest manifest = new Manifest(); - manifest.getMainAttributes().put(ManifestFields.EXPORT_PACKAGE, ConstantCallable.class.getPackage().getName()); - Location pluginJar = PluginJarHelper.createPluginJar(locationFactory, manifest, ConstantCallable.class); + manifest.getMainAttributes() + .put(ManifestFields.EXPORT_PACKAGE, ConstantCallable.class.getPackage().getName()); + Location pluginJar = PluginJarHelper + .createPluginJar(locationFactory, manifest, ConstantCallable.class); // Deploy plug artifact String pluginArtifactName = ConstantCallable.class.getSimpleName() + "_artifact"; URL pluginArtifactUrl = - getRouterBaseURI().resolve(String.format("/v3/namespaces/default/artifacts/%s", pluginArtifactName)).toURL(); + getRouterBaseUri() + .resolve(String.format("/v3/namespaces/default/artifacts/%s", pluginArtifactName)) + .toURL(); response = HttpRequests.execute( - HttpRequest - .post(pluginArtifactUrl) - .withBody((ContentProvider) pluginJar::getInputStream) - .addHeader("Artifact-Extends", String.format("%s[1.0.0-SNAPSHOT,10.0.0]", appArtifactName)) - .addHeader("Artifact-Version", artifactVersion) - .build(), getHttpRequestConfig()); + HttpRequest + .post(pluginArtifactUrl) + .withBody((ContentProvider) pluginJar::getInputStream) + .addHeader("Artifact-Extends", + String.format("%s[1.0.0-SNAPSHOT,10.0.0]", appArtifactName)) + .addHeader("Artifact-Version", artifactVersion) + .build(), getHttpRequestConfig()); Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode()); // Run a preview String expectedOutput = "output_value"; ArtifactId appArtifactId = new ArtifactId(appArtifactName, new ArtifactVersion(artifactVersion), - ArtifactScope.USER); + ArtifactScope.USER); ArtifactSummary artifactSummary = ArtifactSummary.from(appArtifactId); - PreviewConfig previewConfig = new PreviewConfig(PreviewTestAppWithPlugin.TestWorkflow.NAME, ProgramType.WORKFLOW, - Collections.emptyMap(), 2); + PreviewConfig previewConfig = new PreviewConfig(PreviewTestAppWithPlugin.TestWorkflow.NAME, + ProgramType.WORKFLOW, + Collections.emptyMap(), 2); PreviewTestAppWithPlugin.Conf appConf = - new PreviewTestAppWithPlugin.Conf(ConstantCallable.NAME, - Collections.singletonMap("val", expectedOutput)); + new PreviewTestAppWithPlugin.Conf(ConstantCallable.NAME, + Collections.singletonMap("val", expectedOutput)); AppRequest appRequest = new AppRequest<>(artifactSummary, appConf, previewConfig); ApplicationId previewId = runPreview(appRequest); @@ -194,31 +205,35 @@ public void testPreviewAppWithPlugin() throws Exception { waitForPreview(previewId); // Verify the result of preview run - URL url = getRouterBaseURI().resolve(String.format("/v3/namespaces/default/previews/%s/tracers/%s", - previewId.getApplication(), PreviewTestApp.TRACER_NAME)).toURL(); + URL url = getRouterBaseUri() + .resolve(String.format("/v3/namespaces/default/previews/%s/tracers/%s", + previewId.getApplication(), PreviewTestApp.TRACER_NAME)).toURL(); response = HttpRequests.execute(HttpRequest.get(url).build(), getHttpRequestConfig()); Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode()); Map> tracerData = GSON.fromJson(response.getResponseBodyAsString(), - new TypeToken>>() { - }.getType()); + new TypeToken>>() { + }.getType()); Assert.assertEquals(Collections.singletonMap(PreviewTestAppWithPlugin.TRACER_KEY, - Collections.singletonList(expectedOutput)), - tracerData); + Collections.singletonList(expectedOutput)), + tracerData); } /** * Wait for preview to complete with a deadline */ private void waitForPreview(ApplicationId previewId) throws MalformedURLException, - java.util.concurrent.TimeoutException, InterruptedException, java.util.concurrent.ExecutionException { - URL statusUrl = getRouterBaseURI().resolve(String.format("/v3/namespaces/default/previews/%s/status", - previewId.getApplication())).toURL(); + java.util.concurrent.TimeoutException, InterruptedException, java.util.concurrent.ExecutionException { + URL statusUrl = getRouterBaseUri() + .resolve(String.format("/v3/namespaces/default/previews/%s/status", + previewId.getApplication())).toURL(); Tasks.waitFor(PreviewStatus.Status.COMPLETED, () -> { - HttpResponse statusResponse = HttpRequests.execute(HttpRequest.get(statusUrl).build(), getHttpRequestConfig()); + HttpResponse statusResponse = HttpRequests + .execute(HttpRequest.get(statusUrl).build(), getHttpRequestConfig()); if (statusResponse.getResponseCode() != 200) { return null; } - PreviewStatus previewStatus = GSON.fromJson(statusResponse.getResponseBodyAsString(), PreviewStatus.class); + PreviewStatus previewStatus = GSON + .fromJson(statusResponse.getResponseBodyAsString(), PreviewStatus.class); return previewStatus.getStatus(); }, 2, TimeUnit.MINUTES); } @@ -228,10 +243,12 @@ private void waitForPreview(ApplicationId previewId) throws MalformedURLExceptio */ private ApplicationId runPreview(AppRequest appRequest) throws IOException { URL url; - url = getRouterBaseURI().resolve("/v3/namespaces/default/previews").toURL(); - HttpResponse response = HttpRequests.execute(HttpRequest.post(url).withBody(GSON.toJson(appRequest)).build(), - getHttpRequestConfig()); - Assert.assertEquals(response.getResponseBodyAsString(), HttpURLConnection.HTTP_OK, response.getResponseCode()); + url = getRouterBaseUri().resolve("/v3/namespaces/default/previews").toURL(); + HttpResponse response = HttpRequests + .execute(HttpRequest.post(url).withBody(GSON.toJson(appRequest)).build(), + getHttpRequestConfig()); + Assert.assertEquals(response.getResponseBodyAsString(), HttpURLConnection.HTTP_OK, + response.getResponseCode()); return GSON.fromJson(response.getResponseBodyAsString(), ApplicationId.class); } @@ -239,17 +256,20 @@ private ApplicationId runPreview(AppRequest appRequest) throws IOException { /** * Deploy the given application in default namespace */ - private void deployArtifact(Location artifactLocation, String artifactName, String artifactVersion) - throws IOException { + private void deployArtifact(Location artifactLocation, String artifactName, + String artifactVersion) + throws IOException { HttpRequestConfig requestConfig = getHttpRequestConfig(); - URL url = getRouterBaseURI().resolve(String.format("/v3/namespaces/default/artifacts/%s", artifactName)).toURL(); + URL url = getRouterBaseUri() + .resolve(String.format("/v3/namespaces/default/artifacts/%s", artifactName)).toURL(); HttpResponse response = HttpRequests.execute( - HttpRequest.post(url) - .withBody((ContentProvider) artifactLocation::getInputStream) - .addHeader("Artifact-Version", artifactVersion) - .build(), - requestConfig); - Assert.assertEquals(response.getResponseBodyAsString(), HttpURLConnection.HTTP_OK, response.getResponseCode()); + HttpRequest.post(url) + .withBody((ContentProvider) artifactLocation::getInputStream) + .addHeader("Artifact-Version", artifactVersion) + .build(), + requestConfig); + Assert.assertEquals(response.getResponseBodyAsString(), HttpURLConnection.HTTP_OK, + response.getResponseCode()); } /** @@ -257,10 +277,12 @@ private void deployArtifact(Location artifactLocation, String artifactName, Stri */ private void deleteArtfiact(String artifactName, String artifactVersion) throws IOException { HttpRequestConfig requestConfig = getHttpRequestConfig(); - URL url = getRouterBaseURI().resolve(String.format("/v3/namespaces/default/artifacts/%s/versions/%s", - artifactName, artifactVersion)).toURL(); + URL url = getRouterBaseUri() + .resolve(String.format("/v3/namespaces/default/artifacts/%s/versions/%s", + artifactName, artifactVersion)).toURL(); HttpResponse response = HttpRequests.execute(HttpRequest.delete(url).build(), requestConfig); - Assert.assertEquals(response.getResponseBodyAsString(), HttpURLConnection.HTTP_OK, response.getResponseCode()); + Assert.assertEquals(response.getResponseBodyAsString(), HttpURLConnection.HTTP_OK, + response.getResponseCode()); } /** @@ -268,15 +290,19 @@ private void deleteArtfiact(String artifactName, String artifactVersion) throws */ private void deleteAllArtifact() throws IOException { HttpRequestConfig requestConfig = getHttpRequestConfig(); - URL url = getRouterBaseURI().resolve(String.format("/v3/namespaces/default/artifacts")).toURL(); + URL url = getRouterBaseUri().resolve(String.format("/v3/namespaces/default/artifacts")).toURL(); HttpResponse response = HttpRequests.execute(HttpRequest.get(url).build(), requestConfig); - Assert.assertEquals(response.getResponseBodyAsString(), HttpURLConnection.HTTP_OK, response.getResponseCode()); - List summaryList = GSON.fromJson(response.getResponseBodyAsString(), ARTIFACT_SUMMARY_LIST); + Assert.assertEquals(response.getResponseBodyAsString(), HttpURLConnection.HTTP_OK, + response.getResponseCode()); + List summaryList = GSON + .fromJson(response.getResponseBodyAsString(), ARTIFACT_SUMMARY_LIST); for (ArtifactSummary summary : summaryList) { - url = getRouterBaseURI().resolve(String.format("/v3/namespaces/default/artifacts/%s/versions/%s", - summary.getName(), summary.getVersion())).toURL(); + url = getRouterBaseUri() + .resolve(String.format("/v3/namespaces/default/artifacts/%s/versions/%s", + summary.getName(), summary.getVersion())).toURL(); response = HttpRequests.execute(HttpRequest.delete(url).build(), requestConfig); - Assert.assertEquals(response.getResponseBodyAsString(), HttpURLConnection.HTTP_OK, response.getResponseCode()); + Assert.assertEquals(response.getResponseBodyAsString(), HttpURLConnection.HTTP_OK, + response.getResponseCode()); } } @@ -288,6 +314,7 @@ private HttpRequestConfig getHttpRequestConfig() { } private static class NoOpArtifactManager implements ArtifactManager { + @Override public List listArtifacts() throws IOException, AccessException { return Collections.emptyList(); @@ -300,15 +327,15 @@ public List listArtifacts(String namespace) throws IOException, Ac @Override public CloseableClassLoader createClassLoader(ArtifactInfo artifactInfo, - @Nullable ClassLoader parentClassLoader) - throws IOException, AccessException { + @Nullable ClassLoader parentClassLoader) + throws IOException, AccessException { return null; } @Override public CloseableClassLoader createClassLoader(String namespace, ArtifactInfo artifactInfo, - @Nullable ClassLoader parentClassLoader) - throws IOException, AccessException { + @Nullable ClassLoader parentClassLoader) + throws IOException, AccessException { return null; } } diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/RouterServiceMainWithSecurityDisabledTest.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/RouterServiceMainWithSecurityDisabledTest.java index c182e9836d14..0030490daecf 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/RouterServiceMainWithSecurityDisabledTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/RouterServiceMainWithSecurityDisabledTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019 Cask Data, Inc. + * Copyright © 2019-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -29,8 +29,9 @@ public class RouterServiceMainWithSecurityDisabledTest extends MasterServiceMain @Test public void testRouterServiceWithAuthenticationDisabled() throws Exception { - URL url = getRouterBaseURI().resolve("/").toURL(); - HttpResponse response = HttpRequests.execute(HttpRequest.get(url).build(), new DefaultHttpRequestConfig(false)); + URL url = getRouterBaseUri().resolve("/").toURL(); + HttpResponse response = HttpRequests + .execute(HttpRequest.get(url).build(), new DefaultHttpRequestConfig(false)); Assert.assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode()); } diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/RouterServiceMainWithSecurityEnabledTest.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/RouterServiceMainWithSecurityEnabledTest.java index 0845635d1dce..9c6f163297b5 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/RouterServiceMainWithSecurityEnabledTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/RouterServiceMainWithSecurityEnabledTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019 Cask Data, Inc. + * Copyright © 2019-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -63,8 +63,9 @@ private static String realmFile() throws IOException { @Test public void testRouterServiceWithAuthenticationEnabled() throws Exception { - URL url = getRouterBaseURI().resolve("/").toURL(); - HttpResponse response = HttpRequests.execute(HttpRequest.get(url).build(), new DefaultHttpRequestConfig(false)); + URL url = getRouterBaseUri().resolve("/").toURL(); + HttpResponse response = HttpRequests + .execute(HttpRequest.get(url).build(), new DefaultHttpRequestConfig(false)); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, response.getResponseCode()); } diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SupportBundleServiceMainTest.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SupportBundleServiceMainTest.java index d6e8f5ea24f7..125466f8e135 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SupportBundleServiceMainTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SupportBundleServiceMainTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019 Cask Data, Inc. + * Copyright © 2019-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -32,9 +32,9 @@ public class SupportBundleServiceMainTest extends MasterServiceMainTestBase { @Test public void testSupportBundleService() throws Exception { - URL url = getRouterBaseURI().resolve("/v3/support/bundles").toURL(); + URL url = getRouterBaseUri().resolve("/v3/support/bundles").toURL(); HttpResponse response = HttpRequests.execute(HttpRequest.post(url).build(), - new DefaultHttpRequestConfig(false)); + new DefaultHttpRequestConfig(false)); Assert.assertEquals(HttpURLConnection.HTTP_CREATED, response.getResponseCode()); String uuid = response.getResponseBodyAsString(); diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMainTest.java b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMainTest.java index bebc13e6a276..3ce04353d3a4 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMainTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/environment/k8s/SystemMetricsExporterServiceMainTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2022 Cask Data, Inc. + * Copyright © 2022-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -18,8 +18,8 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Injector; -import io.cdap.cdap.metrics.jmx.JMXMetricsCollector; -import io.cdap.cdap.metrics.jmx.JMXMetricsCollectorFactory; +import io.cdap.cdap.metrics.jmx.JmxMetricsCollector; +import io.cdap.cdap.metrics.jmx.JmxMetricsCollectorFactory; import java.util.HashMap; import java.util.Map; import org.junit.Assert; @@ -32,10 +32,11 @@ public class SystemMetricsExporterServiceMainTest extends MasterServiceMainTestB @Test public void testSystemMetricsExporterService() { - Injector injector = getServiceMainInstance(SystemMetricsExporterServiceMain.class).getInjector(); - JMXMetricsCollectorFactory factory = injector.getInstance(JMXMetricsCollectorFactory.class); + Injector injector = getServiceMainInstance(SystemMetricsExporterServiceMain.class) + .getInjector(); + JmxMetricsCollectorFactory factory = injector.getInstance(JmxMetricsCollectorFactory.class); Map metricTags = ImmutableMap.of("key1", "value1", "key2", "value2"); - JMXMetricsCollector metricsCollector = factory.create(metricTags); + JmxMetricsCollector metricsCollector = factory.create(metricTags); // JMX server isn't running, but that shouldn't raise exceptions, errors will be logged. metricsCollector.startAndWait(); Assert.assertTrue(metricsCollector.isRunning()); diff --git a/cdap-master/src/test/java/io/cdap/cdap/metrics/jmx/JMXMetricsCollectorTest.java b/cdap-master/src/test/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollectorTest.java similarity index 81% rename from cdap-master/src/test/java/io/cdap/cdap/metrics/jmx/JMXMetricsCollectorTest.java rename to cdap-master/src/test/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollectorTest.java index d73737d5bc19..af9495a68d15 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/metrics/jmx/JMXMetricsCollectorTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/metrics/jmx/JmxMetricsCollectorTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2021 Cask Data, Inc. + * Copyright © 2021-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -47,9 +47,10 @@ import org.mockito.MockitoAnnotations; /** - * Unit tests for {@link JMXMetricsCollector}. + * Unit tests for {@link JmxMetricsCollector}. */ -public class JMXMetricsCollectorTest { +public class JmxMetricsCollectorTest { + private static final int SERVER_PORT = 11023; private static JMXConnectorServer svr; @Mock @@ -57,16 +58,16 @@ public class JMXMetricsCollectorTest { @BeforeClass public static void setupClass() throws IOException { - svr = createJMXConnectorServer(SERVER_PORT); + svr = createJmxConnectorServer(SERVER_PORT); svr.start(); Assert.assertTrue(svr.isActive()); } - private static JMXConnectorServer createJMXConnectorServer(int port) throws IOException { + private static JMXConnectorServer createJmxConnectorServer(int port) throws IOException { LocateRegistry.createRegistry(port); MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); JMXServiceURL url = new JMXServiceURL( - String.format("service:jmx:rmi://localhost/jndi/rmi://localhost:%d/jmxrmi", port)); + String.format("service:jmx:rmi://localhost/jndi/rmi://localhost:%d/jmxrmi", port)); return JMXConnectorServerFactory.newJMXConnectorServer(url, null, mbs); } @@ -83,20 +84,20 @@ public void beforeEach() { @Test(expected = IllegalArgumentException.class) public void testInvalidPortInConfig() throws Exception { CConfiguration cConf = CConfiguration.create(); - cConf.setInt(Constants.JMXMetricsCollector.SERVER_PORT, -1); - cConf.setInt(Constants.JMXMetricsCollector.POLL_INTERVAL_SECS, 1); + cConf.setInt(Constants.JmxMetricsCollector.SERVER_PORT, -1); + cConf.setInt(Constants.JmxMetricsCollector.POLL_INTERVAL_SECS, 1); Map metricTags = ImmutableMap.of("key1", "value1", "key2", "value2"); - new JMXMetricsCollector(cConf, publisher, metricTags); + new JmxMetricsCollector(cConf, publisher, metricTags); } @Test public void testNumberOfMetricsEmitted() throws InterruptedException, MalformedURLException, - ExecutionException, TimeoutException { + ExecutionException, TimeoutException { CConfiguration cConf = CConfiguration.create(); - cConf.setInt(Constants.JMXMetricsCollector.SERVER_PORT, SERVER_PORT); - cConf.setInt(Constants.JMXMetricsCollector.POLL_INTERVAL_SECS, 1); + cConf.setInt(Constants.JmxMetricsCollector.SERVER_PORT, SERVER_PORT); + cConf.setInt(Constants.JmxMetricsCollector.POLL_INTERVAL_SECS, 1); Map metricTags = ImmutableMap.of("key1", "value1", "key2", "value2"); - JMXMetricsCollector jmxMetrics = new JMXMetricsCollector(cConf, publisher, metricTags); + JmxMetricsCollector jmxMetrics = new JmxMetricsCollector(cConf, publisher, metricTags); jmxMetrics.startAndWait(); verify(publisher, times(1)).initialize(); // Poll should run at 0, 1. 2 secs buffer. diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialIdentity.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialIdentity.java index 573016e9b645..8f154ea1dc39 100644 --- a/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialIdentity.java +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialIdentity.java @@ -16,33 +16,38 @@ package io.cdap.cdap.proto.credential; -import io.cdap.cdap.proto.id.CredentialProfileId; - /** * Defines an identity for credential provisioning. */ public class CredentialIdentity { - private final CredentialProfileId credentialProfile; + private final String profileNamespace; + private final String profileName; private final String identity; private final String secureValue; /** * Constructs an identity. * - * @param credentialProfile The associated profile. - * @param identity The identity. - * @param secureValue The secure value to store for the identity. + * @param profileNamespace The namespace of the associated profile. + * @param profileName The name of the associated profile. + * @param identity The identity. + * @param secureValue The secure value to store for the identity. */ - public CredentialIdentity(CredentialProfileId credentialProfile, String identity, + public CredentialIdentity(String profileNamespace, String profileName, String identity, String secureValue) { - this.credentialProfile = credentialProfile; + this.profileNamespace = profileNamespace; + this.profileName = profileName; this.identity = identity; this.secureValue = secureValue; } - public CredentialProfileId getCredentialProfile() { - return credentialProfile; + public String getProfileNamespace() { + return profileNamespace; + } + + public String getProfileName() { + return profileName; } public String getIdentity() { diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProfile.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProfile.java index 1acd148f2449..cbeb83dd625e 100644 --- a/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProfile.java +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProfile.java @@ -31,8 +31,8 @@ public class CredentialProfile { * Constructs a profile. * * @param credentialProviderType The credential provider type to use for provisioning. - * @param description A description for the profile. - * @param properties Properties for the profile. + * @param description A description for the profile. + * @param properties Properties for the profile. */ public CredentialProfile(String credentialProviderType, String description, Map properties) { diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProvider.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProvider.java new file mode 100644 index 000000000000..cc7d79e32e4e --- /dev/null +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProvider.java @@ -0,0 +1,49 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.proto.credential; + +import java.io.IOException; + +/** + * Provides a credential based on a profile and identity. + */ +public interface CredentialProvider { + + /** + * Provisions a short-lived credential for the provided identity using the provided identity. + * + * @param namespace The identity namespace. + * @param identityName The identity name. + * @return A short-lived credential. + * @throws CredentialProvisioningException If provisioning the credential fails. + * @throws IOException If any transport errors occur. + * @throws NotFoundException If the profile or identity are not found. + */ + ProvisionedCredential provision(String namespace, String identityName) + throws CredentialProvisioningException, IOException, NotFoundException; + + /** + * Validates the provided identity. + * + * @param identity The identity to validate. + * @throws IdentityValidationException If validation fails. + * @throws IOException If any transport errors occur. + * @throws NotFoundException If the profile is not found. + */ + void validateIdentity(CredentialIdentity identity) throws IdentityValidationException, + IOException, NotFoundException; +} diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProvisioningException.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProvisioningException.java new file mode 100644 index 000000000000..47aecf74425d --- /dev/null +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/CredentialProvisioningException.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.proto.credential; + +/** + * Exception thrown during credential provisioning. + */ +public class CredentialProvisioningException extends Exception { + + /** + * Creates a new credential provisioning exception. + * + * @param message The message for the provisioning failure. + */ + public CredentialProvisioningException(String message) { + super(message); + } +} diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/IdentityValidationException.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/IdentityValidationException.java new file mode 100644 index 000000000000..67c072b7ff71 --- /dev/null +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/IdentityValidationException.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.proto.credential; + +/** + * Exception thrown during identity validation. + */ +public class IdentityValidationException extends Exception { + + /** + * Creates a new identity validation exception. + * + * @param cause The cause of identity validation failure. + */ + public IdentityValidationException(Throwable cause) { + super("Failed to validate identity", cause); + } +} diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NotFoundException.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NotFoundException.java new file mode 100644 index 000000000000..797082afbfab --- /dev/null +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/NotFoundException.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.proto.credential; + +/** + * An exception which is thrown when a resource is not found. + */ +public class NotFoundException extends Exception { + + /** + * Exception denoting a resource was not found. + * + * @param message The message for the exception. + */ + public NotFoundException(String message) { + super(message); + } +} diff --git a/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/ProvisionedCredential.java b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/ProvisionedCredential.java new file mode 100644 index 000000000000..5f4e18b393aa --- /dev/null +++ b/cdap-proto/src/main/java/io/cdap/cdap/proto/credential/ProvisionedCredential.java @@ -0,0 +1,57 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.proto.credential; + +import java.time.Instant; + +/** + * A credential provisioned by the {@link CredentialProvider}. + */ +public class ProvisionedCredential { + + private final String value; + private final Instant expiration; + + /** + * Creates a provisioned credential. + * + * @param value The credential value. + * @param expiration The expiration of the credential as an {@link Instant}. + */ + public ProvisionedCredential(String value, Instant expiration) { + this.value = value; + this.expiration = expiration; + } + + /** + * Returns the credential value. + * + * @return The credential value. + */ + public String get() { + return value; + } + + /** + * Returns the instant the credential expires. + * + * @return The credential expiration instant. + */ + public Instant getExpiration() { + return expiration; + } +} diff --git a/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/CredentialProvider.java b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/CredentialProvider.java new file mode 100644 index 000000000000..d30d9ba9886b --- /dev/null +++ b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/CredentialProvider.java @@ -0,0 +1,65 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.spi.credential; + +import io.cdap.cdap.proto.credential.CredentialIdentity; +import io.cdap.cdap.proto.credential.CredentialProvisioningException; +import io.cdap.cdap.proto.credential.ProvisionedCredential; +import io.cdap.cdap.proto.credential.CredentialProfile; + +/** + * Defines an SPI for provisioning a credential. + */ +public interface CredentialProvider { + + /** + * Returns the name of the credential provider. + * + * @return the name of the credential provider. + */ + String getName(); + + /** + * Initializes the credential provider. This is guaranteed to be called once before any other + * methods (except for {@link CredentialProvider#getName()} are called. + * + * @param context The credential provider context to initialize with. + */ + void initialize(CredentialProviderContext context); + + /** + * Provisions a short-lived credential for the provided identity using the provided credential + * profile. + * + * @param profile The credential profile to use. + * @param identity The credential identity to use. + * @return A credential provisioned using the specified profile and identity. + * @throws CredentialProvisioningException If the credential provisioning fails. + */ + ProvisionedCredential provision(CredentialProfile profile, CredentialIdentity identity) + throws CredentialProvisioningException; + + /** + * Validates a credential profile for this specific extension. Implementations of this function + * should only validate fields specific to the profile type statically and should not be used to + * provision a credential. + * + * @param profile The profile to validate. + * @throws ProfileValidationException If validation fails. + */ + void validateProfile(CredentialProfile profile) throws ProfileValidationException; +} diff --git a/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/CredentialProviderContext.java b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/CredentialProviderContext.java new file mode 100644 index 000000000000..7507c0f4f5e8 --- /dev/null +++ b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/CredentialProviderContext.java @@ -0,0 +1,37 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.spi.credential; + +import java.util.Map; + +/** + * Context passed to {@link CredentialProvider} during initialization. + */ +public interface CredentialProviderContext { + + /** + * System properties are derived from the CDAP configuration. Anything in the CDAP configuration + * that is prefixed by 'credential.provider.system.properties.[provider-name].' will be adding as + * an entry in the system properties. For example, if the provider is named 'abc', and there is a + * configuration property 'credential.provider.system.properties.abc.retry.timeout' with value + * '60', the system properties map will contain a key 'retry.timeout' with value '60'. System + * properties are not visible to end users and cannot be overwritten by end users. + * + * @return Unmodifiable system properties for the provider. + */ + Map getProperties(); +} diff --git a/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/ProfileValidationException.java b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/ProfileValidationException.java new file mode 100644 index 000000000000..ebcef1b09f4b --- /dev/null +++ b/cdap-security-spi/src/main/java/io/cdap/cdap/security/spi/credential/ProfileValidationException.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.security.spi.credential; + +/** + * Exception thrown during profile validation. + */ +public class ProfileValidationException extends Exception { + + /** + * Creates a new profile validation exception. + * + * @param message The message for the validation failure. + */ + public ProfileValidationException(String message) { + super(message); + } +} diff --git a/cdap-security/src/main/java/io/cdap/cdap/security/runtime/AuthenticationServerMain.java b/cdap-security/src/main/java/io/cdap/cdap/security/runtime/AuthenticationServerMain.java index 4734505c821d..85399509197a 100644 --- a/cdap-security/src/main/java/io/cdap/cdap/security/runtime/AuthenticationServerMain.java +++ b/cdap-security/src/main/java/io/cdap/cdap/security/runtime/AuthenticationServerMain.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2022 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -26,8 +26,8 @@ import io.cdap.cdap.common.guice.ConfigModule; import io.cdap.cdap.common.guice.IOModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.runtime.DaemonMain; import io.cdap.cdap.security.guice.CoreSecurityRuntimeModule; import io.cdap.cdap.security.guice.ExternalAuthenticationModule; @@ -57,8 +57,8 @@ public void init(String[] args) { Injector injector = Guice.createInjector(new ConfigModule(), new IOModule(), RemoteAuthenticatorModules.getDefaultModule(), - new ZKClientModule(), - new ZKDiscoveryModule(), + new ZkClientModule(), + new ZkDiscoveryModule(), new CoreSecurityRuntimeModule().getDistributedModules(), new ExternalAuthenticationModule()); configuration = injector.getInstance(CConfiguration.class); diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/auth/DistributedKeyManagerTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/auth/DistributedKeyManagerTest.java index 0a363c9037e9..eeae7d21bfad 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/auth/DistributedKeyManagerTest.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/auth/DistributedKeyManagerTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2021 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -28,8 +28,8 @@ import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.guice.ConfigModule; import io.cdap.cdap.common.guice.IOModule; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.io.Codec; import io.cdap.cdap.common.utils.ImmutablePair; import io.cdap.cdap.common.utils.Tasks; @@ -56,6 +56,7 @@ * Tests covering the {@link DistributedKeyManager} implementation. */ public class DistributedKeyManagerTest extends TestTokenManager { + private static final Logger LOG = LoggerFactory.getLogger(DistributedKeyManagerTest.class); private static MiniZooKeeperCluster zkCluster; private static Injector injector1; @@ -66,7 +67,7 @@ public static void setup() throws Exception { HBaseTestingUtility testUtil = new HBaseTestingUtility(); zkCluster = testUtil.startMiniZKCluster(); String zkConnectString = testUtil.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM) + ":" - + zkCluster.getClientPort(); + + zkCluster.getClientPort(); LOG.info("Running ZK cluster at " + zkConnectString); CConfiguration cConf1 = CConfiguration.create(); cConf1.setBoolean(Constants.Security.ENABLED, true); @@ -83,8 +84,8 @@ public static void setup() throws Exception { CoreSecurityModule coreSecurityModule = CoreSecurityRuntimeModule.getDistributedModule(cConf1); modules.add(coreSecurityModule); if (coreSecurityModule.requiresZKClient()) { - modules.add(new ZKClientModule()); - modules.add(new ZKDiscoveryModule()); + modules.add(new ZkClientModule()); + modules.add(new ZkDiscoveryModule()); } injector1 = Guice.createInjector(modules); @@ -95,8 +96,8 @@ public static void setup() throws Exception { coreSecurityModule = CoreSecurityRuntimeModule.getDistributedModule(cConf2); modules.add(coreSecurityModule); if (coreSecurityModule.requiresZKClient()) { - modules.add(new ZKClientModule()); - modules.add(new ZKDiscoveryModule()); + modules.add(new ZkClientModule()); + modules.add(new ZkDiscoveryModule()); } injector2 = Guice.createInjector(modules); @@ -114,16 +115,16 @@ public void testKeyDistribution() throws Exception { TimeUnit.MILLISECONDS.sleep(1000); TestingTokenManager tokenManager1 = - new TestingTokenManager(manager1, injector1.getInstance(UserIdentityCodec.class)); + new TestingTokenManager(manager1, injector1.getInstance(UserIdentityCodec.class)); TestingTokenManager tokenManager2 = - new TestingTokenManager(manager2, injector2.getInstance(UserIdentityCodec.class)); + new TestingTokenManager(manager2, injector2.getInstance(UserIdentityCodec.class)); tokenManager1.startAndWait(); tokenManager2.startAndWait(); long now = System.currentTimeMillis(); UserIdentity ident1 = new UserIdentity("testuser", UserIdentity.IdentifierType.EXTERNAL, - Lists.newArrayList("users", "admins"), now, - now + 60 * 60 * 1000); + Lists.newArrayList("users", "admins"), now, + now + 60 * 60 * 1000); AccessToken token1 = tokenManager1.signIdentifier(ident1); // make sure the second token manager has the secret key required to validate the signature tokenManager2.waitForKey(tokenManager1.getCurrentKey().getKeyId(), 2000, TimeUnit.MILLISECONDS); @@ -141,7 +142,7 @@ public void testKeyDistribution() throws Exception { } @Test - public void testGetACLs() throws Exception { + public void testGetAcls() throws Exception { CConfiguration kerbConf = CConfiguration.create(); kerbConf.set(Constants.Security.KERBEROS_ENABLED, "true"); kerbConf.set(Constants.Security.Authentication.MODE, "MANAGED"); @@ -155,20 +156,24 @@ public void testGetACLs() throws Exception { } @Override - protected ImmutablePair> getTokenManagerAndCodec() throws Exception { + protected ImmutablePair> getTokenManagerAndCodec() + throws Exception { DistributedKeyManager keyManager = getKeyManager(injector1, true); - TokenManager tokenManager = new TokenManager(keyManager, injector1.getInstance(UserIdentityCodec.class)); + TokenManager tokenManager = new TokenManager(keyManager, + injector1.getInstance(UserIdentityCodec.class)); tokenManager.startAndWait(); return new ImmutablePair<>(tokenManager, injector1.getInstance(AccessTokenCodec.class)); } - private DistributedKeyManager getKeyManager(Injector injector, boolean expectLeader) throws Exception { + private DistributedKeyManager getKeyManager(Injector injector, boolean expectLeader) + throws Exception { ZKClientService zk = injector.getInstance(ZKClientService.class); zk.startAndWait(); WaitableDistributedKeyManager keyManager = - new WaitableDistributedKeyManager(injector.getInstance(CConfiguration.class), - injector.getInstance(Key.get(new TypeLiteral>() { })), - zk); + new WaitableDistributedKeyManager(injector.getInstance(CConfiguration.class), + injector.getInstance(Key.get(new TypeLiteral>() { + })), + zk); keyManager.startAndWait(); if (expectLeader) { @@ -178,7 +183,9 @@ private DistributedKeyManager getKeyManager(Injector injector, boolean expectLea } private static class WaitableDistributedKeyManager extends DistributedKeyManager { - WaitableDistributedKeyManager(CConfiguration conf, Codec codec, ZKClientService zk) { + + WaitableDistributedKeyManager(CConfiguration conf, Codec codec, + ZKClientService zk) { super(conf, codec, zk, Lists.newArrayList(ZooDefs.Ids.OPEN_ACL_UNSAFE)); } @@ -192,6 +199,7 @@ public boolean hasKey(int keyId) { } private static class TestingTokenManager extends TokenManager { + private TestingTokenManager(KeyManager keyManager, Codec identifierCodec) { super(keyManager, identifierCodec); } @@ -204,7 +212,7 @@ public KeyIdentifier getCurrentKey() { } void waitForKey(int keyId, long duration, - TimeUnit unit) throws InterruptedException, TimeoutException, ExecutionException { + TimeUnit unit) throws InterruptedException, TimeoutException, ExecutionException { if (keyManager instanceof WaitableDistributedKeyManager) { WaitableDistributedKeyManager waitKeyManager = (WaitableDistributedKeyManager) keyManager; Tasks.waitFor(true, () -> waitKeyManager.hasKey(keyId), duration, unit); @@ -212,7 +220,7 @@ void waitForKey(int keyId, long duration, } void waitForCurrentKey(long duration, - TimeUnit unit) throws InterruptedException, TimeoutException, ExecutionException { + TimeUnit unit) throws InterruptedException, TimeoutException, ExecutionException { if (keyManager instanceof WaitableDistributedKeyManager) { WaitableDistributedKeyManager waitKeyManager = (WaitableDistributedKeyManager) keyManager; Tasks.waitFor(true, () -> waitKeyManager.getCurrentKey() != null, duration, unit); diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/zookeeper/SharedResourceCacheTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/zookeeper/SharedResourceCacheTest.java index d98028b358c2..032ee98ac4b3 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/zookeeper/SharedResourceCacheTest.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/zookeeper/SharedResourceCacheTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -28,7 +28,7 @@ import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.guice.ConfigModule; -import io.cdap.cdap.common.guice.ZKClientModule; +import io.cdap.cdap.common.guice.ZkClientModule; import io.cdap.cdap.common.io.Codec; import java.io.IOException; import java.util.List; @@ -50,6 +50,7 @@ * Tests covering the {@link SharedResourceCache} implementation. */ public class SharedResourceCacheTest { + private static final String ZK_NAMESPACE = "/SharedResourceCacheTest"; private static final Logger LOG = LoggerFactory.getLogger(SharedResourceCacheTest.class); private static MiniZooKeeperCluster zkCluster; @@ -62,14 +63,14 @@ public static void startUp() throws Exception { HBaseTestingUtility testUtil = new HBaseTestingUtility(); zkCluster = testUtil.startMiniZKCluster(); zkConnectString = testUtil.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM) + ":" - + zkCluster.getClientPort(); + + zkCluster.getClientPort(); LOG.info("Running ZK cluster at " + zkConnectString); CConfiguration cConf = CConfiguration.create(); cConf.set(Constants.Zookeeper.QUORUM, zkConnectString); injector1 = Guice.createInjector(new ConfigModule(cConf, testUtil.getConfiguration()), - new ZKClientModule()); + new ZkClientModule()); injector2 = Guice.createInjector(new ConfigModule(cConf, testUtil.getConfiguration()), - new ZKClientModule()); + new ZkClientModule()); } @AfterClass @@ -79,7 +80,7 @@ public static void tearDown() throws Exception { @Test public void testCache() throws Exception { - String parentZNode = ZK_NAMESPACE + "/testCache"; + String parentNode = ZK_NAMESPACE + "/testCache"; List acls = Lists.newArrayList(ZooDefs.Ids.OPEN_ACL_UNSAFE); @@ -87,7 +88,7 @@ public void testCache() throws Exception { ZKClientService zkClient1 = injector1.getInstance(ZKClientService.class); zkClient1.startAndWait(); SharedResourceCache cache1 = - new SharedResourceCache<>(zkClient1, new StringCodec(), parentZNode, acls); + new SharedResourceCache<>(zkClient1, new StringCodec(), parentNode, acls); cache1.init(); // add items to one and wait for them to show up in the second @@ -98,7 +99,7 @@ public void testCache() throws Exception { ZKClientService zkClient2 = injector2.getInstance(ZKClientService.class); zkClient2.startAndWait(); SharedResourceCache cache2 = - new SharedResourceCache<>(zkClient2, new StringCodec(), parentZNode, acls); + new SharedResourceCache<>(zkClient2, new StringCodec(), parentNode, acls); cache2.init(); waitForEntry(cache2, key1, value1, 10000); @@ -178,6 +179,7 @@ public void onResourceDelete(String name) { private static final class StringCodec implements Codec { + @Override public byte[] encode(String object) throws IOException { return Bytes.toBytes(object); @@ -190,7 +192,7 @@ public String decode(byte[] data) throws IOException { } private void waitForEntry(SharedResourceCache cache, String key, String expectedValue, - long timeToWaitMillis) throws InterruptedException { + long timeToWaitMillis) throws InterruptedException { String value = cache.get(key); boolean isPresent = expectedValue.equals(value); @@ -202,7 +204,8 @@ private void waitForEntry(SharedResourceCache cache, String key, String } if (!isPresent) { - throw new RuntimeException("Timed out waiting for expected value '" + expectedValue + "' in cache"); + throw new RuntimeException( + "Timed out waiting for expected value '" + expectedValue + "' in cache"); } } } diff --git a/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java b/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java index ad1f8c8651e6..35bdf6593b91 100644 --- a/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java +++ b/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2022 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -50,7 +50,7 @@ import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.LocalLocationModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; +import io.cdap.cdap.common.guice.ZkClientModule; import io.cdap.cdap.common.io.URLConnections; import io.cdap.cdap.common.logging.common.UncaughtExceptionHandler; import io.cdap.cdap.common.startup.ConfigurationLogger; @@ -279,7 +279,7 @@ public void startUp() throws Exception { cConf.setInt(Constants.ArtifactLocalizer.PORT, artifactLocalizerService.getPort()); // Set the artifact localizer port for the preview conf as well injector.getInstance( - Key.get(CConfiguration.class, Names.named(PreviewConfigModule.PREVIEW_CCONF))) + Key.get(CConfiguration.class, Names.named(PreviewConfigModule.PREVIEW_CCONF))) .setInt(Constants.ArtifactLocalizer.PORT, artifactLocalizerService.getPort()); previewHttpServer.startAndWait(); previewRunnerManager.startAndWait(); @@ -521,7 +521,7 @@ private static List createPersistentModules(CConfiguration cConf, Config new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), - new ZKClientModule(), + new ZkClientModule(), new KafkaClientModule(), new MetricsHandlerModule(), new LogQueryRuntimeModule().getStandaloneModules(), diff --git a/cdap-tms/src/test/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingServiceTest.java b/cdap-tms/src/test/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingServiceTest.java index eb9fa8feec20..086b4201171b 100644 --- a/cdap-tms/src/test/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingServiceTest.java +++ b/cdap-tms/src/test/java/io/cdap/cdap/messaging/distributed/LeaderElectionMessagingServiceTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2018 Cask Data, Inc. + * Copyright © 2017-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -30,8 +30,8 @@ import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.guice.ConfigModule; import io.cdap.cdap.common.guice.DFSLocationModule; -import io.cdap.cdap.common.guice.ZKClientModule; -import io.cdap.cdap.common.guice.ZKDiscoveryModule; +import io.cdap.cdap.common.guice.ZkClientModule; +import io.cdap.cdap.common.guice.ZkDiscoveryModule; import io.cdap.cdap.common.metrics.NoOpMetricsCollectionService; import io.cdap.cdap.common.namespace.InMemoryNamespaceAdmin; import io.cdap.cdap.common.namespace.NamespaceQueryAdmin; @@ -92,7 +92,8 @@ public static void init() throws IOException { cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr()); cConf.setInt(Constants.Zookeeper.CFG_SESSION_TIMEOUT_MILLIS, 2000); cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath()); - cConf.set(Constants.MessagingSystem.HTTP_SERVER_BIND_ADDRESS, InetAddress.getLocalHost().getHostName()); + cConf.set(Constants.MessagingSystem.HTTP_SERVER_BIND_ADDRESS, + InetAddress.getLocalHost().getHostName()); cConf.set(Constants.MessagingSystem.SYSTEM_TOPICS, "topic"); cConf.setLong(Constants.MessagingSystem.HA_FENCING_DELAY_SECONDS, 0L); @@ -109,8 +110,8 @@ public static void finish() { public void testTransition() throws Throwable { final TopicId topicId = NamespaceId.SYSTEM.topic("topic"); - Injector injector1 = createInjector(0); - Injector injector2 = createInjector(1); + final Injector injector1 = createInjector(0); + final Injector injector2 = createInjector(1); // Start a messaging service, which would becomes leader ZKClientService zkClient1 = injector1.getInstance(ZKClientService.class); @@ -145,20 +146,23 @@ public void testTransition() throws Throwable { KillZKSession.kill(zkClient1.getZooKeeperSupplier().get(), zkClient1.getConnectString(), 10000); // Publish one more message and then fetch from the current leader - List messages = Retries.callWithRetries(new Retries.Callable, Throwable>() { - @Override - public List call() throws Throwable { - secondService.publish(StoreRequestBuilder.of(topicId).addPayload("Testing2").build()); - - List messages = new ArrayList<>(); - try (CloseableIterator iterator = secondService.prepareFetch(topicId).fetch()) { - while (iterator.hasNext()) { - messages.add(new String(iterator.next().getPayload(), "UTF-8")); + List messages = Retries + .callWithRetries(new Retries.Callable, Throwable>() { + @Override + public List call() throws Throwable { + secondService.publish(StoreRequestBuilder.of(topicId).addPayload("Testing2").build()); + + List messages = new ArrayList<>(); + try (CloseableIterator iterator = secondService.prepareFetch(topicId) + .fetch()) { + while (iterator.hasNext()) { + messages.add(new String(iterator.next().getPayload(), "UTF-8")); + } + } + return messages; } - } - return messages; - } - }, RetryStrategies.timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS))); + }, RetryStrategies + .timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS))); Assert.assertEquals(Arrays.asList("Testing1", "Testing2"), messages); @@ -180,7 +184,8 @@ public List call() throws Throwable { } return messages; } - }, RetryStrategies.timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS))); + }, RetryStrategies + .timeLimit(10, TimeUnit.SECONDS, RetryStrategies.fixDelay(1, TimeUnit.SECONDS))); Assert.assertEquals(Arrays.asList("Testing1", "Testing2"), messages); @@ -190,7 +195,7 @@ public List call() throws Throwable { @Test public void testFencing() - throws IOException, InterruptedException, ExecutionException, TimeoutException, UnauthorizedException { + throws IOException, InterruptedException, ExecutionException, TimeoutException, UnauthorizedException { final TopicId topicId = NamespaceId.SYSTEM.topic("topic"); // Change the fencing time @@ -243,41 +248,44 @@ private Injector createInjector(int instanceId) { cConf.setInt(Constants.MessagingSystem.CONTAINER_INSTANCE_ID, instanceId); return Guice.createInjector( - new ConfigModule(cConf), - new ZKClientModule(), - new ZKDiscoveryModule(), - new AuthorizationEnforcementModule().getNoOpModules(), - new DFSLocationModule(), - new AbstractModule() { - @Override - protected void configure() { - // Bindings to services for testing only - bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class); - - // Use the same in memory client across all injectors. - bind(NamespaceQueryAdmin.class).toInstance(namespaceQueryAdmin); - } - }, - new PrivateModule() { - @Override - protected void configure() { - // This is very similar to bindings in distributed mode, except we bind to level db instead of HBase - // Also the level DB has to be one instance since unit-test runs in the same process. - bind(TableFactory.class) - .annotatedWith(Names.named(CachingTableFactory.DELEGATE_TABLE_FACTORY)) - .toInstance(levelDBTableFactory); - - // The cache must be in singleton scope - bind(MessageTableCacheProvider.class).to(DefaultMessageTableCacheProvider.class).in(Scopes.SINGLETON); - bind(TableFactory.class).to(CachingTableFactory.class); - - // Bind http handlers - MessagingServerRuntimeModule.bindHandlers(binder(), Constants.MessagingSystem.HANDLER_BINDING_NAME); - - bind(MessagingService.class).to(LeaderElectionMessagingService.class).in(Scopes.SINGLETON); - expose(MessagingService.class); + new ConfigModule(cConf), + new ZkClientModule(), + new ZkDiscoveryModule(), + new AuthorizationEnforcementModule().getNoOpModules(), + new DFSLocationModule(), + new AbstractModule() { + @Override + protected void configure() { + // Bindings to services for testing only + bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class); + + // Use the same in memory client across all injectors. + bind(NamespaceQueryAdmin.class).toInstance(namespaceQueryAdmin); + } + }, + new PrivateModule() { + @Override + protected void configure() { + // This is very similar to bindings in distributed mode, except we bind to level db instead of HBase + // Also the level DB has to be one instance since unit-test runs in the same process. + bind(TableFactory.class) + .annotatedWith(Names.named(CachingTableFactory.DELEGATE_TABLE_FACTORY)) + .toInstance(levelDBTableFactory); + + // The cache must be in singleton scope + bind(MessageTableCacheProvider.class).to(DefaultMessageTableCacheProvider.class) + .in(Scopes.SINGLETON); + bind(TableFactory.class).to(CachingTableFactory.class); + + // Bind http handlers + MessagingServerRuntimeModule + .bindHandlers(binder(), Constants.MessagingSystem.HANDLER_BINDING_NAME); + + bind(MessagingService.class).to(LeaderElectionMessagingService.class) + .in(Scopes.SINGLETON); + expose(MessagingService.class); + } } - } ); } } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LocalLogAppenderResilientTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LocalLogAppenderResilientTest.java index 6641a6ed1a1f..c10c23989037 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LocalLogAppenderResilientTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/LocalLogAppenderResilientTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2022 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -34,7 +34,7 @@ import io.cdap.cdap.common.guice.NamespaceAdminTestModule; import io.cdap.cdap.common.guice.NonCustomLocationUnitTestModule; import io.cdap.cdap.common.guice.RemoteAuthenticatorModules; -import io.cdap.cdap.common.guice.ZKClientModule; +import io.cdap.cdap.common.guice.ZkClientModule; import io.cdap.cdap.common.logging.LoggingContext; import io.cdap.cdap.common.logging.LoggingContextAccessor; import io.cdap.cdap.common.utils.Networks; @@ -91,7 +91,6 @@ public class LocalLogAppenderResilientTest { @Test public void testResilientLogging() throws Exception { - Configuration hConf = new Configuration(); CConfiguration cConf = CConfiguration.create(); File datasetDir = new File(tmpFolder.newFolder(), "datasetUser"); @@ -106,42 +105,46 @@ public void testResilientLogging() throws Exception { cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath()); + Configuration hConf = new Configuration(); + Injector injector = Guice.createInjector( - new ConfigModule(cConf, hConf), - RemoteAuthenticatorModules.getNoOpModule(), - new IOModule(), - new ZKClientModule(), - new KafkaClientModule(), - new InMemoryDiscoveryModule(), - new NonCustomLocationUnitTestModule(), - new DataFabricModules().getInMemoryModules(), - new DataSetsModules().getStandaloneModules(), - new DataSetServiceModules().getInMemoryModules(), - new TransactionMetricsModule(), - new LocalLogAppenderModule(), - new NamespaceAdminTestModule(), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getInMemoryModules(), - new AuthenticationContextModules().getMasterModule(), - new AbstractModule() { - @Override - protected void configure() { - bind(UGIProvider.class).to(UnsupportedUGIProvider.class); - bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class); - bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class); - } - }); + new ConfigModule(cConf, hConf), + RemoteAuthenticatorModules.getNoOpModule(), + new IOModule(), + new ZkClientModule(), + new KafkaClientModule(), + new InMemoryDiscoveryModule(), + new NonCustomLocationUnitTestModule(), + new DataFabricModules().getInMemoryModules(), + new DataSetsModules().getStandaloneModules(), + new DataSetServiceModules().getInMemoryModules(), + new TransactionMetricsModule(), + new LocalLogAppenderModule(), + new NamespaceAdminTestModule(), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getInMemoryModules(), + new AuthenticationContextModules().getMasterModule(), + new AbstractModule() { + @Override + protected void configure() { + bind(UGIProvider.class).to(UnsupportedUGIProvider.class); + bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class); + bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class); + } + }); TransactionManager txManager = injector.getInstance(TransactionManager.class); txManager.startAndWait(); StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class)); - DatasetOpExecutorService opExecutorService = injector.getInstance(DatasetOpExecutorService.class); + DatasetOpExecutorService opExecutorService = injector + .getInstance(DatasetOpExecutorService.class); opExecutorService.startAndWait(); // Start the logging before starting the service. - LoggingContextAccessor.setLoggingContext(new WorkerLoggingContext("TRL_ACCT_1", "APP_1", "WORKER_1", - "RUN", "INSTANCE")); + LoggingContextAccessor + .setLoggingContext(new WorkerLoggingContext("TRL_ACCT_1", "APP_1", "WORKER_1", + "RUN", "INSTANCE")); String logBaseDir = "trl-log/log_files_" + new Random(System.currentTimeMillis()).nextLong(); cConf.set(LoggingConfiguration.LOG_BASE_DIR, logBaseDir); @@ -188,14 +191,15 @@ public void addStatusEvent(Status status) { final CountDownLatch startLatch = new CountDownLatch(1); DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class); - discoveryClient.discover(Constants.Service.DATASET_MANAGER).watchChanges(new ServiceDiscovered.ChangeListener() { - @Override - public void onChange(ServiceDiscovered serviceDiscovered) { - if (!Iterables.isEmpty(serviceDiscovered)) { - startLatch.countDown(); - } - } - }, Threads.SAME_THREAD_EXECUTOR); + discoveryClient.discover(Constants.Service.DATASET_MANAGER) + .watchChanges(new ServiceDiscovered.ChangeListener() { + @Override + public void onChange(ServiceDiscovered serviceDiscovered) { + if (!Iterables.isEmpty(serviceDiscovered)) { + startLatch.countDown(); + } + } + }, Threads.SAME_THREAD_EXECUTOR); startLatch.await(5, TimeUnit.SECONDS); @@ -209,11 +213,12 @@ public void onChange(ServiceDiscovered serviceDiscovered) { appender.stop(); // Verify - we should have at least 5 events. - LoggingContext loggingContext = new WorkerLoggingContext("TRL_ACCT_1", "APP_1", "WORKER_1", "RUN", "INSTANCE"); + LoggingContext loggingContext = new WorkerLoggingContext("TRL_ACCT_1", "APP_1", "WORKER_1", + "RUN", "INSTANCE"); FileLogReader logTail = injector.getInstance(FileLogReader.class); LoggingTester.LogCallback logCallback1 = new LoggingTester.LogCallback(); logTail.getLogPrev(loggingContext, ReadRange.LATEST, 10, Filter.EMPTY_FILTER, - logCallback1); + logCallback1); List allEvents = logCallback1.getEvents(); Assert.assertTrue(allEvents.toString(), allEvents.size() >= 5); diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/kafka/TestKafkaLogging.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/kafka/TestKafkaLogging.java index ecdf427e1795..fc188a48619c 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/kafka/TestKafkaLogging.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/appender/kafka/TestKafkaLogging.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2019 Cask Data, Inc. + * Copyright © 2014-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -63,14 +63,16 @@ public static void init() throws Exception { Logger logger = LoggerFactory.getLogger("TestKafkaLogging"); LoggingTester loggingTester = new LoggingTester(); - loggingTester.generateLogs(logger, new WorkerLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "RUN1", "INSTANCE1")); + loggingTester.generateLogs(logger, + new WorkerLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "RUN1", "INSTANCE1")); appender.stop(); } @Test public void testGetNext() throws Exception { // Check with null runId and null instanceId - LoggingContext loggingContext = new WorkerLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "RUN1", "INSTANCE1"); + LoggingContext loggingContext = new WorkerLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "RUN1", + "INSTANCE1"); KafkaLogReader logReader = KAFKA_TESTER.getInjector().getInstance(KafkaLogReader.class); LoggingTester tester = new LoggingTester(); tester.testGetNext(logReader, loggingContext); @@ -78,7 +80,8 @@ public void testGetNext() throws Exception { @Test public void testGetPrev() throws Exception { - LoggingContext loggingContext = new WorkerLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "RUN1", "INSTANCE1"); + LoggingContext loggingContext = new WorkerLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "RUN1", + "INSTANCE1"); KafkaLogReader logReader = KAFKA_TESTER.getInjector().getInstance(KafkaLogReader.class); LoggingTester tester = new LoggingTester(); tester.testGetPrev(logReader, loggingContext); @@ -88,38 +91,41 @@ public void testGetPrev() throws Exception { @Test public void testPartitionKey() throws Exception { - CConfiguration cConf = KAFKA_TESTER.getCConf(); + CConfiguration cConf = KAFKA_TESTER.getCconf(); // set kafka partition key to application cConf.set(Constants.Logging.LOG_PUBLISH_PARTITION_KEY, "application"); Logger logger = LoggerFactory.getLogger("TestKafkaLogging"); - LoggingContext loggingContext = new WorkerLoggingContext("TKL_NS_2", "APP_2", "FLOW_2", "RUN2", "INSTANCE2"); + LoggingContext loggingContext = new WorkerLoggingContext("TKL_NS_2", "APP_2", "FLOW_2", "RUN2", + "INSTANCE2"); LoggingContextAccessor.setLoggingContext(loggingContext); for (int i = 0; i < 40; ++i) { - logger.warn("TKL_NS_2 Test log message {} {} {}", i, "arg1", "arg2", new Exception("test exception")); + logger.warn("TKL_NS_2 Test log message {} {} {}", i, "arg1", "arg2", + new Exception("test exception")); } loggingContext = new WorkerLoggingContext("TKL_NS_2", "APP_2", "FLOW_3", "RUN3", "INSTANCE3"); LoggingContextAccessor.setLoggingContext(loggingContext); for (int i = 0; i < 40; ++i) { - logger.warn("TKL_NS_2 Test log message {} {} {}", i, "arg1", "arg2", new Exception("test exception")); + logger.warn("TKL_NS_2 Test log message {} {} {}", i, "arg1", "arg2", + new Exception("test exception")); } final Multimap actual = ArrayListMultimap.create(); - KAFKA_TESTER.getPublishedMessages(KAFKA_TESTER.getCConf().get(Constants.Logging.KAFKA_TOPIC), - ImmutableSet.of(0, 1), 40, new Function() { - @Override - public String apply(final FetchedMessage input) { - try { - Map.Entry entry = convertFetchedMessage(input); - actual.put(entry.getKey(), entry.getValue()); - } catch (IOException e) { - // should never happen + KAFKA_TESTER.getPublishedMessages(KAFKA_TESTER.getCconf().get(Constants.Logging.KAFKA_TOPIC), + ImmutableSet.of(0, 1), 40, new Function() { + @Override + public String apply(final FetchedMessage input) { + try { + Map.Entry entry = convertFetchedMessage(input); + actual.put(entry.getKey(), entry.getValue()); + } catch (IOException e) { + // should never happen + } + return ""; } - return ""; - } - }); + }); boolean isPresent = false; @@ -136,10 +142,12 @@ public String apply(final FetchedMessage input) { cConf.set(Constants.Logging.LOG_PUBLISH_PARTITION_KEY, "program"); } - private Map.Entry convertFetchedMessage(FetchedMessage message) throws IOException { + private Map.Entry convertFetchedMessage(FetchedMessage message) + throws IOException { LoggingEventSerializer serializer = new LoggingEventSerializer(); ILoggingEvent iLoggingEvent = serializer.fromBytes(message.getPayload()); - LoggingContext loggingContext = LoggingContextHelper.getLoggingContext(iLoggingEvent.getMDCPropertyMap()); + LoggingContext loggingContext = LoggingContextHelper + .getLoggingContext(iLoggingEvent.getMDCPropertyMap()); String key = loggingContext.getLogPartition(); return Maps.immutableEntry(message.getTopicPartition().getPartition(), key); } diff --git a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFrameworkTest.java b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFrameworkTest.java index b9efefac91eb..79a282ec35d9 100644 --- a/cdap-watchdog/src/test/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFrameworkTest.java +++ b/cdap-watchdog/src/test/java/io/cdap/cdap/logging/framework/distributed/DistributedLogFrameworkTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2018 Cask Data, Inc. + * Copyright © 2017-2023 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -32,7 +32,7 @@ import io.cdap.cdap.common.guice.KafkaClientModule; import io.cdap.cdap.common.guice.LocalLocationModule; import io.cdap.cdap.common.guice.NamespaceAdminTestModule; -import io.cdap.cdap.common.guice.ZKClientModule; +import io.cdap.cdap.common.guice.ZkClientModule; import io.cdap.cdap.common.logging.LoggingContext; import io.cdap.cdap.common.logging.ServiceLoggingContext; import io.cdap.cdap.common.metrics.NoOpMetricsCollectionService; @@ -96,17 +96,17 @@ public class DistributedLogFrameworkTest { @ClassRule public static final KafkaTester KAFKA_TESTER = new KafkaTester( - Collections.singletonMap(Constants.Logging.NUM_PARTITIONS, "1"), - Collections.emptyList(), 1 + Collections.singletonMap(Constants.Logging.NUM_PARTITIONS, "1"), + Collections.emptyList(), 1 ); private Injector injector; @BeforeClass public static void init() { - CConfiguration cConf = KAFKA_TESTER.getCConf(); + CConfiguration cConf = KAFKA_TESTER.getCconf(); KAFKA_TESTER.createTopic(cConf.get(Constants.Logging.KAFKA_TOPIC), - cConf.getInt(Constants.Logging.NUM_PARTITIONS)); + cConf.getInt(Constants.Logging.NUM_PARTITIONS)); } @Before @@ -137,35 +137,38 @@ public void testFramework() throws Exception { // Send some logs to Kafka. LoggingContext context = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), - Constants.Logging.COMPONENT_NAME, - "test"); + Constants.Logging.COMPONENT_NAME, + "test"); // Make sure all events get flushed in the same batch - long eventTimeBase = System.currentTimeMillis() + cConf.getInt(Constants.Logging.PIPELINE_EVENT_DELAY_MS); + long eventTimeBase = + System.currentTimeMillis() + cConf.getInt(Constants.Logging.PIPELINE_EVENT_DELAY_MS); final int msgCount = 50; for (int i = 0; i < msgCount; i++) { // Publish logs in descending timestamp order publishLog( - cConf.get(Constants.Logging.KAFKA_TOPIC), context, - ImmutableList.of( - createLoggingEvent("io.cdap.test." + i, Level.INFO, "Testing " + i, eventTimeBase - i) - ) + cConf.get(Constants.Logging.KAFKA_TOPIC), context, + ImmutableList.of( + createLoggingEvent("io.cdap.test." + i, Level.INFO, "Testing " + i, eventTimeBase - i) + ) ); } // Read the logs back. They should be sorted by timestamp order. final FileMetaDataReader metaDataReader = injector.getInstance(FileMetaDataReader.class); Tasks.waitFor(true, () -> { - List locations = metaDataReader.listFiles(new LogPathIdentifier(NamespaceId.SYSTEM.getNamespace(), - Constants.Logging.COMPONENT_NAME, - "test"), 0, Long.MAX_VALUE); + List locations = metaDataReader + .listFiles(new LogPathIdentifier(NamespaceId.SYSTEM.getNamespace(), + Constants.Logging.COMPONENT_NAME, + "test"), 0, Long.MAX_VALUE); if (locations.size() != 1) { return false; } LogLocation location = locations.get(0); int i = 0; try { - try (CloseableIterator iter = location.readLog(Filter.EMPTY_FILTER, 0, Long.MAX_VALUE, msgCount)) { + try (CloseableIterator iter = location + .readLog(Filter.EMPTY_FILTER, 0, Long.MAX_VALUE, msgCount)) { while (iter.hasNext()) { String expectedMsg = "Testing " + (msgCount - i - 1); LogEvent event = iter.next(); @@ -196,11 +199,11 @@ public void testFramework() throws Exception { private CheckpointManager getCheckpointManager(String kafkaTopic) { TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class); return new KafkaCheckpointManager(transactionRunner, - Constants.Logging.SYSTEM_PIPELINE_CHECKPOINT_PREFIX + kafkaTopic); + Constants.Logging.SYSTEM_PIPELINE_CHECKPOINT_PREFIX + kafkaTopic); } private Injector createInjector() throws IOException { - CConfiguration cConf = CConfiguration.copy(KAFKA_TESTER.getCConf()); + CConfiguration cConf = CConfiguration.copy(KAFKA_TESTER.getCconf()); cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath()); // The event delay cannot be too small, otherwise the events will be out of order, especially on slow machine @@ -209,36 +212,38 @@ private Injector createInjector() throws IOException { MockTwillContext mockTwillContext = new MockTwillContext(); return Guice.createInjector( - new ConfigModule(cConf), - new ZKClientModule(), - new InMemoryDiscoveryModule(), - new KafkaClientModule(), - new LocalLocationModule(), - new DistributedLogFrameworkModule(mockTwillContext.getInstanceId(), mockTwillContext.getInstanceCount()), - new DataSetsModules().getInMemoryModules(), - new TransactionModules().getInMemoryModules(), - new TransactionExecutorModule(), - new SystemDatasetRuntimeModule().getInMemoryModules(), - new NamespaceAdminTestModule(), - new AuthorizationTestModule(), - new AuthorizationEnforcementModule().getInMemoryModules(), - new AuthenticationContextModules().getNoOpModule(), - new StorageModule(), - new AbstractModule() { - @Override - protected void configure() { - bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class); - bind(UGIProvider.class).to(CurrentUGIProvider.class); - bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); + new ConfigModule(cConf), + new ZkClientModule(), + new InMemoryDiscoveryModule(), + new KafkaClientModule(), + new LocalLocationModule(), + new DistributedLogFrameworkModule(mockTwillContext.getInstanceId(), + mockTwillContext.getInstanceCount()), + new DataSetsModules().getInMemoryModules(), + new TransactionModules().getInMemoryModules(), + new TransactionExecutorModule(), + new SystemDatasetRuntimeModule().getInMemoryModules(), + new NamespaceAdminTestModule(), + new AuthorizationTestModule(), + new AuthorizationEnforcementModule().getInMemoryModules(), + new AuthenticationContextModules().getNoOpModule(), + new StorageModule(), + new AbstractModule() { + @Override + protected void configure() { + bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class); + bind(UGIProvider.class).to(CurrentUGIProvider.class); + bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class); + } } - } ); } /** * Creates an {@link ILoggingEvent}. */ - private ILoggingEvent createLoggingEvent(String loggerName, Level level, String message, long timestamp) { + private ILoggingEvent createLoggingEvent(String loggerName, Level level, String message, + long timestamp) { LoggingEvent event = new LoggingEvent(); event.setLevel(level); event.setLoggerName(loggerName); @@ -252,12 +257,13 @@ private ILoggingEvent createLoggingEvent(String loggerName, Level level, String */ private void publishLog(String topic, LoggingContext context, Iterable events) { KafkaPublisher.Preparer preparer = KAFKA_TESTER.getKafkaClient() - .getPublisher(KafkaPublisher.Ack.LEADER_RECEIVED, Compression.NONE) - .prepare(topic); + .getPublisher(KafkaPublisher.Ack.LEADER_RECEIVED, Compression.NONE) + .prepare(topic); LoggingEventSerializer serializer = new LoggingEventSerializer(); for (ILoggingEvent event : events) { - preparer.add(ByteBuffer.wrap(serializer.toBytes(new LogMessage(event, context))), context.getLogPartition()); + preparer.add(ByteBuffer.wrap(serializer.toBytes(new LogMessage(event, context))), + context.getLogPartition()); } preparer.send(); }