From c09105995cc30dfd1f28f6cff8f2d102a8a3bb4f Mon Sep 17 00:00:00 2001 From: itsankit-google Date: Fri, 14 Jul 2023 12:44:15 +0000 Subject: [PATCH] CDAP-19428: Allow configurable GCP Dataproc Oauth Scopes --- .../provisioner/dataproc/DataprocClient.java | 2 +- .../provisioner/dataproc/DataprocConf.java | 20 +++++++++++++++++-- .../src/main/resources/gcp-dataproc.json | 9 +++++++++ .../dataproc/DataprocProvisionerTest.java | 5 +++++ 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocClient.java b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocClient.java index eddccc3b80eb..40224e0677fd 100644 --- a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocClient.java +++ b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocClient.java @@ -183,7 +183,7 @@ ClusterOperationMetadata createCluster(String name, String imageVersion, metadata.putAll(conf.getClusterMetaData()); GceClusterConfig.Builder clusterConfig = GceClusterConfig.newBuilder() - .addServiceAccountScopes(DataprocConf.CLOUD_PLATFORM_SCOPE) + .addAllServiceAccountScopes(conf.getScopes()) .setShieldedInstanceConfig( ShieldedInstanceConfig.newBuilder() .setEnableSecureBoot(conf.isSecureBootEnabled()) diff --git a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocConf.java b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocConf.java index 3d485c3f7fa1..99870fb56f06 100644 --- a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocConf.java +++ b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocConf.java @@ -64,6 +64,7 @@ final class DataprocConf { // Dataproc will pass it to GCE when creating the GCE cluster. // It can be overridden by profile runtime arguments (system.profile.properties.serviceAccount) static final String SERVICE_ACCOUNT = "serviceAccount"; + static final String SCOPES = "scopes"; static final String ROOT_URL = "root.url"; static final String GCS_BUCKET = "gcsBucket"; @@ -153,6 +154,7 @@ final class DataprocConf { private final Map clusterMetaData; private final Map clusterLabels; private final List networkTags; + private final List scopes; private final String initActions; private final String autoScalingPolicy; private final int idleTtlMinutes; @@ -199,7 +201,7 @@ private DataprocConf(@Nullable String accountKey, String region, String zone, St @Nullable String customImageUri, @Nullable Map clusterMetaData, @Nullable Map clusterLabels, List networkTags, - @Nullable String initActions, boolean runtimeJobManagerEnabled, + List scopes, @Nullable String initActions, boolean runtimeJobManagerEnabled, Map clusterProperties, @Nullable String autoScalingPolicy, int idleTtlMinutes, @Nullable String tokenEndpoint, boolean secureBootEnabled, boolean vTpmEnabled, boolean integrityMonitoringEnabled, boolean clusterReuseEnabled, @@ -213,6 +215,7 @@ private DataprocConf(@Nullable String accountKey, String region, String zone, St this.zone = zone; this.projectId = projectId; this.tempBucket = tempBucket; + this.scopes = scopes; this.clusterReuseEnabled = clusterReuseEnabled; this.clusterReuseThresholdMinutes = clusterReuseThresholdMinutes; this.clusterReuseRetryDelayMs = clusterReuseRetryDelayMs; @@ -426,6 +429,10 @@ List getNetworkTags() { return Collections.unmodifiableList(networkTags); } + List getScopes() { + return Collections.unmodifiableList(scopes); + } + List getInitActions() { if (Strings.isNullOrEmpty(initActions)) { return Collections.emptyList(); @@ -771,6 +778,15 @@ static DataprocConf create(Map properties) { properties.getOrDefault(DataprocUtils.TROUBLESHOOTING_DOCS_URL_KEY, DataprocUtils.TROUBLESHOOTING_DOCS_URL_DEFAULT); + final String scopesProperty = String.format("%s,%s", + Optional.ofNullable(getString(properties, SCOPES)).orElse(""), CLOUD_PLATFORM_SCOPE); + List scopes = Collections.unmodifiableList( + Arrays.stream(scopesProperty.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .distinct() + .collect(Collectors.toList())); + return new DataprocConf(accountKey, region, zone, projectId, networkHostProjectId, network, subnet, masterNumNodes, masterCpus, masterMemoryMb, masterDiskGb, @@ -782,7 +798,7 @@ static DataprocConf create(Map properties) { stackdriverLoggingEnabled, stackdriverMonitoringEnabled, componentGatewayEnabled, skipDelete, imageVersion, customImageUri, clusterMetaData, clusterLabels, networkTags, - initActions, runtimeJobManagerEnabled, clusterProps, autoScalingPolicy, idleTtl, + scopes, initActions, runtimeJobManagerEnabled, clusterProps, autoScalingPolicy, idleTtl, tokenEndpoint, secureBootEnabled, vTpmEnabled, integrityMonitoringEnabled, clusterReuseEnabled, clusterReuseThresholdMinutes, clusterReuseRetryDelayMs, clusterReuseRetryMaxMs, clusterReuseUpdateMaxMs, clusterReuseKey, diff --git a/cdap-runtime-ext-dataproc/src/main/resources/gcp-dataproc.json b/cdap-runtime-ext-dataproc/src/main/resources/gcp-dataproc.json index 6eec9023f0e2..5c2fc055a89e 100644 --- a/cdap-runtime-ext-dataproc/src/main/resources/gcp-dataproc.json +++ b/cdap-runtime-ext-dataproc/src/main/resources/gcp-dataproc.json @@ -493,6 +493,15 @@ "placeholder": "projects//locations//keyRings//cryptoKeys/" } }, + { + "name": "scopes", + "label": "OAuth Scopes", + "widget-type": "csv", + "description": "The OAuth 2.0 scopes that you might need to request to access Google APIs, depending on the level of access you need. Google Cloud Platform Scope (https://www.googleapis.com/auth/cloud-platform) is always included.", + "widget-attributes" : { + "value-placeholder": "" + } + }, { "name": "initActions", "label": "Initialization Actions", diff --git a/cdap-runtime-ext-dataproc/src/test/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocProvisionerTest.java b/cdap-runtime-ext-dataproc/src/test/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocProvisionerTest.java index 5826cb539ecb..fa5881ed5ef2 100644 --- a/cdap-runtime-ext-dataproc/src/test/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocProvisionerTest.java +++ b/cdap-runtime-ext-dataproc/src/test/java/io/cdap/cdap/runtime/spi/provisioner/dataproc/DataprocProvisionerTest.java @@ -121,6 +121,9 @@ public void testRunKey() throws Exception { @Test public void testDataprocConf() { Map props = new HashMap<>(); + String scopes = String.format("%s,https://www.googleapis.com/auth/drive", + DataprocConf.CLOUD_PLATFORM_SCOPE); + props.put(DataprocConf.PROJECT_ID_KEY, "pid"); props.put("accountKey", "key"); props.put("region", "region1"); @@ -138,6 +141,7 @@ public void testDataprocConf() { props.put("idleTTL", "20"); props.put("clusterReuseRetryDelayMs", "20"); props.put("clusterReuseRetryMaxMs", "200"); + props.put(DataprocConf.SCOPES, scopes); DataprocConf conf = DataprocConf.create(props); @@ -167,6 +171,7 @@ public void testDataprocConf() { Assert.assertEquals(20, conf.getClusterReuseRetryDelayMs()); Assert.assertEquals(200, conf.getClusterReuseRetryMaxMs()); + Assert.assertEquals(scopes, String.join(",", conf.getScopes())); } @Test