Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master'
Browse files Browse the repository at this point in the history 
* upstream/master:
  [GOBBLIN-1832] Emit warning instead of failing job for retention of Hive Table Views (apache#3695)
  [GOBBLIN-1831] Use flowexecutionid in kafka monitor and jobnames (apache#3694)
  [GOBBLIN-1824]Improving the Efficiency of Work Planning in Manifest-Based DistCp Jobs (apache#3686)
  [GOBBLIN-1829] Fixes bug where the wrong workunit event was being tracked for keepin… (apache#3691)
  [GOBBLIN-1828] Implement Timeout for Creating Writer Functionality (apache#3690)
  [GOBBLIN-1827] Add check that if nested field is optional and has a non-null default… (apache#3689)
  [GOBBLIN-1826] Change isAssignableFrom() to isSuperTypeOf() per Guava 20 javadocs to… (apache#3688)
  [GOBBLIN-1822]Logging Abnormal Helix Task States (apache#3685)
  [GOBBLIN-1819] Log helix workflow information and timeout information during submission wait / polling (apache#3681)
  [GOBBLIN-1821] Let flow execution ID propagate to the Job ID if it exists (apache#3684)
  [GOBBLIN-1810] Support general iceberg catalog (support configurable behavior for metadata retention policy) (apache#3680)
  Add null default value to observability events that are additionally added (apache#3682)
  [GOBBLIN-1816] Add job properties and GaaS instance ID to observability event (apache#3676)
  [GOBBLIN-1785] add MR_JARS_BASE_DIR and logic to delete old mr jar dirs (apache#3642)
  initiliaze yarn clients in yarn app launcher so that a child class can override the yarn client creation logic (apache#3679)
  [GOBBLIN-1811]Fix Iceberg Registration Serialization (apache#3673)
  [GOBBLIN-1817] change some deprecated code and fix minor codestyle (apache#3678)
  [GOBBLIN-1812] Mockito should only be test compile (apache#3674)
  [GOBBLIN-1813] Helix workflows submission timeouts are configurable (apache#3677)
  [GOBBLIN-1810] Support general iceberg catalog in icebergMetadataWriter (apache#3672)
  Refactor yarn app launchers to support extending these classes (apache#3671)
  [GOBBLIN-1808] Bump Guava version from 15.0 to 20.0 (apache#3669)
  [GOBBLIN-1806] Submit dataset summary event post commit and integrate them into GaaSObservabilityEvent (apache#3667)
  [GOBBLIN-1814] Add `MRJobLauncher` configurability for any failing mapper to be fatal to the MR job (apache#3675)
  Add new lookback version finder for use with iceberg retention (apache#3670)
  • Loading branch information
@phet
phet committed May 17, 2023
2 parents 4e4f95a + 05c732c commit 90dd1ff
Show file tree
Hide file tree
Showing 88 changed files with 1,323 additions and 337 deletions.
2 changes: 1 addition & 1 deletion bin/gobblin-compaction.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ else
$FWDIR_LIB/gobblin-api-$GOBBLIN_VERSION.jar
$FWDIR_LIB/gobblin-compaction-$GOBBLIN_VERSION.jar
$FWDIR_LIB/gobblin-utility-$GOBBLIN_VERSION.jar
$FWDIR_LIB/guava-15.0.jar
$FWDIR_LIB/guava-20.0.jar
)
LIBJARS=$(join , "${LIBJARS[@]}")

Expand Down
2 changes: 1 addition & 1 deletion conf/aws/application.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ gobblin.aws.master.s3.jars.uri="https://s3-us-west-2.amazonaws.com/gobblin-libs/
## All Gobblin Jars and Configuration on S3
gobblin.aws.master.s3.conf.files="application.conf,log4j.properties,quartz.properties"
gobblin.aws.master.s3.jars.uri="https://s3-us-west-2.amazonaws.com/gobblin-libs/latest-jars/"
gobblin.aws.master.s3.jars.files="ST4-4.0.4.jar,activation-1.1.1.jar,annotations-2.0.1.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.2.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,asm-3.1.jar,asm-commons-3.1.jar,asm-tree-3.1.jar,avro-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-ipc-1.7.7.jar,avro-mapred-1.7.7-hadoop2.jar,aws-java-sdk-applicationautoscaling-1.11.8.jar,aws-java-sdk-autoscaling-1.11.8.jar,aws-java-sdk-core-1.11.8.jar,aws-java-sdk-ec2-1.11.8.jar,aws-java-sdk-iam-1.11.8.jar,aws-java-sdk-kms-1.11.8.jar,aws-java-sdk-s3-1.11.8.jar,aws-java-sdk-sts-1.11.8.jar,azkaban-2.5.0.jar,bcpg-jdk15on-1.52.jar,bcprov-jdk15on-1.52.jar,bonecp-0.8.0.RELEASE.jar,bsh-2.0b4.jar,c3p0-0.9.1.1.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,cglib-nodep-2.2.jar,codemodel-2.2.jar,commons-cli-1.3.1.jar,commons-codec-1.10.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.10.jar,commons-configuration-1.10.jar,commons-daemon-1.0.13.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-email-1.4.jar,commons-httpclient-3.1.jar,commons-io-2.5.jar,commons-lang-2.6.jar,commons-lang3-3.4.jar,commons-logging-1.2.jar,commons-math3-3.5.jar,commons-net-3.1.jar,commons-pool-1.5.4.jar,commons-pool2-2.4.2.jar,commons-vfs2-2.0.jar,config-1.2.1.jar,curator-client-2.10.0.jar,curator-framework-2.10.0.jar,curator-recipes-2.10.0.jar,d2-1.15.9.jar,data-1.15.9.jar,data-transform-1.15.9.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,degrader-1.15.9.jar,derby-10.12.1.1.jar,eigenbase-properties-1.1.5.jar,flyway-core-3.2.1.jar,generator-1.15.9.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jpa_3.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,gobblin-admin-"${gobblin.aws.version}".jar,gobblin-api-"${gobblin.aws.version}".jar,gobblin-aws-"${gobblin.aws.version}".jar,gobblin-azkaban-"${gobblin.aws.version}".jar,gobblin-cluster-"${gobblin.aws.version}".jar,gobblin-compaction-"${gobblin.aws.version}".jar,gobblin-config-client-"${gobblin.aws.version}".jar,gobblin-config-core-"${gobblin.aws.version}".jar,gobblin-core-"${gobblin.aws.version}".jar,gobblin-data-management-"${gobblin.aws.version}".jar,gobblin-example-"${gobblin.aws.version}".jar,gobblin-hive-registration-"${gobblin.aws.version}".jar,gobblin-iceberg-"${gobblin.aws.version}".jar,gobblin-kafka-"${gobblin.aws.version}".jar,gobblin-metastore-"${gobblin.aws.version}".jar,gobblin-metrics-"${gobblin.aws.version}".jar,gobblin-rest-api-"${gobblin.aws.version}".jar,gobblin-rest-api-data-template-"${gobblin.aws.version}".jar,gobblin-rest-api-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-server-"${gobblin.aws.version}".jar,gobblin-runtime-"${gobblin.aws.version}".jar,gobblin-test-harness-"${gobblin.aws.version}".jar,gobblin-tunnel-"${gobblin.aws.version}".jar,gobblin-utility-"${gobblin.aws.version}".jar,gobblin-yarn-"${gobblin.aws.version}".jar,groovy-all-2.1.6.jar,gson-2.6.2.jar,guava-15.0.jar,guava-retrying-2.0.0.jar,guice-4.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.3.0.jar,hadoop-auth-2.3.0.jar,hadoop-common-2.3.0.jar,hadoop-hdfs-2.3.0.jar,hadoop-mapreduce-client-common-2.3.0.jar,hadoop-mapreduce-client-core-2.3.0.jar,hadoop-yarn-api-2.3.0.jar,hadoop-yarn-client-2.3.0.jar,hadoop-yarn-common-2.3.0.jar,hadoop-yarn-server-common-2.3.0.jar,hamcrest-core-1.1.jar,helix-core-0.6.6-SNAPSHOT.jar,hive-ant-1.0.1.jar,hive-common-1.0.1.jar,hive-exec-1.0.1-core.jar,hive-jdbc-1.0.1.jar,hive-metastore-1.0.1.jar,hive-serde-1.0.1.jar,hive-service-1.0.1.jar,hive-shims-0.20-1.0.1.jar,hive-shims-0.20S-1.0.1.jar,hive-shims-0.23-1.0.1.jar,hive-shims-1.0.1.jar,hive-shims-common-1.0.1.jar,hive-shims-common-secure-1.0.1.jar,httpclient-4.5.2.jar,httpcore-4.4.4.jar,influxdb-java-2.1.jar,jackson-annotations-2.6.0.jar,jackson-core-2.6.6.jar,jackson-core-asl-1.9.13.jar,jackson-databind-2.6.6.jar,jackson-dataformat-cbor-2.6.6.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.13.jar,jackson-xc-1.8.3.jar,janino-2.7.6.jar,jansi-1.11.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,jasypt-1.9.2.jar,java-xmlbuilder-0.4.jar,javassist-3.18.2-GA.jar,javax.inject-1.jar,javax.mail-1.5.2.jar,javax.servlet-api-3.1.0.jar,jaxb-api-2.2.2.jar,jaxb-impl-2.2.3-1.jar,jcommander-1.48.jar,jdo-api-3.0.1.jar,jdo2-api-2.1.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.9.0.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-http-9.2.14.v20151106.jar,jetty-io-9.2.14.v20151106.jar,jetty-security-9.2.14.v20151106.jar,jetty-server-9.2.14.v20151106.jar,jetty-servlet-9.2.14.v20151106.jar,jetty-util-6.1.26.jar,jetty-util-9.2.14.v20151106.jar,jline-0.9.94.jar,joda-time-2.9.3.jar,jopt-simple-3.2.jar,jpam-1.1.jar,jsch-0.1.53.jar,json-20070829.jar,jsp-api-2.1.jar,jsr305-3.0.0.jar,jta-1.1.jar,junit-3.8.1.jar,kafka-clients-0.8.2.2.jar,kafka_2.11-0.8.2.2.jar,li-jersey-uri-1.15.9.jar,libfb303-0.9.0.jar,libthrift-0.9.3.jar,log4j-1.2.17.jar,lombok-1.16.8.jar,lz4-1.2.0.jar,mail-1.4.1.jar,maven-scm-api-1.4.jar,maven-scm-provider-svn-commons-1.4.jar,maven-scm-provider-svnexe-1.4.jar,metrics-core-2.2.0.jar,metrics-core-3.1.0.jar,metrics-graphite-3.1.0.jar,metrics-jvm-3.1.0.jar,mina-core-1.1.7.jar,mockito-core-1.10.19.jar,mysql-connector-java-5.1.38.jar,netty-3.2.3.Final.jar,netty-3.7.0.Final.jar,objenesis-2.1.jar,okhttp-2.4.0.jar,okio-1.4.0.jar,opencsv-2.3.jar,paranamer-2.3.jar,parseq-1.3.6.jar,pegasus-common-1.15.9.jar,pentaho-aggdesigner-algorithm-5.1.5-jhyde.jar,plexus-utils-1.5.6.jar,protobuf-java-2.5.0.jar,quartz-2.2.3.jar,r2-1.15.9.jar,reflections-0.9.10.jar,regexp-1.3.jar,restli-client-1.15.9.jar,restli-common-1.15.9.jar,restli-docgen-1.15.9.jar,restli-netty-standalone-1.15.9.jar,restli-server-1.15.9.jar,restli-tools-1.15.9.jar,retrofit-1.9.0.jar,scala-library-2.11.8.jar,scala-parser-combinators_2.11-1.0.2.jar,scala-xml_2.11-1.0.2.jar,servlet-api-2.5-20081211.jar,servlet-api-2.5.jar,slf4j-api-1.7.21.jar,slf4j-log4j12-1.7.21.jar,snappy-0.3.jar,snappy-java-1.1.1.7.jar,stax-api-1.0-2.jar,stax-api-1.0.1.jar,testng-6.9.10.jar,transaction-api-1.1.jar,velocity-1.7.jar,xmlenc-0.52.jar,zkclient-0.5.jar,zookeeper-3.4.6.jar"
gobblin.aws.master.s3.jars.files="ST4-4.0.4.jar,activation-1.1.1.jar,annotations-2.0.1.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.2.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,asm-3.1.jar,asm-commons-3.1.jar,asm-tree-3.1.jar,avro-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-ipc-1.7.7.jar,avro-mapred-1.7.7-hadoop2.jar,aws-java-sdk-applicationautoscaling-1.11.8.jar,aws-java-sdk-autoscaling-1.11.8.jar,aws-java-sdk-core-1.11.8.jar,aws-java-sdk-ec2-1.11.8.jar,aws-java-sdk-iam-1.11.8.jar,aws-java-sdk-kms-1.11.8.jar,aws-java-sdk-s3-1.11.8.jar,aws-java-sdk-sts-1.11.8.jar,azkaban-2.5.0.jar,bcpg-jdk15on-1.52.jar,bcprov-jdk15on-1.52.jar,bonecp-0.8.0.RELEASE.jar,bsh-2.0b4.jar,c3p0-0.9.1.1.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,cglib-nodep-2.2.jar,codemodel-2.2.jar,commons-cli-1.3.1.jar,commons-codec-1.10.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.10.jar,commons-configuration-1.10.jar,commons-daemon-1.0.13.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-email-1.4.jar,commons-httpclient-3.1.jar,commons-io-2.5.jar,commons-lang-2.6.jar,commons-lang3-3.4.jar,commons-logging-1.2.jar,commons-math3-3.5.jar,commons-net-3.1.jar,commons-pool-1.5.4.jar,commons-pool2-2.4.2.jar,commons-vfs2-2.0.jar,config-1.2.1.jar,curator-client-2.10.0.jar,curator-framework-2.10.0.jar,curator-recipes-2.10.0.jar,d2-1.15.9.jar,data-1.15.9.jar,data-transform-1.15.9.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,degrader-1.15.9.jar,derby-10.12.1.1.jar,eigenbase-properties-1.1.5.jar,flyway-core-3.2.1.jar,generator-1.15.9.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jpa_3.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,gobblin-admin-"${gobblin.aws.version}".jar,gobblin-api-"${gobblin.aws.version}".jar,gobblin-aws-"${gobblin.aws.version}".jar,gobblin-azkaban-"${gobblin.aws.version}".jar,gobblin-cluster-"${gobblin.aws.version}".jar,gobblin-compaction-"${gobblin.aws.version}".jar,gobblin-config-client-"${gobblin.aws.version}".jar,gobblin-config-core-"${gobblin.aws.version}".jar,gobblin-core-"${gobblin.aws.version}".jar,gobblin-data-management-"${gobblin.aws.version}".jar,gobblin-example-"${gobblin.aws.version}".jar,gobblin-hive-registration-"${gobblin.aws.version}".jar,gobblin-iceberg-"${gobblin.aws.version}".jar,gobblin-kafka-"${gobblin.aws.version}".jar,gobblin-metastore-"${gobblin.aws.version}".jar,gobblin-metrics-"${gobblin.aws.version}".jar,gobblin-rest-api-"${gobblin.aws.version}".jar,gobblin-rest-api-data-template-"${gobblin.aws.version}".jar,gobblin-rest-api-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-server-"${gobblin.aws.version}".jar,gobblin-runtime-"${gobblin.aws.version}".jar,gobblin-test-harness-"${gobblin.aws.version}".jar,gobblin-tunnel-"${gobblin.aws.version}".jar,gobblin-utility-"${gobblin.aws.version}".jar,gobblin-yarn-"${gobblin.aws.version}".jar,groovy-all-2.1.6.jar,gson-2.6.2.jar,guava-20.0.jar,guava-retrying-2.0.0.jar,guice-4.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.3.0.jar,hadoop-auth-2.3.0.jar,hadoop-common-2.3.0.jar,hadoop-hdfs-2.3.0.jar,hadoop-mapreduce-client-common-2.3.0.jar,hadoop-mapreduce-client-core-2.3.0.jar,hadoop-yarn-api-2.3.0.jar,hadoop-yarn-client-2.3.0.jar,hadoop-yarn-common-2.3.0.jar,hadoop-yarn-server-common-2.3.0.jar,hamcrest-core-1.1.jar,helix-core-0.6.6-SNAPSHOT.jar,hive-ant-1.0.1.jar,hive-common-1.0.1.jar,hive-exec-1.0.1-core.jar,hive-jdbc-1.0.1.jar,hive-metastore-1.0.1.jar,hive-serde-1.0.1.jar,hive-service-1.0.1.jar,hive-shims-0.20-1.0.1.jar,hive-shims-0.20S-1.0.1.jar,hive-shims-0.23-1.0.1.jar,hive-shims-1.0.1.jar,hive-shims-common-1.0.1.jar,hive-shims-common-secure-1.0.1.jar,httpclient-4.5.2.jar,httpcore-4.4.4.jar,influxdb-java-2.1.jar,jackson-annotations-2.6.0.jar,jackson-core-2.6.6.jar,jackson-core-asl-1.9.13.jar,jackson-databind-2.6.6.jar,jackson-dataformat-cbor-2.6.6.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.13.jar,jackson-xc-1.8.3.jar,janino-2.7.6.jar,jansi-1.11.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,jasypt-1.9.2.jar,java-xmlbuilder-0.4.jar,javassist-3.18.2-GA.jar,javax.inject-1.jar,javax.mail-1.5.2.jar,javax.servlet-api-3.1.0.jar,jaxb-api-2.2.2.jar,jaxb-impl-2.2.3-1.jar,jcommander-1.48.jar,jdo-api-3.0.1.jar,jdo2-api-2.1.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.9.0.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-http-9.2.14.v20151106.jar,jetty-io-9.2.14.v20151106.jar,jetty-security-9.2.14.v20151106.jar,jetty-server-9.2.14.v20151106.jar,jetty-servlet-9.2.14.v20151106.jar,jetty-util-6.1.26.jar,jetty-util-9.2.14.v20151106.jar,jline-0.9.94.jar,joda-time-2.9.3.jar,jopt-simple-3.2.jar,jpam-1.1.jar,jsch-0.1.53.jar,json-20070829.jar,jsp-api-2.1.jar,jsr305-3.0.0.jar,jta-1.1.jar,junit-3.8.1.jar,kafka-clients-0.8.2.2.jar,kafka_2.11-0.8.2.2.jar,li-jersey-uri-1.15.9.jar,libfb303-0.9.0.jar,libthrift-0.9.3.jar,log4j-1.2.17.jar,lombok-1.16.8.jar,lz4-1.2.0.jar,mail-1.4.1.jar,maven-scm-api-1.4.jar,maven-scm-provider-svn-commons-1.4.jar,maven-scm-provider-svnexe-1.4.jar,metrics-core-2.2.0.jar,metrics-core-3.1.0.jar,metrics-graphite-3.1.0.jar,metrics-jvm-3.1.0.jar,mina-core-1.1.7.jar,mockito-core-1.10.19.jar,mysql-connector-java-5.1.38.jar,netty-3.2.3.Final.jar,netty-3.7.0.Final.jar,objenesis-2.1.jar,okhttp-2.4.0.jar,okio-1.4.0.jar,opencsv-2.3.jar,paranamer-2.3.jar,parseq-1.3.6.jar,pegasus-common-1.15.9.jar,pentaho-aggdesigner-algorithm-5.1.5-jhyde.jar,plexus-utils-1.5.6.jar,protobuf-java-2.5.0.jar,quartz-2.2.3.jar,r2-1.15.9.jar,reflections-0.9.10.jar,regexp-1.3.jar,restli-client-1.15.9.jar,restli-common-1.15.9.jar,restli-docgen-1.15.9.jar,restli-netty-standalone-1.15.9.jar,restli-server-1.15.9.jar,restli-tools-1.15.9.jar,retrofit-1.9.0.jar,scala-library-2.11.8.jar,scala-parser-combinators_2.11-1.0.2.jar,scala-xml_2.11-1.0.2.jar,servlet-api-2.5-20081211.jar,servlet-api-2.5.jar,slf4j-api-1.7.21.jar,slf4j-log4j12-1.7.21.jar,snappy-0.3.jar,snappy-java-1.1.1.7.jar,stax-api-1.0-2.jar,stax-api-1.0.1.jar,testng-6.9.10.jar,transaction-api-1.1.jar,velocity-1.7.jar,xmlenc-0.52.jar,zkclient-0.5.jar,zookeeper-3.4.6.jar"

gobblin.aws.worker.s3.conf.uri=${gobblin.aws.master.s3.conf.uri}
gobblin.aws.worker.s3.conf.files=${gobblin.aws.master.s3.conf.files}
Expand Down
7 changes: 7 additions & 0 deletions gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,7 @@ public class ConfigurationKeys {
public static final String WRITER_BYTES_WRITTEN = WRITER_PREFIX + ".bytes.written";
public static final String WRITER_EARLIEST_TIMESTAMP = WRITER_PREFIX + ".earliest.timestamp";
public static final String WRITER_AVERAGE_TIMESTAMP = WRITER_PREFIX + ".average.timestamp";
public static final String WRITER_COUNT_METRICS_FROM_FAILED_TASKS = WRITER_PREFIX + ".jobTaskSummary.countMetricsFromFailedTasks";

/**
* Configuration properties used by the quality checker.
Expand Down Expand Up @@ -686,12 +687,18 @@ public class ConfigurationKeys {
*/
public static final String MR_JOB_ROOT_DIR_KEY = "mr.job.root.dir";
/** Specifies a static location in HDFS to upload jars to. Useful for sharing jars across different Gobblin runs.*/
@Deprecated // Deprecated; use MR_JARS_BASE_DIR instead
public static final String MR_JARS_DIR = "mr.jars.dir";
// dir pointed by MR_JARS_BASE_DIR has month partitioned dirs to store jar files and are cleaned up on a regular basis
// retention feature is not available for dir pointed by MR_JARS_DIR
public static final String MR_JARS_BASE_DIR = "mr.jars.base.dir";
public static final String MR_JOB_MAX_MAPPERS_KEY = "mr.job.max.mappers";
public static final String MR_JOB_MAPPER_FAILURE_IS_FATAL_KEY = "mr.job.map.failure.is.fatal";
public static final String MR_TARGET_MAPPER_SIZE = "mr.target.mapper.size";
public static final String MR_REPORT_METRICS_AS_COUNTERS_KEY = "mr.report.metrics.as.counters";
public static final boolean DEFAULT_MR_REPORT_METRICS_AS_COUNTERS = false;
public static final int DEFAULT_MR_JOB_MAX_MAPPERS = 100;
public static final boolean DEFAULT_MR_JOB_MAPPER_FAILURE_IS_FATAL = false;
public static final String DEFAULT_ENABLE_MR_SPECULATIVE_EXECUTION = "false";

/**
Expand Down
2 changes: 1 addition & 1 deletion gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public class ServiceConfigKeys {
public static final String GOBBLIN_SERVICE_TOPOLOGY_CATALOG_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "topologyCatalog.enabled";
public static final String GOBBLIN_SERVICE_FLOW_CATALOG_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "flowCatalog.enabled";
public static final String GOBBLIN_SERVICE_SCHEDULER_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "scheduler.enabled";

public static final String GOBBLIN_SERVICE_INSTANCE_NAME = GOBBLIN_SERVICE_PREFIX + "instance.name";

public static final String GOBBLIN_SERVICE_RESTLI_SERVER_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "restliServer.enabled";
public static final String GOBBLIN_SERVICE_TOPOLOGY_SPEC_FACTORY_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "topologySpecFactory.enabled";
Expand Down
Loading

0 comments on commit 90dd1ff

Please sign in to comment.