diff --git a/bin/gobblin-compaction.sh b/bin/gobblin-compaction.sh index baa8a669505..1924f385a01 100755 --- a/bin/gobblin-compaction.sh +++ b/bin/gobblin-compaction.sh @@ -131,7 +131,7 @@ else $FWDIR_LIB/gobblin-api-$GOBBLIN_VERSION.jar $FWDIR_LIB/gobblin-compaction-$GOBBLIN_VERSION.jar $FWDIR_LIB/gobblin-utility-$GOBBLIN_VERSION.jar - $FWDIR_LIB/guava-15.0.jar + $FWDIR_LIB/guava-20.0.jar ) LIBJARS=$(join , "${LIBJARS[@]}") diff --git a/conf/aws/application.conf b/conf/aws/application.conf index 5b085072792..ccbf3ba864e 100644 --- a/conf/aws/application.conf +++ b/conf/aws/application.conf @@ -159,7 +159,7 @@ gobblin.aws.master.s3.jars.uri="https://s3-us-west-2.amazonaws.com/gobblin-libs/ ## All Gobblin Jars and Configuration on S3 gobblin.aws.master.s3.conf.files="application.conf,log4j.properties,quartz.properties" gobblin.aws.master.s3.jars.uri="https://s3-us-west-2.amazonaws.com/gobblin-libs/latest-jars/" -gobblin.aws.master.s3.jars.files="ST4-4.0.4.jar,activation-1.1.1.jar,annotations-2.0.1.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.2.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,asm-3.1.jar,asm-commons-3.1.jar,asm-tree-3.1.jar,avro-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-ipc-1.7.7.jar,avro-mapred-1.7.7-hadoop2.jar,aws-java-sdk-applicationautoscaling-1.11.8.jar,aws-java-sdk-autoscaling-1.11.8.jar,aws-java-sdk-core-1.11.8.jar,aws-java-sdk-ec2-1.11.8.jar,aws-java-sdk-iam-1.11.8.jar,aws-java-sdk-kms-1.11.8.jar,aws-java-sdk-s3-1.11.8.jar,aws-java-sdk-sts-1.11.8.jar,azkaban-2.5.0.jar,bcpg-jdk15on-1.52.jar,bcprov-jdk15on-1.52.jar,bonecp-0.8.0.RELEASE.jar,bsh-2.0b4.jar,c3p0-0.9.1.1.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,cglib-nodep-2.2.jar,codemodel-2.2.jar,commons-cli-1.3.1.jar,commons-codec-1.10.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.10.jar,commons-configuration-1.10.jar,commons-daemon-1.0.13.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-email-1.4.jar,commons-httpclient-3.1.jar,commons-io-2.5.jar,commons-lang-2.6.jar,commons-lang3-3.4.jar,commons-logging-1.2.jar,commons-math3-3.5.jar,commons-net-3.1.jar,commons-pool-1.5.4.jar,commons-pool2-2.4.2.jar,commons-vfs2-2.0.jar,config-1.2.1.jar,curator-client-2.10.0.jar,curator-framework-2.10.0.jar,curator-recipes-2.10.0.jar,d2-1.15.9.jar,data-1.15.9.jar,data-transform-1.15.9.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,degrader-1.15.9.jar,derby-10.12.1.1.jar,eigenbase-properties-1.1.5.jar,flyway-core-3.2.1.jar,generator-1.15.9.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jpa_3.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,gobblin-admin-"${gobblin.aws.version}".jar,gobblin-api-"${gobblin.aws.version}".jar,gobblin-aws-"${gobblin.aws.version}".jar,gobblin-azkaban-"${gobblin.aws.version}".jar,gobblin-cluster-"${gobblin.aws.version}".jar,gobblin-compaction-"${gobblin.aws.version}".jar,gobblin-config-client-"${gobblin.aws.version}".jar,gobblin-config-core-"${gobblin.aws.version}".jar,gobblin-core-"${gobblin.aws.version}".jar,gobblin-data-management-"${gobblin.aws.version}".jar,gobblin-example-"${gobblin.aws.version}".jar,gobblin-hive-registration-"${gobblin.aws.version}".jar,gobblin-iceberg-"${gobblin.aws.version}".jar,gobblin-kafka-"${gobblin.aws.version}".jar,gobblin-metastore-"${gobblin.aws.version}".jar,gobblin-metrics-"${gobblin.aws.version}".jar,gobblin-rest-api-"${gobblin.aws.version}".jar,gobblin-rest-api-data-template-"${gobblin.aws.version}".jar,gobblin-rest-api-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-server-"${gobblin.aws.version}".jar,gobblin-runtime-"${gobblin.aws.version}".jar,gobblin-test-harness-"${gobblin.aws.version}".jar,gobblin-tunnel-"${gobblin.aws.version}".jar,gobblin-utility-"${gobblin.aws.version}".jar,gobblin-yarn-"${gobblin.aws.version}".jar,groovy-all-2.1.6.jar,gson-2.6.2.jar,guava-15.0.jar,guava-retrying-2.0.0.jar,guice-4.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.3.0.jar,hadoop-auth-2.3.0.jar,hadoop-common-2.3.0.jar,hadoop-hdfs-2.3.0.jar,hadoop-mapreduce-client-common-2.3.0.jar,hadoop-mapreduce-client-core-2.3.0.jar,hadoop-yarn-api-2.3.0.jar,hadoop-yarn-client-2.3.0.jar,hadoop-yarn-common-2.3.0.jar,hadoop-yarn-server-common-2.3.0.jar,hamcrest-core-1.1.jar,helix-core-0.6.6-SNAPSHOT.jar,hive-ant-1.0.1.jar,hive-common-1.0.1.jar,hive-exec-1.0.1-core.jar,hive-jdbc-1.0.1.jar,hive-metastore-1.0.1.jar,hive-serde-1.0.1.jar,hive-service-1.0.1.jar,hive-shims-0.20-1.0.1.jar,hive-shims-0.20S-1.0.1.jar,hive-shims-0.23-1.0.1.jar,hive-shims-1.0.1.jar,hive-shims-common-1.0.1.jar,hive-shims-common-secure-1.0.1.jar,httpclient-4.5.2.jar,httpcore-4.4.4.jar,influxdb-java-2.1.jar,jackson-annotations-2.6.0.jar,jackson-core-2.6.6.jar,jackson-core-asl-1.9.13.jar,jackson-databind-2.6.6.jar,jackson-dataformat-cbor-2.6.6.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.13.jar,jackson-xc-1.8.3.jar,janino-2.7.6.jar,jansi-1.11.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,jasypt-1.9.2.jar,java-xmlbuilder-0.4.jar,javassist-3.18.2-GA.jar,javax.inject-1.jar,javax.mail-1.5.2.jar,javax.servlet-api-3.1.0.jar,jaxb-api-2.2.2.jar,jaxb-impl-2.2.3-1.jar,jcommander-1.48.jar,jdo-api-3.0.1.jar,jdo2-api-2.1.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.9.0.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-http-9.2.14.v20151106.jar,jetty-io-9.2.14.v20151106.jar,jetty-security-9.2.14.v20151106.jar,jetty-server-9.2.14.v20151106.jar,jetty-servlet-9.2.14.v20151106.jar,jetty-util-6.1.26.jar,jetty-util-9.2.14.v20151106.jar,jline-0.9.94.jar,joda-time-2.9.3.jar,jopt-simple-3.2.jar,jpam-1.1.jar,jsch-0.1.53.jar,json-20070829.jar,jsp-api-2.1.jar,jsr305-3.0.0.jar,jta-1.1.jar,junit-3.8.1.jar,kafka-clients-0.8.2.2.jar,kafka_2.11-0.8.2.2.jar,li-jersey-uri-1.15.9.jar,libfb303-0.9.0.jar,libthrift-0.9.3.jar,log4j-1.2.17.jar,lombok-1.16.8.jar,lz4-1.2.0.jar,mail-1.4.1.jar,maven-scm-api-1.4.jar,maven-scm-provider-svn-commons-1.4.jar,maven-scm-provider-svnexe-1.4.jar,metrics-core-2.2.0.jar,metrics-core-3.1.0.jar,metrics-graphite-3.1.0.jar,metrics-jvm-3.1.0.jar,mina-core-1.1.7.jar,mockito-core-1.10.19.jar,mysql-connector-java-5.1.38.jar,netty-3.2.3.Final.jar,netty-3.7.0.Final.jar,objenesis-2.1.jar,okhttp-2.4.0.jar,okio-1.4.0.jar,opencsv-2.3.jar,paranamer-2.3.jar,parseq-1.3.6.jar,pegasus-common-1.15.9.jar,pentaho-aggdesigner-algorithm-5.1.5-jhyde.jar,plexus-utils-1.5.6.jar,protobuf-java-2.5.0.jar,quartz-2.2.3.jar,r2-1.15.9.jar,reflections-0.9.10.jar,regexp-1.3.jar,restli-client-1.15.9.jar,restli-common-1.15.9.jar,restli-docgen-1.15.9.jar,restli-netty-standalone-1.15.9.jar,restli-server-1.15.9.jar,restli-tools-1.15.9.jar,retrofit-1.9.0.jar,scala-library-2.11.8.jar,scala-parser-combinators_2.11-1.0.2.jar,scala-xml_2.11-1.0.2.jar,servlet-api-2.5-20081211.jar,servlet-api-2.5.jar,slf4j-api-1.7.21.jar,slf4j-log4j12-1.7.21.jar,snappy-0.3.jar,snappy-java-1.1.1.7.jar,stax-api-1.0-2.jar,stax-api-1.0.1.jar,testng-6.9.10.jar,transaction-api-1.1.jar,velocity-1.7.jar,xmlenc-0.52.jar,zkclient-0.5.jar,zookeeper-3.4.6.jar" +gobblin.aws.master.s3.jars.files="ST4-4.0.4.jar,activation-1.1.1.jar,annotations-2.0.1.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.2.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,asm-3.1.jar,asm-commons-3.1.jar,asm-tree-3.1.jar,avro-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-ipc-1.7.7.jar,avro-mapred-1.7.7-hadoop2.jar,aws-java-sdk-applicationautoscaling-1.11.8.jar,aws-java-sdk-autoscaling-1.11.8.jar,aws-java-sdk-core-1.11.8.jar,aws-java-sdk-ec2-1.11.8.jar,aws-java-sdk-iam-1.11.8.jar,aws-java-sdk-kms-1.11.8.jar,aws-java-sdk-s3-1.11.8.jar,aws-java-sdk-sts-1.11.8.jar,azkaban-2.5.0.jar,bcpg-jdk15on-1.52.jar,bcprov-jdk15on-1.52.jar,bonecp-0.8.0.RELEASE.jar,bsh-2.0b4.jar,c3p0-0.9.1.1.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,cglib-nodep-2.2.jar,codemodel-2.2.jar,commons-cli-1.3.1.jar,commons-codec-1.10.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.10.jar,commons-configuration-1.10.jar,commons-daemon-1.0.13.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-email-1.4.jar,commons-httpclient-3.1.jar,commons-io-2.5.jar,commons-lang-2.6.jar,commons-lang3-3.4.jar,commons-logging-1.2.jar,commons-math3-3.5.jar,commons-net-3.1.jar,commons-pool-1.5.4.jar,commons-pool2-2.4.2.jar,commons-vfs2-2.0.jar,config-1.2.1.jar,curator-client-2.10.0.jar,curator-framework-2.10.0.jar,curator-recipes-2.10.0.jar,d2-1.15.9.jar,data-1.15.9.jar,data-transform-1.15.9.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,degrader-1.15.9.jar,derby-10.12.1.1.jar,eigenbase-properties-1.1.5.jar,flyway-core-3.2.1.jar,generator-1.15.9.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jpa_3.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,gobblin-admin-"${gobblin.aws.version}".jar,gobblin-api-"${gobblin.aws.version}".jar,gobblin-aws-"${gobblin.aws.version}".jar,gobblin-azkaban-"${gobblin.aws.version}".jar,gobblin-cluster-"${gobblin.aws.version}".jar,gobblin-compaction-"${gobblin.aws.version}".jar,gobblin-config-client-"${gobblin.aws.version}".jar,gobblin-config-core-"${gobblin.aws.version}".jar,gobblin-core-"${gobblin.aws.version}".jar,gobblin-data-management-"${gobblin.aws.version}".jar,gobblin-example-"${gobblin.aws.version}".jar,gobblin-hive-registration-"${gobblin.aws.version}".jar,gobblin-iceberg-"${gobblin.aws.version}".jar,gobblin-kafka-"${gobblin.aws.version}".jar,gobblin-metastore-"${gobblin.aws.version}".jar,gobblin-metrics-"${gobblin.aws.version}".jar,gobblin-rest-api-"${gobblin.aws.version}".jar,gobblin-rest-api-data-template-"${gobblin.aws.version}".jar,gobblin-rest-api-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-server-"${gobblin.aws.version}".jar,gobblin-runtime-"${gobblin.aws.version}".jar,gobblin-test-harness-"${gobblin.aws.version}".jar,gobblin-tunnel-"${gobblin.aws.version}".jar,gobblin-utility-"${gobblin.aws.version}".jar,gobblin-yarn-"${gobblin.aws.version}".jar,groovy-all-2.1.6.jar,gson-2.6.2.jar,guava-20.0.jar,guava-retrying-2.0.0.jar,guice-4.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.3.0.jar,hadoop-auth-2.3.0.jar,hadoop-common-2.3.0.jar,hadoop-hdfs-2.3.0.jar,hadoop-mapreduce-client-common-2.3.0.jar,hadoop-mapreduce-client-core-2.3.0.jar,hadoop-yarn-api-2.3.0.jar,hadoop-yarn-client-2.3.0.jar,hadoop-yarn-common-2.3.0.jar,hadoop-yarn-server-common-2.3.0.jar,hamcrest-core-1.1.jar,helix-core-0.6.6-SNAPSHOT.jar,hive-ant-1.0.1.jar,hive-common-1.0.1.jar,hive-exec-1.0.1-core.jar,hive-jdbc-1.0.1.jar,hive-metastore-1.0.1.jar,hive-serde-1.0.1.jar,hive-service-1.0.1.jar,hive-shims-0.20-1.0.1.jar,hive-shims-0.20S-1.0.1.jar,hive-shims-0.23-1.0.1.jar,hive-shims-1.0.1.jar,hive-shims-common-1.0.1.jar,hive-shims-common-secure-1.0.1.jar,httpclient-4.5.2.jar,httpcore-4.4.4.jar,influxdb-java-2.1.jar,jackson-annotations-2.6.0.jar,jackson-core-2.6.6.jar,jackson-core-asl-1.9.13.jar,jackson-databind-2.6.6.jar,jackson-dataformat-cbor-2.6.6.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.13.jar,jackson-xc-1.8.3.jar,janino-2.7.6.jar,jansi-1.11.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,jasypt-1.9.2.jar,java-xmlbuilder-0.4.jar,javassist-3.18.2-GA.jar,javax.inject-1.jar,javax.mail-1.5.2.jar,javax.servlet-api-3.1.0.jar,jaxb-api-2.2.2.jar,jaxb-impl-2.2.3-1.jar,jcommander-1.48.jar,jdo-api-3.0.1.jar,jdo2-api-2.1.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.9.0.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-http-9.2.14.v20151106.jar,jetty-io-9.2.14.v20151106.jar,jetty-security-9.2.14.v20151106.jar,jetty-server-9.2.14.v20151106.jar,jetty-servlet-9.2.14.v20151106.jar,jetty-util-6.1.26.jar,jetty-util-9.2.14.v20151106.jar,jline-0.9.94.jar,joda-time-2.9.3.jar,jopt-simple-3.2.jar,jpam-1.1.jar,jsch-0.1.53.jar,json-20070829.jar,jsp-api-2.1.jar,jsr305-3.0.0.jar,jta-1.1.jar,junit-3.8.1.jar,kafka-clients-0.8.2.2.jar,kafka_2.11-0.8.2.2.jar,li-jersey-uri-1.15.9.jar,libfb303-0.9.0.jar,libthrift-0.9.3.jar,log4j-1.2.17.jar,lombok-1.16.8.jar,lz4-1.2.0.jar,mail-1.4.1.jar,maven-scm-api-1.4.jar,maven-scm-provider-svn-commons-1.4.jar,maven-scm-provider-svnexe-1.4.jar,metrics-core-2.2.0.jar,metrics-core-3.1.0.jar,metrics-graphite-3.1.0.jar,metrics-jvm-3.1.0.jar,mina-core-1.1.7.jar,mockito-core-1.10.19.jar,mysql-connector-java-5.1.38.jar,netty-3.2.3.Final.jar,netty-3.7.0.Final.jar,objenesis-2.1.jar,okhttp-2.4.0.jar,okio-1.4.0.jar,opencsv-2.3.jar,paranamer-2.3.jar,parseq-1.3.6.jar,pegasus-common-1.15.9.jar,pentaho-aggdesigner-algorithm-5.1.5-jhyde.jar,plexus-utils-1.5.6.jar,protobuf-java-2.5.0.jar,quartz-2.2.3.jar,r2-1.15.9.jar,reflections-0.9.10.jar,regexp-1.3.jar,restli-client-1.15.9.jar,restli-common-1.15.9.jar,restli-docgen-1.15.9.jar,restli-netty-standalone-1.15.9.jar,restli-server-1.15.9.jar,restli-tools-1.15.9.jar,retrofit-1.9.0.jar,scala-library-2.11.8.jar,scala-parser-combinators_2.11-1.0.2.jar,scala-xml_2.11-1.0.2.jar,servlet-api-2.5-20081211.jar,servlet-api-2.5.jar,slf4j-api-1.7.21.jar,slf4j-log4j12-1.7.21.jar,snappy-0.3.jar,snappy-java-1.1.1.7.jar,stax-api-1.0-2.jar,stax-api-1.0.1.jar,testng-6.9.10.jar,transaction-api-1.1.jar,velocity-1.7.jar,xmlenc-0.52.jar,zkclient-0.5.jar,zookeeper-3.4.6.jar" gobblin.aws.worker.s3.conf.uri=${gobblin.aws.master.s3.conf.uri} gobblin.aws.worker.s3.conf.files=${gobblin.aws.master.s3.conf.files} diff --git a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java index b3165ca05b8..f0e15bf94a4 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java @@ -438,6 +438,7 @@ public class ConfigurationKeys { public static final String WRITER_BYTES_WRITTEN = WRITER_PREFIX + ".bytes.written"; public static final String WRITER_EARLIEST_TIMESTAMP = WRITER_PREFIX + ".earliest.timestamp"; public static final String WRITER_AVERAGE_TIMESTAMP = WRITER_PREFIX + ".average.timestamp"; + public static final String WRITER_COUNT_METRICS_FROM_FAILED_TASKS = WRITER_PREFIX + ".jobTaskSummary.countMetricsFromFailedTasks"; /** * Configuration properties used by the quality checker. @@ -686,12 +687,18 @@ public class ConfigurationKeys { */ public static final String MR_JOB_ROOT_DIR_KEY = "mr.job.root.dir"; /** Specifies a static location in HDFS to upload jars to. Useful for sharing jars across different Gobblin runs.*/ + @Deprecated // Deprecated; use MR_JARS_BASE_DIR instead public static final String MR_JARS_DIR = "mr.jars.dir"; + // dir pointed by MR_JARS_BASE_DIR has month partitioned dirs to store jar files and are cleaned up on a regular basis + // retention feature is not available for dir pointed by MR_JARS_DIR + public static final String MR_JARS_BASE_DIR = "mr.jars.base.dir"; public static final String MR_JOB_MAX_MAPPERS_KEY = "mr.job.max.mappers"; + public static final String MR_JOB_MAPPER_FAILURE_IS_FATAL_KEY = "mr.job.map.failure.is.fatal"; public static final String MR_TARGET_MAPPER_SIZE = "mr.target.mapper.size"; public static final String MR_REPORT_METRICS_AS_COUNTERS_KEY = "mr.report.metrics.as.counters"; public static final boolean DEFAULT_MR_REPORT_METRICS_AS_COUNTERS = false; public static final int DEFAULT_MR_JOB_MAX_MAPPERS = 100; + public static final boolean DEFAULT_MR_JOB_MAPPER_FAILURE_IS_FATAL = false; public static final String DEFAULT_ENABLE_MR_SPECULATIVE_EXECUTION = "false"; /** diff --git a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java index d93eefbe24e..ef4323538d9 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java @@ -32,7 +32,7 @@ public class ServiceConfigKeys { public static final String GOBBLIN_SERVICE_TOPOLOGY_CATALOG_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "topologyCatalog.enabled"; public static final String GOBBLIN_SERVICE_FLOW_CATALOG_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "flowCatalog.enabled"; public static final String GOBBLIN_SERVICE_SCHEDULER_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "scheduler.enabled"; - + public static final String GOBBLIN_SERVICE_INSTANCE_NAME = GOBBLIN_SERVICE_PREFIX + "instance.name"; public static final String GOBBLIN_SERVICE_RESTLI_SERVER_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "restliServer.enabled"; public static final String GOBBLIN_SERVICE_TOPOLOGY_SPEC_FACTORY_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "topologySpecFactory.enabled"; diff --git a/gobblin-aws/src/main/java/org/apache/gobblin/aws/GobblinAWSConfigurationKeys.java b/gobblin-aws/src/main/java/org/apache/gobblin/aws/GobblinAWSConfigurationKeys.java index 13d1272c3eb..c44d15c4130 100644 --- a/gobblin-aws/src/main/java/org/apache/gobblin/aws/GobblinAWSConfigurationKeys.java +++ b/gobblin-aws/src/main/java/org/apache/gobblin/aws/GobblinAWSConfigurationKeys.java @@ -129,7 +129,7 @@ public class GobblinAWSConfigurationKeys { // of an occurrence of this bug in the JDK and how resolving it reduced the size of the JDK by 1 megabyte. public static final String DEFAULT_MASTER_S3_JARS_FILES; static { - DEFAULT_MASTER_S3_JARS_FILES = "ST4-4.0.4.jar,activation-1.1.1.jar,annotations-2.0.1.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.2.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,asm-3.1.jar,asm-commons-3.1.jar,asm-tree-3.1.jar,avro-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-ipc-1.7.7.jar,avro-mapred-1.7.7-hadoop2.jar,aws-java-sdk-applicationautoscaling-1.11.8.jar,aws-java-sdk-autoscaling-1.11.8.jar,aws-java-sdk-core-1.11.8.jar,aws-java-sdk-ec2-1.11.8.jar,aws-java-sdk-iam-1.11.8.jar,aws-java-sdk-kms-1.11.8.jar,aws-java-sdk-s3-1.11.8.jar,aws-java-sdk-sts-1.11.8.jar,azkaban-2.5.0.jar,bcpg-jdk15on-1.52.jar,bcprov-jdk15on-1.52.jar,bonecp-0.8.0.RELEASE.jar,bsh-2.0b4.jar,c3p0-0.9.1.1.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,cglib-nodep-2.2.jar,codemodel-2.2.jar,commons-cli-1.3.1.jar,commons-codec-1.10.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.10.jar,commons-configuration-1.10.jar,commons-daemon-1.0.13.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-email-1.4.jar,commons-httpclient-3.1.jar,commons-io-2.5.jar,commons-lang-2.6.jar,commons-lang3-3.4.jar,commons-logging-1.2.jar,commons-math3-3.5.jar,commons-net-3.1.jar,commons-pool-1.5.4.jar,commons-pool2-2.4.2.jar,commons-vfs2-2.0.jar,config-1.2.1.jar,curator-client-2.10.0.jar,curator-framework-2.10.0.jar,curator-recipes-2.10.0.jar,d2-1.15.9.jar,data-1.15.9.jar,data-transform-1.15.9.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,degrader-1.15.9.jar,derby-10.12.1.1.jar,eigenbase-properties-1.1.5.jar,flyway-core-3.2.1.jar,generator-1.15.9.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jpa_3.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,gobblin-admin-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-api-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-aws-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-azkaban-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-cluster-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-compaction-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-config-client-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-config-core-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-core-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-data-management-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-example-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-hive-registration-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-kafka-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-metastore-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-metrics-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-rest-api-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-rest-api-data-template-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-rest-api-rest-client-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-rest-client-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-rest-server-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-runtime-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-test-harness-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-tunnel-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-utility-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-yarn-" + DEFAULT_GOBBLIN_VERSION + ".jar,groovy-all-2.1.6.jar,gson-2.6.2.jar,guava-15.0.jar,guava-retrying-2.0.0.jar,guice-4.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.3.0.jar,hadoop-auth-2.3.0.jar,hadoop-common-2.3.0.jar,hadoop-hdfs-2.3.0.jar,hadoop-mapreduce-client-common-2.3.0.jar,hadoop-mapreduce-client-core-2.3.0.jar,hadoop-yarn-api-2.3.0.jar,hadoop-yarn-client-2.3.0.jar,hadoop-yarn-common-2.3.0.jar,hadoop-yarn-server-common-2.3.0.jar,hamcrest-core-1.1.jar,helix-core-0.6.6-SNAPSHOT.jar,hive-ant-1.0.1.jar,hive-common-1.0.1.jar,hive-exec-1.0.1-core.jar,hive-jdbc-1.0.1.jar,hive-metastore-1.0.1.jar,hive-serde-1.0.1.jar,hive-service-1.0.1.jar,hive-shims-0.20-1.0.1.jar,hive-shims-0.20S-1.0.1.jar,hive-shims-0.23-1.0.1.jar,hive-shims-1.0.1.jar,hive-shims-common-1.0.1.jar,hive-shims-common-secure-1.0.1.jar,httpclient-4.5.2.jar,httpcore-4.4.4.jar,influxdb-java-2.1.jar,jackson-annotations-2.6.0.jar,jackson-core-2.6.6.jar,jackson-core-asl-1.9.13.jar,jackson-databind-2.6.6.jar,jackson-dataformat-cbor-2.6.6.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.13.jar,jackson-xc-1.8.3.jar,janino-2.7.6.jar,jansi-1.11.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,jasypt-1.9.2.jar,java-xmlbuilder-0.4.jar,javassist-3.18.2-GA.jar,javax.inject-1.jar,javax.mail-1.5.2.jar,javax.servlet-api-3.1.0.jar,jaxb-api-2.2.2.jar,jaxb-impl-2.2.3-1.jar,jcommander-1.48.jar,jdo-api-3.0.1.jar,jdo2-api-2.1.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.9.0.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-http-9.2.14.v20151106.jar,jetty-io-9.2.14.v20151106.jar,jetty-security-9.2.14.v20151106.jar,jetty-server-9.2.14.v20151106.jar,jetty-servlet-9.2.14.v20151106.jar,jetty-util-6.1.26.jar,jetty-util-9.2.14.v20151106.jar,jline-0.9.94.jar,joda-time-2.9.3.jar,jopt-simple-3.2.jar,jpam-1.1.jar,jsch-0.1.53.jar,json-20070829.jar,jsp-api-2.1.jar,jsr305-3.0.0.jar,jta-1.1.jar,junit-3.8.1.jar,kafka-clients-0.8.2.2.jar,kafka_2.11-0.8.2.2.jar,li-jersey-uri-1.15.9.jar,libfb303-0.9.0.jar,libthrift-0.9.3.jar,log4j-1.2.17.jar,lombok-1.16.8.jar,lz4-1.2.0.jar,mail-1.4.1.jar,maven-scm-api-1.4.jar,maven-scm-provider-svn-commons-1.4.jar,maven-scm-provider-svnexe-1.4.jar,metrics-core-2.2.0.jar,metrics-core-3.1.0.jar,metrics-graphite-3.1.0.jar,metrics-jvm-3.1.0.jar,mina-core-1.1.7.jar,mockito-core-1.10.19.jar,mysql-connector-java-5.1.38.jar,netty-3.2.3.Final.jar,netty-3.7.0.Final.jar,objenesis-2.1.jar,okhttp-2.4.0.jar,okio-1.4.0.jar,opencsv-2.3.jar,paranamer-2.3.jar,parseq-1.3.6.jar,pegasus-common-1.15.9.jar,pentaho-aggdesigner-algorithm-5.1.5-jhyde.jar,plexus-utils-1.5.6.jar,protobuf-java-2.5.0.jar,quartz-2.2.3.jar,r2-1.15.9.jar,reflections-0.9.10.jar,regexp-1.3.jar,restli-client-1.15.9.jar,restli-common-1.15.9.jar,restli-docgen-1.15.9.jar,restli-netty-standalone-1.15.9.jar,restli-server-1.15.9.jar,restli-tools-1.15.9.jar,retrofit-1.9.0.jar,scala-library-2.11.8.jar,scala-parser-combinators_2.11-1.0.2.jar,scala-xml_2.11-1.0.2.jar,servlet-api-2.5-20081211.jar,servlet-api-2.5.jar,slf4j-api-1.7.21.jar,slf4j-log4j12-1.7.21.jar,snappy-0.3.jar,snappy-java-1.1.1.7.jar,stax-api-1.0-2.jar,stax-api-1.0.1.jar,testng-6.9.10.jar,transaction-api-1.1.jar,velocity-1.7.jar,xmlenc-0.52.jar,zkclient-0.5.jar,zookeeper-3.4.6.jar"; + DEFAULT_MASTER_S3_JARS_FILES = "ST4-4.0.4.jar,activation-1.1.1.jar,annotations-2.0.1.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.2.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,asm-3.1.jar,asm-commons-3.1.jar,asm-tree-3.1.jar,avro-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-ipc-1.7.7.jar,avro-mapred-1.7.7-hadoop2.jar,aws-java-sdk-applicationautoscaling-1.11.8.jar,aws-java-sdk-autoscaling-1.11.8.jar,aws-java-sdk-core-1.11.8.jar,aws-java-sdk-ec2-1.11.8.jar,aws-java-sdk-iam-1.11.8.jar,aws-java-sdk-kms-1.11.8.jar,aws-java-sdk-s3-1.11.8.jar,aws-java-sdk-sts-1.11.8.jar,azkaban-2.5.0.jar,bcpg-jdk15on-1.52.jar,bcprov-jdk15on-1.52.jar,bonecp-0.8.0.RELEASE.jar,bsh-2.0b4.jar,c3p0-0.9.1.1.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,cglib-nodep-2.2.jar,codemodel-2.2.jar,commons-cli-1.3.1.jar,commons-codec-1.10.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.10.jar,commons-configuration-1.10.jar,commons-daemon-1.0.13.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-email-1.4.jar,commons-httpclient-3.1.jar,commons-io-2.5.jar,commons-lang-2.6.jar,commons-lang3-3.4.jar,commons-logging-1.2.jar,commons-math3-3.5.jar,commons-net-3.1.jar,commons-pool-1.5.4.jar,commons-pool2-2.4.2.jar,commons-vfs2-2.0.jar,config-1.2.1.jar,curator-client-2.10.0.jar,curator-framework-2.10.0.jar,curator-recipes-2.10.0.jar,d2-1.15.9.jar,data-1.15.9.jar,data-transform-1.15.9.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,degrader-1.15.9.jar,derby-10.12.1.1.jar,eigenbase-properties-1.1.5.jar,flyway-core-3.2.1.jar,generator-1.15.9.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jpa_3.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,gobblin-admin-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-api-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-aws-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-azkaban-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-cluster-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-compaction-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-config-client-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-config-core-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-core-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-data-management-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-example-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-hive-registration-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-kafka-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-metastore-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-metrics-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-rest-api-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-rest-api-data-template-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-rest-api-rest-client-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-rest-client-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-rest-server-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-runtime-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-test-harness-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-tunnel-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-utility-" + DEFAULT_GOBBLIN_VERSION + ".jar,gobblin-yarn-" + DEFAULT_GOBBLIN_VERSION + ".jar,groovy-all-2.1.6.jar,gson-2.6.2.jar,guava-20.0.jar,guava-retrying-2.0.0.jar,guice-4.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.3.0.jar,hadoop-auth-2.3.0.jar,hadoop-common-2.3.0.jar,hadoop-hdfs-2.3.0.jar,hadoop-mapreduce-client-common-2.3.0.jar,hadoop-mapreduce-client-core-2.3.0.jar,hadoop-yarn-api-2.3.0.jar,hadoop-yarn-client-2.3.0.jar,hadoop-yarn-common-2.3.0.jar,hadoop-yarn-server-common-2.3.0.jar,hamcrest-core-1.1.jar,helix-core-0.6.6-SNAPSHOT.jar,hive-ant-1.0.1.jar,hive-common-1.0.1.jar,hive-exec-1.0.1-core.jar,hive-jdbc-1.0.1.jar,hive-metastore-1.0.1.jar,hive-serde-1.0.1.jar,hive-service-1.0.1.jar,hive-shims-0.20-1.0.1.jar,hive-shims-0.20S-1.0.1.jar,hive-shims-0.23-1.0.1.jar,hive-shims-1.0.1.jar,hive-shims-common-1.0.1.jar,hive-shims-common-secure-1.0.1.jar,httpclient-4.5.2.jar,httpcore-4.4.4.jar,influxdb-java-2.1.jar,jackson-annotations-2.6.0.jar,jackson-core-2.6.6.jar,jackson-core-asl-1.9.13.jar,jackson-databind-2.6.6.jar,jackson-dataformat-cbor-2.6.6.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.13.jar,jackson-xc-1.8.3.jar,janino-2.7.6.jar,jansi-1.11.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,jasypt-1.9.2.jar,java-xmlbuilder-0.4.jar,javassist-3.18.2-GA.jar,javax.inject-1.jar,javax.mail-1.5.2.jar,javax.servlet-api-3.1.0.jar,jaxb-api-2.2.2.jar,jaxb-impl-2.2.3-1.jar,jcommander-1.48.jar,jdo-api-3.0.1.jar,jdo2-api-2.1.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.9.0.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-http-9.2.14.v20151106.jar,jetty-io-9.2.14.v20151106.jar,jetty-security-9.2.14.v20151106.jar,jetty-server-9.2.14.v20151106.jar,jetty-servlet-9.2.14.v20151106.jar,jetty-util-6.1.26.jar,jetty-util-9.2.14.v20151106.jar,jline-0.9.94.jar,joda-time-2.9.3.jar,jopt-simple-3.2.jar,jpam-1.1.jar,jsch-0.1.53.jar,json-20070829.jar,jsp-api-2.1.jar,jsr305-3.0.0.jar,jta-1.1.jar,junit-3.8.1.jar,kafka-clients-0.8.2.2.jar,kafka_2.11-0.8.2.2.jar,li-jersey-uri-1.15.9.jar,libfb303-0.9.0.jar,libthrift-0.9.3.jar,log4j-1.2.17.jar,lombok-1.16.8.jar,lz4-1.2.0.jar,mail-1.4.1.jar,maven-scm-api-1.4.jar,maven-scm-provider-svn-commons-1.4.jar,maven-scm-provider-svnexe-1.4.jar,metrics-core-2.2.0.jar,metrics-core-3.1.0.jar,metrics-graphite-3.1.0.jar,metrics-jvm-3.1.0.jar,mina-core-1.1.7.jar,mockito-core-1.10.19.jar,mysql-connector-java-5.1.38.jar,netty-3.2.3.Final.jar,netty-3.7.0.Final.jar,objenesis-2.1.jar,okhttp-2.4.0.jar,okio-1.4.0.jar,opencsv-2.3.jar,paranamer-2.3.jar,parseq-1.3.6.jar,pegasus-common-1.15.9.jar,pentaho-aggdesigner-algorithm-5.1.5-jhyde.jar,plexus-utils-1.5.6.jar,protobuf-java-2.5.0.jar,quartz-2.2.3.jar,r2-1.15.9.jar,reflections-0.9.10.jar,regexp-1.3.jar,restli-client-1.15.9.jar,restli-common-1.15.9.jar,restli-docgen-1.15.9.jar,restli-netty-standalone-1.15.9.jar,restli-server-1.15.9.jar,restli-tools-1.15.9.jar,retrofit-1.9.0.jar,scala-library-2.11.8.jar,scala-parser-combinators_2.11-1.0.2.jar,scala-xml_2.11-1.0.2.jar,servlet-api-2.5-20081211.jar,servlet-api-2.5.jar,slf4j-api-1.7.21.jar,slf4j-log4j12-1.7.21.jar,snappy-0.3.jar,snappy-java-1.1.1.7.jar,stax-api-1.0-2.jar,stax-api-1.0.1.jar,testng-6.9.10.jar,transaction-api-1.1.jar,velocity-1.7.jar,xmlenc-0.52.jar,zkclient-0.5.jar,zookeeper-3.4.6.jar"; } // Gobblin AWS worker configuration properties. diff --git a/gobblin-aws/src/test/resources/GobblinAWSClusterLauncherTest.conf b/gobblin-aws/src/test/resources/GobblinAWSClusterLauncherTest.conf index 8a4904a4a33..444109b3104 100644 --- a/gobblin-aws/src/test/resources/GobblinAWSClusterLauncherTest.conf +++ b/gobblin-aws/src/test/resources/GobblinAWSClusterLauncherTest.conf @@ -152,7 +152,7 @@ gobblin.aws.master.s3.jars.uri="https://s3-us-west-2.amazonaws.com/gobblin-libs/ ## All Gobblin Jars and Configuration on S3 gobblin.aws.master.s3.conf.files="application.conf,log4j-aws.properties,quartz.properties" gobblin.aws.master.s3.jars.uri="https://s3-us-west-2.amazonaws.com/gobblin-libs/latest-jars/" -gobblin.aws.master.s3.jars.files="ST4-4.0.4.jar,activation-1.1.1.jar,annotations-2.0.1.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.2.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,asm-3.1.jar,asm-commons-3.1.jar,asm-tree-3.1.jar,avro-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-ipc-1.7.7.jar,avro-mapred-1.7.7-hadoop2.jar,aws-java-sdk-applicationautoscaling-1.11.8.jar,aws-java-sdk-autoscaling-1.11.8.jar,aws-java-sdk-core-1.11.8.jar,aws-java-sdk-ec2-1.11.8.jar,aws-java-sdk-iam-1.11.8.jar,aws-java-sdk-kms-1.11.8.jar,aws-java-sdk-s3-1.11.8.jar,aws-java-sdk-sts-1.11.8.jar,azkaban-2.5.0.jar,bcpg-jdk15on-1.52.jar,bcprov-jdk15on-1.52.jar,bonecp-0.8.0.RELEASE.jar,bsh-2.0b4.jar,c3p0-0.9.1.1.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,cglib-nodep-2.2.jar,codemodel-2.2.jar,commons-cli-1.3.1.jar,commons-codec-1.10.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.10.jar,commons-configuration-1.10.jar,commons-daemon-1.0.13.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-email-1.4.jar,commons-httpclient-3.1.jar,commons-io-2.5.jar,commons-lang-2.6.jar,commons-lang3-3.4.jar,commons-logging-1.2.jar,commons-math3-3.5.jar,commons-net-3.1.jar,commons-pool-1.5.4.jar,commons-pool2-2.4.2.jar,commons-vfs2-2.0.jar,config-1.2.1.jar,curator-client-2.10.0.jar,curator-framework-2.10.0.jar,curator-recipes-2.10.0.jar,d2-1.15.9.jar,data-1.15.9.jar,data-transform-1.15.9.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,degrader-1.15.9.jar,derby-10.12.1.1.jar,eigenbase-properties-1.1.5.jar,flyway-core-3.2.1.jar,generator-1.15.9.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jpa_3.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,gobblin-admin-"${gobblin.aws.version}".jar,gobblin-api-"${gobblin.aws.version}".jar,gobblin-aws-"${gobblin.aws.version}".jar,gobblin-azkaban-"${gobblin.aws.version}".jar,gobblin-cluster-"${gobblin.aws.version}".jar,gobblin-compaction-"${gobblin.aws.version}".jar,gobblin-config-client-"${gobblin.aws.version}".jar,gobblin-config-core-"${gobblin.aws.version}".jar,gobblin-core-"${gobblin.aws.version}".jar,gobblin-data-management-"${gobblin.aws.version}".jar,gobblin-example-"${gobblin.aws.version}".jar,gobblin-hive-registration-"${gobblin.aws.version}".jar,gobblin-kafka-"${gobblin.aws.version}".jar,gobblin-metastore-"${gobblin.aws.version}".jar,gobblin-metrics-"${gobblin.aws.version}".jar,gobblin-rest-api-"${gobblin.aws.version}".jar,gobblin-rest-api-data-template-"${gobblin.aws.version}".jar,gobblin-rest-api-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-server-"${gobblin.aws.version}".jar,gobblin-runtime-"${gobblin.aws.version}".jar,gobblin-test-harness-"${gobblin.aws.version}".jar,gobblin-tunnel-"${gobblin.aws.version}".jar,gobblin-utility-"${gobblin.aws.version}".jar,gobblin-yarn-"${gobblin.aws.version}".jar,groovy-all-2.1.6.jar,gson-2.6.2.jar,guava-15.0.jar,guava-retrying-2.0.0.jar,guice-4.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.3.0.jar,hadoop-auth-2.3.0.jar,hadoop-common-2.3.0.jar,hadoop-hdfs-2.3.0.jar,hadoop-mapreduce-client-common-2.3.0.jar,hadoop-mapreduce-client-core-2.3.0.jar,hadoop-yarn-api-2.3.0.jar,hadoop-yarn-client-2.3.0.jar,hadoop-yarn-common-2.3.0.jar,hadoop-yarn-server-common-2.3.0.jar,hamcrest-core-1.1.jar,helix-core-0.6.6-SNAPSHOT.jar,hive-ant-1.0.1.jar,hive-common-1.0.1.jar,hive-exec-1.0.1-core.jar,hive-jdbc-1.0.1.jar,hive-metastore-1.0.1.jar,hive-serde-1.0.1.jar,hive-service-1.0.1.jar,hive-shims-0.20-1.0.1.jar,hive-shims-0.20S-1.0.1.jar,hive-shims-0.23-1.0.1.jar,hive-shims-1.0.1.jar,hive-shims-common-1.0.1.jar,hive-shims-common-secure-1.0.1.jar,httpclient-4.5.2.jar,httpcore-4.4.4.jar,influxdb-java-2.1.jar,jackson-annotations-2.6.0.jar,jackson-core-2.6.6.jar,jackson-core-asl-1.9.13.jar,jackson-databind-2.6.6.jar,jackson-dataformat-cbor-2.6.6.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.13.jar,jackson-xc-1.8.3.jar,janino-2.7.6.jar,jansi-1.11.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,jasypt-1.9.2.jar,java-xmlbuilder-0.4.jar,javassist-3.18.2-GA.jar,javax.inject-1.jar,javax.mail-1.5.2.jar,javax.servlet-api-3.1.0.jar,jaxb-api-2.2.2.jar,jaxb-impl-2.2.3-1.jar,jcommander-1.48.jar,jdo-api-3.0.1.jar,jdo2-api-2.1.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.9.0.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-http-9.2.14.v20151106.jar,jetty-io-9.2.14.v20151106.jar,jetty-security-9.2.14.v20151106.jar,jetty-server-9.2.14.v20151106.jar,jetty-servlet-9.2.14.v20151106.jar,jetty-util-6.1.26.jar,jetty-util-9.2.14.v20151106.jar,jline-0.9.94.jar,joda-time-2.9.3.jar,jopt-simple-3.2.jar,jpam-1.1.jar,jsch-0.1.53.jar,json-20070829.jar,jsp-api-2.1.jar,jsr305-3.0.0.jar,jta-1.1.jar,junit-3.8.1.jar,kafka-clients-0.8.2.2.jar,kafka_2.11-0.8.2.2.jar,li-jersey-uri-1.15.9.jar,libfb303-0.9.0.jar,libthrift-0.9.3.jar,log4j-1.2.17.jar,lombok-1.16.8.jar,lz4-1.2.0.jar,mail-1.4.1.jar,maven-scm-api-1.4.jar,maven-scm-provider-svn-commons-1.4.jar,maven-scm-provider-svnexe-1.4.jar,metrics-core-2.2.0.jar,metrics-core-3.1.0.jar,metrics-graphite-3.1.0.jar,metrics-jvm-3.1.0.jar,mina-core-1.1.7.jar,mockito-core-1.10.19.jar,mysql-connector-java-5.1.38.jar,netty-3.2.3.Final.jar,netty-3.7.0.Final.jar,objenesis-2.1.jar,okhttp-2.4.0.jar,okio-1.4.0.jar,opencsv-2.3.jar,paranamer-2.3.jar,parseq-1.3.6.jar,pegasus-common-1.15.9.jar,pentaho-aggdesigner-algorithm-5.1.5-jhyde.jar,plexus-utils-1.5.6.jar,protobuf-java-2.5.0.jar,quartz-2.2.3.jar,r2-1.15.9.jar,reflections-0.9.10.jar,regexp-1.3.jar,restli-client-1.15.9.jar,restli-common-1.15.9.jar,restli-docgen-1.15.9.jar,restli-netty-standalone-1.15.9.jar,restli-server-1.15.9.jar,restli-tools-1.15.9.jar,retrofit-1.9.0.jar,scala-library-2.11.8.jar,scala-parser-combinators_2.11-1.0.2.jar,scala-xml_2.11-1.0.2.jar,servlet-api-2.5-20081211.jar,servlet-api-2.5.jar,slf4j-api-1.7.21.jar,slf4j-log4j12-1.7.21.jar,snappy-0.3.jar,snappy-java-1.1.1.7.jar,stax-api-1.0-2.jar,stax-api-1.0.1.jar,testng-6.9.10.jar,transaction-api-1.1.jar,velocity-1.7.jar,xmlenc-0.52.jar,zkclient-0.3.jar,zookeeper-3.4.6.jar" +gobblin.aws.master.s3.jars.files="ST4-4.0.4.jar,activation-1.1.1.jar,annotations-2.0.1.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.2.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,asm-3.1.jar,asm-commons-3.1.jar,asm-tree-3.1.jar,avro-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-ipc-1.7.7.jar,avro-mapred-1.7.7-hadoop2.jar,aws-java-sdk-applicationautoscaling-1.11.8.jar,aws-java-sdk-autoscaling-1.11.8.jar,aws-java-sdk-core-1.11.8.jar,aws-java-sdk-ec2-1.11.8.jar,aws-java-sdk-iam-1.11.8.jar,aws-java-sdk-kms-1.11.8.jar,aws-java-sdk-s3-1.11.8.jar,aws-java-sdk-sts-1.11.8.jar,azkaban-2.5.0.jar,bcpg-jdk15on-1.52.jar,bcprov-jdk15on-1.52.jar,bonecp-0.8.0.RELEASE.jar,bsh-2.0b4.jar,c3p0-0.9.1.1.jar,calcite-avatica-1.2.0-incubating.jar,calcite-core-1.2.0-incubating.jar,calcite-linq4j-1.2.0-incubating.jar,cglib-nodep-2.2.jar,codemodel-2.2.jar,commons-cli-1.3.1.jar,commons-codec-1.10.jar,commons-collections-3.2.1.jar,commons-compiler-2.7.6.jar,commons-compress-1.10.jar,commons-configuration-1.10.jar,commons-daemon-1.0.13.jar,commons-dbcp-1.4.jar,commons-el-1.0.jar,commons-email-1.4.jar,commons-httpclient-3.1.jar,commons-io-2.5.jar,commons-lang-2.6.jar,commons-lang3-3.4.jar,commons-logging-1.2.jar,commons-math3-3.5.jar,commons-net-3.1.jar,commons-pool-1.5.4.jar,commons-pool2-2.4.2.jar,commons-vfs2-2.0.jar,config-1.2.1.jar,curator-client-2.10.0.jar,curator-framework-2.10.0.jar,curator-recipes-2.10.0.jar,d2-1.15.9.jar,data-1.15.9.jar,data-transform-1.15.9.jar,datanucleus-api-jdo-3.2.6.jar,datanucleus-core-3.2.10.jar,datanucleus-rdbms-3.2.9.jar,degrader-1.15.9.jar,derby-10.12.1.1.jar,eigenbase-properties-1.1.5.jar,flyway-core-3.2.1.jar,generator-1.15.9.jar,geronimo-annotation_1.0_spec-1.1.1.jar,geronimo-jaspic_1.0_spec-1.0.jar,geronimo-jpa_3.0_spec-1.0.jar,geronimo-jta_1.1_spec-1.1.1.jar,gobblin-admin-"${gobblin.aws.version}".jar,gobblin-api-"${gobblin.aws.version}".jar,gobblin-aws-"${gobblin.aws.version}".jar,gobblin-azkaban-"${gobblin.aws.version}".jar,gobblin-cluster-"${gobblin.aws.version}".jar,gobblin-compaction-"${gobblin.aws.version}".jar,gobblin-config-client-"${gobblin.aws.version}".jar,gobblin-config-core-"${gobblin.aws.version}".jar,gobblin-core-"${gobblin.aws.version}".jar,gobblin-data-management-"${gobblin.aws.version}".jar,gobblin-example-"${gobblin.aws.version}".jar,gobblin-hive-registration-"${gobblin.aws.version}".jar,gobblin-kafka-"${gobblin.aws.version}".jar,gobblin-metastore-"${gobblin.aws.version}".jar,gobblin-metrics-"${gobblin.aws.version}".jar,gobblin-rest-api-"${gobblin.aws.version}".jar,gobblin-rest-api-data-template-"${gobblin.aws.version}".jar,gobblin-rest-api-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-client-"${gobblin.aws.version}".jar,gobblin-rest-server-"${gobblin.aws.version}".jar,gobblin-runtime-"${gobblin.aws.version}".jar,gobblin-test-harness-"${gobblin.aws.version}".jar,gobblin-tunnel-"${gobblin.aws.version}".jar,gobblin-utility-"${gobblin.aws.version}".jar,gobblin-yarn-"${gobblin.aws.version}".jar,groovy-all-2.1.6.jar,gson-2.6.2.jar,guava-20.0.jar,guava-retrying-2.0.0.jar,guice-4.0.jar,guice-servlet-3.0.jar,hadoop-annotations-2.3.0.jar,hadoop-auth-2.3.0.jar,hadoop-common-2.3.0.jar,hadoop-hdfs-2.3.0.jar,hadoop-mapreduce-client-common-2.3.0.jar,hadoop-mapreduce-client-core-2.3.0.jar,hadoop-yarn-api-2.3.0.jar,hadoop-yarn-client-2.3.0.jar,hadoop-yarn-common-2.3.0.jar,hadoop-yarn-server-common-2.3.0.jar,hamcrest-core-1.1.jar,helix-core-0.6.6-SNAPSHOT.jar,hive-ant-1.0.1.jar,hive-common-1.0.1.jar,hive-exec-1.0.1-core.jar,hive-jdbc-1.0.1.jar,hive-metastore-1.0.1.jar,hive-serde-1.0.1.jar,hive-service-1.0.1.jar,hive-shims-0.20-1.0.1.jar,hive-shims-0.20S-1.0.1.jar,hive-shims-0.23-1.0.1.jar,hive-shims-1.0.1.jar,hive-shims-common-1.0.1.jar,hive-shims-common-secure-1.0.1.jar,httpclient-4.5.2.jar,httpcore-4.4.4.jar,influxdb-java-2.1.jar,jackson-annotations-2.6.0.jar,jackson-core-2.6.6.jar,jackson-core-asl-1.9.13.jar,jackson-databind-2.6.6.jar,jackson-dataformat-cbor-2.6.6.jar,jackson-jaxrs-1.8.3.jar,jackson-mapper-asl-1.9.13.jar,jackson-xc-1.8.3.jar,janino-2.7.6.jar,jansi-1.11.jar,jasper-compiler-5.5.23.jar,jasper-runtime-5.5.23.jar,jasypt-1.9.2.jar,java-xmlbuilder-0.4.jar,javassist-3.18.2-GA.jar,javax.inject-1.jar,javax.mail-1.5.2.jar,javax.servlet-api-3.1.0.jar,jaxb-api-2.2.2.jar,jaxb-impl-2.2.3-1.jar,jcommander-1.48.jar,jdo-api-3.0.1.jar,jdo2-api-2.1.jar,jersey-core-1.9.jar,jersey-guice-1.9.jar,jersey-json-1.9.jar,jersey-server-1.9.jar,jets3t-0.9.0.jar,jettison-1.1.jar,jetty-6.1.26.jar,jetty-all-7.6.0.v20120127.jar,jetty-http-9.2.14.v20151106.jar,jetty-io-9.2.14.v20151106.jar,jetty-security-9.2.14.v20151106.jar,jetty-server-9.2.14.v20151106.jar,jetty-servlet-9.2.14.v20151106.jar,jetty-util-6.1.26.jar,jetty-util-9.2.14.v20151106.jar,jline-0.9.94.jar,joda-time-2.9.3.jar,jopt-simple-3.2.jar,jpam-1.1.jar,jsch-0.1.53.jar,json-20070829.jar,jsp-api-2.1.jar,jsr305-3.0.0.jar,jta-1.1.jar,junit-3.8.1.jar,kafka-clients-0.8.2.2.jar,kafka_2.11-0.8.2.2.jar,li-jersey-uri-1.15.9.jar,libfb303-0.9.0.jar,libthrift-0.9.3.jar,log4j-1.2.17.jar,lombok-1.16.8.jar,lz4-1.2.0.jar,mail-1.4.1.jar,maven-scm-api-1.4.jar,maven-scm-provider-svn-commons-1.4.jar,maven-scm-provider-svnexe-1.4.jar,metrics-core-2.2.0.jar,metrics-core-3.1.0.jar,metrics-graphite-3.1.0.jar,metrics-jvm-3.1.0.jar,mina-core-1.1.7.jar,mockito-core-1.10.19.jar,mysql-connector-java-5.1.38.jar,netty-3.2.3.Final.jar,netty-3.7.0.Final.jar,objenesis-2.1.jar,okhttp-2.4.0.jar,okio-1.4.0.jar,opencsv-2.3.jar,paranamer-2.3.jar,parseq-1.3.6.jar,pegasus-common-1.15.9.jar,pentaho-aggdesigner-algorithm-5.1.5-jhyde.jar,plexus-utils-1.5.6.jar,protobuf-java-2.5.0.jar,quartz-2.2.3.jar,r2-1.15.9.jar,reflections-0.9.10.jar,regexp-1.3.jar,restli-client-1.15.9.jar,restli-common-1.15.9.jar,restli-docgen-1.15.9.jar,restli-netty-standalone-1.15.9.jar,restli-server-1.15.9.jar,restli-tools-1.15.9.jar,retrofit-1.9.0.jar,scala-library-2.11.8.jar,scala-parser-combinators_2.11-1.0.2.jar,scala-xml_2.11-1.0.2.jar,servlet-api-2.5-20081211.jar,servlet-api-2.5.jar,slf4j-api-1.7.21.jar,slf4j-log4j12-1.7.21.jar,snappy-0.3.jar,snappy-java-1.1.1.7.jar,stax-api-1.0-2.jar,stax-api-1.0.1.jar,testng-6.9.10.jar,transaction-api-1.1.jar,velocity-1.7.jar,xmlenc-0.52.jar,zkclient-0.3.jar,zookeeper-3.4.6.jar" gobblin.aws.worker.s3.conf.uri=${gobblin.aws.master.s3.conf.uri} gobblin.aws.worker.s3.conf.files=${gobblin.aws.master.s3.conf.files} diff --git a/gobblin-cluster/build.gradle b/gobblin-cluster/build.gradle index 3493c2b153f..4b68445c125 100644 --- a/gobblin-cluster/build.gradle +++ b/gobblin-cluster/build.gradle @@ -60,6 +60,7 @@ dependencies { testCompile externalDependency.curatorFramework testCompile externalDependency.curatorTest testCompile externalDependency.assertj + testCompile externalDependency.mockito } task testJar(type: Jar, dependsOn: testClasses) { diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java index d2a60781d45..31a8547aa98 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java @@ -158,6 +158,9 @@ public class GobblinClusterConfigurationKeys { public static final String HELIX_TASK_TIMEOUT_SECONDS = "helix.task.timeout.seconds"; public static final String HELIX_TASK_MAX_ATTEMPTS_KEY = "helix.task.maxAttempts"; + public static final String HELIX_WORKFLOW_SUBMISSION_TIMEOUT_SECONDS = GOBBLIN_CLUSTER_PREFIX + "workflowSubmissionTimeoutSeconds"; + public static final long DEFAULT_HELIX_WORKFLOW_SUBMISSION_TIMEOUT_SECONDS = 300; + public static final String HELIX_WORKFLOW_DELETE_TIMEOUT_SECONDS = GOBBLIN_CLUSTER_PREFIX + "workflowDeleteTimeoutSeconds"; public static final long DEFAULT_HELIX_WORKFLOW_DELETE_TIMEOUT_SECONDS = 300; @@ -219,4 +222,4 @@ public class GobblinClusterConfigurationKeys { public static final String CONTAINER_ID_KEY = GOBBLIN_HELIX_PREFIX + "containerId"; public static final String GOBBLIN_CLUSTER_SYSTEM_PROPERTY_PREFIX = GOBBLIN_CLUSTER_PREFIX + "sysProps"; -} \ No newline at end of file +} diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixDistributeJobExecutionLauncher.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixDistributeJobExecutionLauncher.java index f2ba083c561..6faaebd6300 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixDistributeJobExecutionLauncher.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixDistributeJobExecutionLauncher.java @@ -19,6 +19,7 @@ import java.io.Closeable; import java.io.IOException; +import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Optional; @@ -96,6 +97,8 @@ class GobblinHelixDistributeJobExecutionLauncher implements JobExecutionLauncher private final long helixJobStopTimeoutSeconds; + private final long helixWorkflowSubmissionTimeoutSeconds; + private boolean jobSubmitted; // A conditional variable for which the condition is satisfied if a cancellation is requested @@ -135,6 +138,9 @@ public GobblinHelixDistributeJobExecutionLauncher(Builder builder) { this.helixJobStopTimeoutSeconds = ConfigUtils.getLong(this.combinedConfigs, GobblinClusterConfigurationKeys.HELIX_JOB_STOP_TIMEOUT_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_HELIX_JOB_STOP_TIMEOUT_SECONDS); + this.helixWorkflowSubmissionTimeoutSeconds = ConfigUtils.getLong(this.combinedConfigs, + GobblinClusterConfigurationKeys.HELIX_WORKFLOW_SUBMISSION_TIMEOUT_SECONDS, + GobblinClusterConfigurationKeys.DEFAULT_HELIX_WORKFLOW_SUBMISSION_TIMEOUT_SECONDS); } @Override @@ -252,7 +258,8 @@ private void submitJobToHelix(String jobName, String jobId, JobConfig.Builder jo jobId, taskDriver, this.planningJobHelixManager, - this.workFlowExpiryTimeSeconds); + Duration.ofSeconds(this.workFlowExpiryTimeSeconds), + Duration.ofSeconds(this.helixWorkflowSubmissionTimeoutSeconds)); this.jobSubmitted = true; } diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java index 6308ddc2d33..c0578a99703 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.net.URI; +import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,6 +45,7 @@ import com.github.rholder.retry.Retryer; import com.github.rholder.retry.RetryerBuilder; import com.github.rholder.retry.StopStrategies; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import com.typesafe.config.Config; import com.typesafe.config.ConfigValueFactory; @@ -131,6 +133,7 @@ public class GobblinHelixJobLauncher extends AbstractJobLauncher { private final Config jobConfig; private final long workFlowExpiryTimeSeconds; private final long helixJobStopTimeoutSeconds; + private final long helixWorkflowSubmissionTimeoutSeconds; private Map workUnitToHelixConfig; private Retryer taskRetryer; @@ -163,6 +166,10 @@ public GobblinHelixJobLauncher(Properties jobProps, final HelixManager helixMana ConfigUtils.getLong(jobConfig, GobblinClusterConfigurationKeys.HELIX_JOB_STOP_TIMEOUT_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_HELIX_JOB_STOP_TIMEOUT_SECONDS); + this.helixWorkflowSubmissionTimeoutSeconds = ConfigUtils.getLong(jobConfig, + GobblinClusterConfigurationKeys.HELIX_WORKFLOW_SUBMISSION_TIMEOUT_SECONDS, + GobblinClusterConfigurationKeys.DEFAULT_HELIX_WORKFLOW_SUBMISSION_TIMEOUT_SECONDS); + Config stateStoreJobConfig = ConfigUtils.propertiesToConfig(jobProps) .withValue(ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigValueFactory.fromAnyRef( new URI(appWorkDir.toUri().getScheme(), null, appWorkDir.toUri().getHost(), appWorkDir.toUri().getPort(), @@ -434,7 +441,9 @@ JobConfig.Builder translateGobblinJobConfigToHelixJobConfig(JobState gobblinJobS */ private void submitJobToHelix(JobConfig.Builder jobConfigBuilder) throws Exception { HelixUtils.submitJobToWorkFlow(jobConfigBuilder, this.helixWorkFlowName, this.jobContext.getJobId(), - this.helixTaskDriver, this.helixManager, this.workFlowExpiryTimeSeconds); + this.helixTaskDriver, this.helixManager, + Duration.ofSeconds(this.workFlowExpiryTimeSeconds), + Duration.ofSeconds(this.helixWorkflowSubmissionTimeoutSeconds)); } public void launchJob(@Nullable JobListener jobListener) throws JobException { @@ -447,11 +456,13 @@ public void launchJob(@Nullable JobListener jobListener) throws JobException { if (this.runningMap.replace(this.jobContext.getJobName(), false, true)) { LOGGER.info("Job {} will be executed, add into running map.", this.jobContext.getJobId()); isLaunched = true; - super.launchJob(jobListener); + launchJobImpl(jobListener); } else { LOGGER.warn("Job {} will not be executed because other jobs are still running.", this.jobContext.getJobId()); } - // TODO: Better error handling + + // TODO: Better error handling. The current impl swallows exceptions for jobs that were started by this method call. + // One potential way to improve the error handling is to make this error swallowing conifgurable } catch (Throwable t) { errorInJobLaunching = t; } finally { @@ -467,6 +478,29 @@ public void launchJob(@Nullable JobListener jobListener) throws JobException { } } + + /** + * This method looks silly at first glance but exists for a reason. + * + * The method {@link GobblinHelixJobLauncher#launchJob(JobListener)} contains boiler plate for handling exceptions and + * mutating the runningMap to communicate state back to the {@link GobblinHelixJobScheduler}. The boiler plate swallows + * exceptions when launching the job because many use cases require that 1 job failure should not affect other jobs by causing the + * entire process to fail through an uncaught exception. + * + * This method is useful for unit testing edge cases where we expect {@link JobException}s during the underlying launch operation. + * It would be nice to not swallow exceptions, but the implications of doing that will require careful refactoring since + * the class {@link GobblinHelixJobLauncher} and {@link GobblinHelixJobScheduler} are shared for 2 quite different cases + * between GaaS and streaming. GaaS typically requiring many short lifetime workflows (where a failure is tolerated) and + * streaming requiring a small number of long running workflows (where failure to submit is unexpected and is not + * tolerated) + * + * @throws JobException + */ + @VisibleForTesting + void launchJobImpl(@Nullable JobListener jobListener) throws JobException { + super.launchJob(jobListener); + } + private TaskConfig getTaskConfig(WorkUnit workUnit, ParallelRunner stateSerDeRunner) throws IOException { String workUnitFilePath = persistWorkUnit(new Path(this.inputWorkUnitDir, this.jobContext.getJobId()), workUnit, stateSerDeRunner); @@ -585,4 +619,4 @@ private void cleanupWorkingDirectory() throws IOException { this.fs.delete(jobStateFilePath, false); } } -} \ No newline at end of file +} diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixJobsMapping.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixJobsMapping.java index ab4d87c4e27..8128aa0aab1 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixJobsMapping.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixJobsMapping.java @@ -36,6 +36,7 @@ import org.apache.gobblin.util.ClassAliasResolver; import org.apache.gobblin.util.ConfigUtils; import org.apache.gobblin.util.JobLauncherUtils; +import org.apache.gobblin.util.PropertiesUtils; /** @@ -96,12 +97,14 @@ public HelixJobsMapping(Config sysConfig, URI fsUri, String rootDir) { public static String createPlanningJobId (Properties jobPlanningProps) { return JobLauncherUtils.newJobId(GobblinClusterConfigurationKeys.PLANNING_JOB_NAME_PREFIX - + JobState.getJobNameFromProps(jobPlanningProps)); + + JobState.getJobNameFromProps(jobPlanningProps), + PropertiesUtils.getPropAsLong(jobPlanningProps, ConfigurationKeys.FLOW_EXECUTION_ID_KEY, System.currentTimeMillis())); } public static String createActualJobId (Properties jobProps) { - return JobLauncherUtils.newJobId(GobblinClusterConfigurationKeys.ACTUAL_JOB_NAME_PREFIX - + JobState.getJobNameFromProps(jobProps)); + return JobLauncherUtils.newJobId(GobblinClusterConfigurationKeys.ACTUAL_JOB_NAME_PREFIX + + JobState.getJobNameFromProps(jobProps), + PropertiesUtils.getPropAsLong(jobProps, ConfigurationKeys.FLOW_EXECUTION_ID_KEY, System.currentTimeMillis())); } @Nullable diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixRetriggeringJobCallable.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixRetriggeringJobCallable.java index 37da2c0bf89..2dfb4b7736a 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixRetriggeringJobCallable.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixRetriggeringJobCallable.java @@ -17,26 +17,23 @@ package org.apache.gobblin.cluster; -import com.google.common.collect.Lists; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.time.Duration; import java.util.HashMap; import java.util.Optional; import java.util.Properties; import java.util.concurrent.Callable; import java.util.concurrent.locks.Lock; -import org.apache.gobblin.metrics.MetricContext; -import org.apache.gobblin.metrics.Tag; -import org.apache.gobblin.metrics.event.EventSubmitter; -import org.apache.gobblin.metrics.event.TimingEvent; import org.apache.hadoop.fs.Path; import org.apache.helix.HelixException; import org.apache.helix.HelixManager; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.io.Closer; import com.google.common.util.concurrent.Striped; import com.typesafe.config.Config; @@ -45,6 +42,10 @@ import org.apache.gobblin.annotation.Alpha; import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.metrics.MetricContext; +import org.apache.gobblin.metrics.Tag; +import org.apache.gobblin.metrics.event.EventSubmitter; +import org.apache.gobblin.metrics.event.TimingEvent; import org.apache.gobblin.runtime.JobException; import org.apache.gobblin.runtime.api.JobExecutionMonitor; import org.apache.gobblin.runtime.api.MutableJobCatalog; @@ -326,7 +327,10 @@ private void runJobExecutionLauncher() throws JobException { // make sure the planning job is initialized (or visible) to other parallel running threads, // so that the same critical section check (querying Helix for job completeness) // can be applied. - HelixUtils.waitJobInitialization(planningJobHelixManager, newPlanningId, newPlanningId); + Duration submissionTimeout = Duration.ofSeconds(PropertiesUtils + .getPropAsLong(sysProps, GobblinClusterConfigurationKeys.HELIX_WORKFLOW_SUBMISSION_TIMEOUT_SECONDS, + GobblinClusterConfigurationKeys.DEFAULT_HELIX_WORKFLOW_SUBMISSION_TIMEOUT_SECONDS)); + HelixUtils.waitJobInitialization(planningJobHelixManager, newPlanningId, newPlanningId, submissionTimeout); } finally { planningJobHelixManager.disconnect(); // end of the critical section to check if a job with same job name is running @@ -414,4 +418,4 @@ void cancel() throws JobException { this.jobScheduler.jobSchedulerMetrics.numCancellationComplete.incrementAndGet(); } -} \ No newline at end of file +} diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java index 308229b3405..d6200296717 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java @@ -17,7 +17,7 @@ package org.apache.gobblin.cluster; -import com.google.common.collect.Lists; +import java.time.Duration; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; @@ -28,15 +28,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import lombok.extern.slf4j.Slf4j; -import org.apache.gobblin.configuration.ConfigurationKeys; -import org.apache.gobblin.metrics.Tag; -import org.apache.gobblin.metrics.event.TimingEvent; -import org.apache.gobblin.runtime.JobException; -import org.apache.gobblin.runtime.JobState; -import org.apache.gobblin.runtime.listeners.JobListener; -import org.apache.gobblin.util.Id; -import org.apache.gobblin.util.JobLauncherUtils; + import org.apache.helix.HelixAdmin; import org.apache.helix.HelixDataAccessor; import org.apache.helix.HelixException; @@ -57,7 +49,21 @@ import org.apache.helix.task.WorkflowContext; import org.apache.helix.tools.ClusterSetup; -import static org.apache.helix.task.TaskState.*; +import com.google.common.collect.Lists; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.metrics.Tag; +import org.apache.gobblin.metrics.event.TimingEvent; +import org.apache.gobblin.runtime.JobException; +import org.apache.gobblin.runtime.JobState; +import org.apache.gobblin.runtime.listeners.JobListener; +import org.apache.gobblin.util.Id; +import org.apache.gobblin.util.JobLauncherUtils; +import org.apache.gobblin.util.PropertiesUtils; + +import static org.apache.helix.task.TaskState.STOPPED; /** @@ -112,23 +118,12 @@ public static String getHelixInstanceName( return namePrefix + "_" + instanceId; } - // We have switched from Helix JobQueue to WorkFlow based job execution. - @Deprecated - public static void submitJobToQueue( - JobConfig.Builder jobConfigBuilder, - String queueName, - String jobName, - TaskDriver helixTaskDriver, - HelixManager helixManager, - long jobQueueDeleteTimeoutSeconds) throws Exception { - submitJobToWorkFlow(jobConfigBuilder, queueName, jobName, helixTaskDriver, helixManager, jobQueueDeleteTimeoutSeconds); - } - static void waitJobInitialization( HelixManager helixManager, - String workFlowName, - String jobName) throws Exception { - WorkflowContext workflowContext = TaskDriver.getWorkflowContext(helixManager, workFlowName); + String workflowName, + String jobName, + Duration timeout) throws Exception { + WorkflowContext workflowContext = TaskDriver.getWorkflowContext(helixManager, workflowName); // If the helix job is deleted from some other thread or a completely external process, // method waitJobCompletion() needs to differentiate between the cases where @@ -136,18 +131,20 @@ static void waitJobInitialization( // 2) it did get initialized but deleted soon after, in which case we should stop waiting // To overcome this issue, we wait here till workflowContext gets initialized long start = System.currentTimeMillis(); - long timeoutMillis = TimeUnit.MINUTES.toMillis(5L); - while (workflowContext == null || workflowContext.getJobState(TaskUtil.getNamespacedJobName(workFlowName, jobName)) == null) { - if (System.currentTimeMillis() - start > timeoutMillis) { - log.error("Job cannot be initialized within {} milliseconds, considered as an error", timeoutMillis); - throw new JobException("Job cannot be initialized within {} milliseconds, considered as an error"); + while (workflowContext == null || workflowContext.getJobState(TaskUtil.getNamespacedJobName(workflowName, jobName)) == null) { + if (System.currentTimeMillis() - start > timeout.toMillis()) { + String errorDescription = String.format("Job cannot be initialized within %s milliseconds, considered as an error. " + + "workflowName=%s, jobName=%s, timeSubmittedEpoch=%s", timeout.toMillis(), workflowName, jobName, start); + log.error(errorDescription); + throw new JobException(errorDescription); } - workflowContext = TaskDriver.getWorkflowContext(helixManager, workFlowName); + workflowContext = TaskDriver.getWorkflowContext(helixManager, workflowName); Thread.sleep(TimeUnit.SECONDS.toMillis(1L)); - log.info("Waiting for work flow initialization."); + log.info("Waiting for workflow initialization. workflowName={}, jobName={}, timeSubmittedEpoch={}, timeoutSeconds={}", + workflowName, jobName, start, timeout.getSeconds()); } - log.info("Work flow {} initialized", workFlowName); + log.info("Workflow {} initialized. timeToInitMs={}", workflowName, System.currentTimeMillis() - start); } /** @@ -165,7 +162,8 @@ public static List> initBaseEventTags(Properties jobProps, if (jobProps.containsKey(ConfigurationKeys.JOB_ID_KEY)) { jobId = jobProps.getProperty(ConfigurationKeys.JOB_ID_KEY); } else { - jobId = JobLauncherUtils.newJobId(JobState.getJobNameFromProps(jobProps)); + jobId = JobLauncherUtils.newJobId(JobState.getJobNameFromProps(jobProps), + PropertiesUtils.getPropAsLong(jobProps, ConfigurationKeys.FLOW_EXECUTION_ID_KEY, System.currentTimeMillis())); jobProps.put(ConfigurationKeys.JOB_ID_KEY, jobId); } @@ -235,16 +233,17 @@ public static void submitJobToWorkFlow(JobConfig.Builder jobConfigBuilder, String jobName, TaskDriver helixTaskDriver, HelixManager helixManager, - long workFlowExpiryTime) throws Exception { + Duration workFlowExpiryTime, + Duration submissionTimeout) throws Exception { - WorkflowConfig workFlowConfig = new WorkflowConfig.Builder().setExpiry(workFlowExpiryTime, TimeUnit.SECONDS).build(); - // Create a work flow for each job with the name being the queue name + WorkflowConfig workFlowConfig = new WorkflowConfig.Builder().setExpiry(workFlowExpiryTime.getSeconds(), TimeUnit.SECONDS).build(); + // Create a workflow for each Gobblin job using the Gobblin job name as the workflow name Workflow workFlow = new Workflow.Builder(workFlowName).setWorkflowConfig(workFlowConfig).addJob(jobName, jobConfigBuilder).build(); // start the workflow helixTaskDriver.start(workFlow); - log.info("Created a work flow {}", workFlowName); + log.info("Created a workflow {}", workFlowName); - waitJobInitialization(helixManager, workFlowName, jobName); + waitJobInitialization(helixManager, workFlowName, jobName, submissionTimeout); } static void waitJobCompletion(HelixManager helixManager, String workFlowName, String jobName, diff --git a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobLauncherTest.java b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobLauncherTest.java index e171591d55e..975dad38b79 100644 --- a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobLauncherTest.java +++ b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobLauncherTest.java @@ -286,6 +286,19 @@ public void testLaunchMultipleJobs() throws Exception { Assert.assertEquals(testListener.getCompletes().get() == 1, true); } + public void testTimeout() throws Exception { + final ConcurrentHashMap runningMap = new ConcurrentHashMap<>(); + + final Properties props = generateJobProperties(this.baseConfig, "testTimeoutTest", "_12345"); + props.setProperty(GobblinClusterConfigurationKeys.HELIX_WORKFLOW_SUBMISSION_TIMEOUT_SECONDS, "0"); + final GobblinHelixJobLauncher gobblinHelixJobLauncher = this.closer.register( + new GobblinHelixJobLauncher(props, this.helixManager, this.appWorkDir, ImmutableList.>of(), runningMap, + java.util.Optional.empty())); + + // using launchJobImpl because we do not want to swallow the exception + Assert.assertThrows(JobException.class, () -> gobblinHelixJobLauncher.launchJobImpl(null)); + } + @Test(enabled = false, dependsOnMethods = {"testLaunchJob", "testLaunchMultipleJobs"}) public void testJobCleanup() throws Exception { final ConcurrentHashMap runningMap = new ConcurrentHashMap<>(); diff --git a/gobblin-compaction/build.gradle b/gobblin-compaction/build.gradle index 64b5fdecdfb..5930aabca05 100644 --- a/gobblin-compaction/build.gradle +++ b/gobblin-compaction/build.gradle @@ -36,7 +36,6 @@ dependencies { compile externalDependency.avro compile externalDependency.commonsLang compile externalDependency.commonsMath - compile externalDependency.mockito compile externalDependency.testng runtimeOnly externalDependency.hadoopCommon @@ -58,6 +57,7 @@ dependencies { testCompile externalDependency.testng testCompile externalDependency.calciteAvatica + testCompile externalDependency.mockito } diff --git a/gobblin-core/src/main/java/org/apache/gobblin/converter/serde/HiveSerDeConverter.java b/gobblin-core/src/main/java/org/apache/gobblin/converter/serde/HiveSerDeConverter.java index 4e696703fef..1f23a9462e2 100644 --- a/gobblin-core/src/main/java/org/apache/gobblin/converter/serde/HiveSerDeConverter.java +++ b/gobblin-core/src/main/java/org/apache/gobblin/converter/serde/HiveSerDeConverter.java @@ -23,8 +23,10 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.Serializer; import org.apache.hadoop.hive.serde2.avro.AvroObjectInspectorGenerator; import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -69,8 +71,8 @@ @Slf4j public class HiveSerDeConverter extends InstrumentedConverter { - private SerDe serializer; - private SerDe deserializer; + private Serializer serializer; + private Deserializer deserializer; @Override public HiveSerDeConverter init(WorkUnitState state) { @@ -78,8 +80,8 @@ public HiveSerDeConverter init(WorkUnitState state) { Configuration conf = HadoopUtils.getConfFromState(state); try { - this.serializer = HiveSerDeWrapper.getSerializer(state).getSerDe(); - this.deserializer = HiveSerDeWrapper.getDeserializer(state).getSerDe(); + this.serializer = (Serializer) HiveSerDeWrapper.getSerializer(state).getSerDe(); + this.deserializer = (Deserializer) HiveSerDeWrapper.getDeserializer(state).getSerDe(); this.deserializer.initialize(conf, state.getProperties()); setColumnsIfPossible(state); this.serializer.initialize(conf, state.getProperties()); diff --git a/gobblin-core/src/main/java/org/apache/gobblin/writer/HiveWritableHdfsDataWriterBuilder.java b/gobblin-core/src/main/java/org/apache/gobblin/writer/HiveWritableHdfsDataWriterBuilder.java index 5dba8617aca..6b056848e8f 100644 --- a/gobblin-core/src/main/java/org/apache/gobblin/writer/HiveWritableHdfsDataWriterBuilder.java +++ b/gobblin-core/src/main/java/org/apache/gobblin/writer/HiveWritableHdfsDataWriterBuilder.java @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.avro.Schema; +import org.apache.hadoop.hive.serde2.Serializer; import org.apache.hadoop.io.Writable; import com.google.common.base.Preconditions; @@ -54,7 +55,7 @@ public DataWriter build() throws IOException { if (!properties.contains(WRITER_WRITABLE_CLASS) || !properties.contains(WRITER_OUTPUT_FORMAT_CLASS)) { HiveSerDeWrapper serializer = HiveSerDeWrapper.getSerializer(properties); - properties.setProp(WRITER_WRITABLE_CLASS, serializer.getSerDe().getSerializedClass().getName()); + properties.setProp(WRITER_WRITABLE_CLASS, ((Serializer) serializer.getSerDe()).getSerializedClass().getName()); properties.setProp(WRITER_OUTPUT_FORMAT_CLASS, serializer.getOutputFormatClassName()); } diff --git a/gobblin-core/src/main/java/org/apache/gobblin/writer/PartitionedDataWriter.java b/gobblin-core/src/main/java/org/apache/gobblin/writer/PartitionedDataWriter.java index 800676da669..d5f0e817700 100644 --- a/gobblin-core/src/main/java/org/apache/gobblin/writer/PartitionedDataWriter.java +++ b/gobblin-core/src/main/java/org/apache/gobblin/writer/PartitionedDataWriter.java @@ -22,9 +22,12 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import org.apache.avro.SchemaBuilder; @@ -115,6 +118,7 @@ public class PartitionedDataWriter extends WriterWrapper implements Fin @Getter @VisibleForTesting private long totalBytesFromEvictedWriters; + private ExecutorService createWriterPool; public PartitionedDataWriter(DataWriterBuilder builder, final State state) @@ -125,6 +129,7 @@ public PartitionedDataWriter(DataWriterBuilder builder, final State state) this.isSpeculativeAttemptSafe = true; this.isWatermarkCapable = true; this.baseWriterId = builder.getWriterId(); + this.createWriterPool = Executors.newSingleThreadExecutor(); this.closer = Closer.create(); this.writerBuilder = builder; this.controlMessageHandler = new PartitionDataWriterMessageHandler(); @@ -170,9 +175,12 @@ public DataWriter get() { try { log.info(String.format("Adding one more writer to loading cache of existing writer " + "with size = %d", partitionWriters.size())); - return createPartitionWriter(key); - } catch (IOException e) { + Future> future = createWriterPool.submit(() -> createPartitionWriter(key)); + return future.get(writeTimeoutInterval, TimeUnit.SECONDS); + } catch (ExecutionException | InterruptedException e) { throw new RuntimeException("Error creating writer", e); + } catch (TimeoutException e) { + throw new RuntimeException(String.format("Failed to create writer due to timeout. The operation timed out after %s seconds.", writeTimeoutInterval), e); } } }, state), state, key); @@ -326,6 +334,7 @@ public synchronized void close() serializePartitionInfoToState(); } finally { closeWritersInCache(); + this.createWriterPool.shutdown(); this.closer.close(); } } diff --git a/gobblin-core/src/test/java/org/apache/gobblin/writer/PartitionedWriterTest.java b/gobblin-core/src/test/java/org/apache/gobblin/writer/PartitionedWriterTest.java index 68f3343f85c..3148e5e5b8f 100644 --- a/gobblin-core/src/test/java/org/apache/gobblin/writer/PartitionedWriterTest.java +++ b/gobblin-core/src/test/java/org/apache/gobblin/writer/PartitionedWriterTest.java @@ -17,6 +17,7 @@ package org.apache.gobblin.writer; +import com.google.common.util.concurrent.UncheckedExecutionException; import java.io.IOException; import java.util.List; @@ -166,12 +167,25 @@ public void testControlMessageHandler() throws IOException { writer.close(); } + @Test + public void testTimeoutWhenCreatingWriter() throws IOException { + State state = new State(); + state.setProp(ConfigurationKeys.WRITER_PARTITIONER_CLASS, TestPartitioner.class.getCanonicalName()); + state.setProp(PartitionedDataWriter.PARTITIONED_WRITER_CACHE_TTL_SECONDS, 6); + TestPartitionAwareWriterBuilder builder = new TestPartitionAwareWriterBuilder(true); + + PartitionedDataWriter writer = new PartitionedDataWriter(builder, state); + + String record1 = "abc"; + Assert.expectThrows(UncheckedExecutionException.class, () -> writer.writeEnvelope(new RecordEnvelope(record1))); + } + @Test public void testPartitionWriterCacheRemovalListener() throws IOException, InterruptedException { State state = new State(); state.setProp(ConfigurationKeys.WRITER_PARTITIONER_CLASS, TestPartitioner.class.getCanonicalName()); - state.setProp(PartitionedDataWriter.PARTITIONED_WRITER_CACHE_TTL_SECONDS, 1); + state.setProp(PartitionedDataWriter.PARTITIONED_WRITER_CACHE_TTL_SECONDS, 3); TestPartitionAwareWriterBuilder builder = new TestPartitionAwareWriterBuilder(); PartitionedDataWriter writer = new PartitionedDataWriter(builder, state); @@ -183,7 +197,7 @@ public void testPartitionWriterCacheRemovalListener() writer.writeEnvelope(new RecordEnvelope(record2)); //Sleep for more than cache expiration interval - Thread.sleep(1500); + Thread.sleep(3500); //Call cache clean up to ensure removal of expired entries. writer.getPartitionWriters().cleanUp(); diff --git a/gobblin-core/src/test/java/org/apache/gobblin/writer/test/TestPartitionAwareWriterBuilder.java b/gobblin-core/src/test/java/org/apache/gobblin/writer/test/TestPartitionAwareWriterBuilder.java index 8da42d728ad..75eafcf51cd 100644 --- a/gobblin-core/src/test/java/org/apache/gobblin/writer/test/TestPartitionAwareWriterBuilder.java +++ b/gobblin-core/src/test/java/org/apache/gobblin/writer/test/TestPartitionAwareWriterBuilder.java @@ -37,10 +37,17 @@ public class TestPartitionAwareWriterBuilder extends PartitionAwareDataWriterBuilder { public final Queue actions = Queues.newArrayDeque(); + private boolean testTimeout; public enum Actions { BUILD, WRITE, COMMIT, CLEANUP, CLOSE } + public TestPartitionAwareWriterBuilder() { + this(false); + } + public TestPartitionAwareWriterBuilder(boolean testTimeout) { + this.testTimeout = testTimeout; + } @Override public boolean validatePartitionSchema(Schema partitionSchema) { @@ -50,6 +57,13 @@ public boolean validatePartitionSchema(Schema partitionSchema) { @Override public DataWriter build() throws IOException { + if (testTimeout) { + try { + Thread.sleep(10*1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } String partition = this.partition.get().get(TestPartitioner.PARTITION).toString(); this.actions.add(new Action(Actions.BUILD, partition, null)); if (partition.matches(".*\\d+.*")) { diff --git a/gobblin-data-management/build.gradle b/gobblin-data-management/build.gradle index 47b8fa6b585..c3b3129ea62 100644 --- a/gobblin-data-management/build.gradle +++ b/gobblin-data-management/build.gradle @@ -34,13 +34,11 @@ dependencies { compile externalDependency.slf4j compile externalDependency.jodaTime compile externalDependency.metricsCore - compile externalDependency.mockito compile externalDependency.commonsCodec compile externalDependency.commonsCompress compile externalDependency.commonsIo compile externalDependency.gson compile externalDependency.commonsCodec - compile externalDependency.mockito compile externalDependency.typesafeConfig compile externalDependency.findBugsAnnotations compile externalDependency.testng @@ -61,6 +59,7 @@ dependencies { testCompile externalDependency.hamcrest testCompile externalDependency.testng testCompile externalDependency.hiveJdbc + testCompile externalDependency.mockito testRuntime project(":gobblin-modules:gobblin-crypto-provider") // for GPG testCompile project(":gobblin-modules:gobblin-crypto") // for GPG diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/CopyableFile.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/CopyableFile.java index 23c27b61da1..85fa80f0fe8 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/CopyableFile.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/CopyableFile.java @@ -17,10 +17,13 @@ package org.apache.gobblin.data.management.copy; +import com.google.common.cache.Cache; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -58,6 +61,7 @@ @Setter @NoArgsConstructor(access = AccessLevel.PROTECTED) @EqualsAndHashCode(callSuper = true) +@Slf4j public class CopyableFile extends CopyEntity implements File { private static final byte[] EMPTY_CHECKSUM = new byte[0]; @@ -375,6 +379,36 @@ public static List resolveReplicatedOwnerAndPermissionsRecur return ownerAndPermissions; } + /** + * Compute the correct {@link OwnerAndPermission} obtained from replicating source owner and permissions and applying + * the {@link PreserveAttributes} rules for fromPath and every ancestor up to but excluding toPath. + * Unlike the resolveReplicatedOwnerAndPermissionsRecursively() method, this method utilizes permissionMap as a cache to minimize the number of calls to HDFS. + * It is recommended to use this method when recursively calculating permissions for numerous files that share the same ancestor. + * + * @return A list of the computed {@link OwnerAndPermission}s starting from fromPath, up to but excluding toPath. + * @throws IOException if toPath is not an ancestor of fromPath. + */ + public static List resolveReplicatedOwnerAndPermissionsRecursivelyWithCache(FileSystem sourceFs, Path fromPath, + Path toPath, CopyConfiguration copyConfiguration, Cache permissionMap) + throws IOException, ExecutionException { + + if (!PathUtils.isAncestor(toPath, fromPath)) { + throw new IOException(String.format("toPath %s must be an ancestor of fromPath %s.", toPath, fromPath)); + } + + List ownerAndPermissions = Lists.newArrayList(); + Path currentPath = fromPath; + + while (currentPath.getParent() != null && PathUtils.isAncestor(toPath, currentPath.getParent())) { + Path finalCurrentPath = currentPath; + ownerAndPermissions.add(permissionMap.get(finalCurrentPath.toString(), () -> resolveReplicatedOwnerAndPermission(sourceFs, + finalCurrentPath, copyConfiguration))); + currentPath = currentPath.getParent(); + } + + return ownerAndPermissions; + } + public static Map resolveReplicatedAncestorOwnerAndPermissionsRecursively(FileSystem sourceFs, Path fromPath, Path toPath, CopyConfiguration copyConfiguration) throws IOException { Preconditions.checkArgument(sourceFs.getFileStatus(fromPath).isDirectory(), "Source path must be a directory."); diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/ManifestBasedDataset.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/ManifestBasedDataset.java index 17de89458f2..fbb88640b23 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/ManifestBasedDataset.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/ManifestBasedDataset.java @@ -18,6 +18,8 @@ package org.apache.gobblin.data.management.copy; import com.google.common.base.Optional; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.JsonIOException; @@ -27,6 +29,7 @@ import java.util.Iterator; import java.util.List; import java.util.Properties; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.commit.CommitStep; import org.apache.gobblin.data.management.copy.entities.PrePublishStep; @@ -47,12 +50,15 @@ public class ManifestBasedDataset implements IterableCopyableDataset { private static final String DELETE_FILE_NOT_EXIST_ON_SOURCE = ManifestBasedDatasetFinder.CONFIG_PREFIX + ".deleteFileNotExistOnSource"; private static final String COMMON_FILES_PARENT = ManifestBasedDatasetFinder.CONFIG_PREFIX + ".commonFilesParent"; + private static final String PERMISSION_CACHE_TTL_SECONDS = ManifestBasedDatasetFinder.CONFIG_PREFIX + ".permission.cache.ttl.seconds"; + private static final String DEFAULT_PERMISSION_CACHE_TTL_SECONDS = "30"; private static final String DEFAULT_COMMON_FILES_PARENT = "/"; private final FileSystem fs; private final Path manifestPath; private final Properties properties; private final boolean deleteFileThatNotExistOnSource; private final String commonFilesParent; + private final int permissionCacheTTLSeconds; public ManifestBasedDataset(final FileSystem fs, Path manifestPath, Properties properties) { this.fs = fs; @@ -60,6 +66,7 @@ public ManifestBasedDataset(final FileSystem fs, Path manifestPath, Properties p this.properties = properties; this.deleteFileThatNotExistOnSource = Boolean.parseBoolean(properties.getProperty(DELETE_FILE_NOT_EXIST_ON_SOURCE, "false")); this.commonFilesParent = properties.getProperty(COMMON_FILES_PARENT, DEFAULT_COMMON_FILES_PARENT); + this.permissionCacheTTLSeconds = Integer.parseInt(properties.getProperty(PERMISSION_CACHE_TTL_SECONDS, DEFAULT_PERMISSION_CACHE_TTL_SECONDS)); } @Override @@ -82,25 +89,31 @@ public Iterator> getFileSetIterator(FileSystem targetFs, Cop List toDelete = Lists.newArrayList(); //todo: put permission preserve logic here? try { + long startTime = System.currentTimeMillis(); manifests = CopyManifest.getReadIterator(this.fs, this.manifestPath); + Cache permissionMap = CacheBuilder.newBuilder().expireAfterAccess(permissionCacheTTLSeconds, TimeUnit.SECONDS).build(); + int numFiles = 0; while (manifests.hasNext()) { + numFiles++; + CopyManifest.CopyableUnit file = manifests.next(); //todo: We can use fileSet to partition the data in case of some softbound issue //todo: After partition, change this to directly return iterator so that we can save time if we meet resources limitation - CopyManifest.CopyableUnit file = manifests.next(); Path fileToCopy = new Path(file.fileName); - if (this.fs.exists(fileToCopy)) { + if (fs.exists(fileToCopy)) { boolean existOnTarget = targetFs.exists(fileToCopy); - FileStatus srcFile = this.fs.getFileStatus(fileToCopy); - if (!existOnTarget || shouldCopy(this.fs, targetFs, srcFile, targetFs.getFileStatus(fileToCopy), configuration)) { - CopyableFile copyableFile = - CopyableFile.fromOriginAndDestination(this.fs, srcFile, fileToCopy, configuration) + FileStatus srcFile = fs.getFileStatus(fileToCopy); + OwnerAndPermission replicatedPermission = CopyableFile.resolveReplicatedOwnerAndPermission(fs, srcFile, configuration); + if (!existOnTarget || shouldCopy(targetFs, srcFile, targetFs.getFileStatus(fileToCopy), replicatedPermission)) { + CopyableFile.Builder copyableFileBuilder = + CopyableFile.fromOriginAndDestination(fs, srcFile, fileToCopy, configuration) .fileSet(datasetURN()) .datasetOutputPath(fileToCopy.toString()) - .ancestorsOwnerAndPermission(CopyableFile - .resolveReplicatedOwnerAndPermissionsRecursively(this.fs, fileToCopy.getParent(), - new Path(this.commonFilesParent), configuration)) - .build(); - copyableFile.setFsDatasets(this.fs, targetFs); + .ancestorsOwnerAndPermission( + CopyableFile.resolveReplicatedOwnerAndPermissionsRecursivelyWithCache(fs, fileToCopy.getParent(), + new Path(commonFilesParent), configuration, permissionMap)) + .destinationOwnerAndPermission(replicatedPermission); + CopyableFile copyableFile = copyableFileBuilder.build(); + copyableFile.setFsDatasets(fs, targetFs); copyEntities.add(copyableFile); if (existOnTarget && srcFile.isFile()) { // this is to match the existing publishing behavior where we won't rewrite the target when it's already existed @@ -108,7 +121,7 @@ public Iterator> getFileSetIterator(FileSystem targetFs, Cop toDelete.add(targetFs.getFileStatus(fileToCopy)); } } - } else if (this.deleteFileThatNotExistOnSource && targetFs.exists(fileToCopy)){ + } else if (deleteFileThatNotExistOnSource && targetFs.exists(fileToCopy)) { toDelete.add(targetFs.getFileStatus(fileToCopy)); } } @@ -117,6 +130,7 @@ public Iterator> getFileSetIterator(FileSystem targetFs, Cop CommitStep step = new DeleteFileCommitStep(targetFs, toDelete, this.properties, Optional.absent()); copyEntities.add(new PrePublishStep(datasetURN(), Maps.newHashMap(), step, 1)); } + log.info(String.format("Workunits calculation took %s milliseconds to process %s files", System.currentTimeMillis() - startTime, numFiles)); } catch (JsonIOException| JsonSyntaxException e) { //todo: update error message to point to a sample json file instead of schema which is hard to understand log.warn(String.format("Failed to read Manifest path %s on filesystem %s, please make sure it's in correct json format with schema" @@ -134,11 +148,10 @@ public Iterator> getFileSetIterator(FileSystem targetFs, Cop return Collections.singleton(new FileSet.Builder<>(datasetURN(), this).add(copyEntities).build()).iterator(); } - private static boolean shouldCopy(FileSystem srcFs, FileSystem targetFs, FileStatus fileInSource, FileStatus fileInTarget, CopyConfiguration copyConfiguration) + private static boolean shouldCopy(FileSystem targetFs, FileStatus fileInSource, FileStatus fileInTarget, OwnerAndPermission replicatedPermission) throws IOException { if (fileInSource.isDirectory() || fileInSource.getModificationTime() == fileInTarget.getModificationTime()) { // if source is dir or source and dst has same version, we compare the permission to determine whether it needs another sync - OwnerAndPermission replicatedPermission = CopyableFile.resolveReplicatedOwnerAndPermission(srcFs, fileInSource, copyConfiguration); return !replicatedPermission.hasSameOwnerAndPermission(targetFs, fileInTarget); } return fileInSource.getModificationTime() > fileInTarget.getModificationTime(); diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java index 7032a88e2b4..f2e39c316b6 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveDataset.java @@ -17,7 +17,6 @@ package org.apache.gobblin.data.management.copy.hive; -import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.util.Collections; import java.util.Comparator; @@ -26,11 +25,6 @@ import java.util.Map; import java.util.Properties; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; - import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -40,16 +34,21 @@ import org.apache.hadoop.hive.ql.metadata.Table; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; -import com.google.common.base.Optional; -import com.google.common.collect.Iterators; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigValue; import com.typesafe.config.ConfigValueType; -import com.google.common.collect.ImmutableSet; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.annotation.Alpha; import org.apache.gobblin.configuration.State; @@ -68,8 +67,6 @@ import org.apache.gobblin.util.PathUtils; import org.apache.gobblin.util.request_allocation.PushDownRequestor; -import static org.apache.gobblin.data.management.copy.hive.HiveTargetPathHelper.*; - /** * Hive dataset implementing {@link CopyableDataset}. */ @@ -149,7 +146,7 @@ public HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table tabl public Iterator> getFileSetIterator(FileSystem targetFs, CopyConfiguration configuration) throws IOException { if (!canCopyTable(configuration)) { - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } try { return new HiveCopyEntityHelper(this, configuration, targetFs).getCopyEntities(configuration); @@ -158,7 +155,7 @@ public Iterator> getFileSetIterator(FileSystem targetFs, Cop if (configuration.isAbortOnSingleDatasetFailure()) { throw new RuntimeException(ioe); } - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } } @@ -171,7 +168,7 @@ public Iterator> getFileSetIterator(FileSystem targetFs, Cop Comparator> prioritizer, PushDownRequestor> requestor) throws IOException { if (!canCopyTable(configuration)) { - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } try { List> fileSetList = Lists.newArrayList(new HiveCopyEntityHelper(this, configuration, targetFs) @@ -183,7 +180,7 @@ public Iterator> getFileSetIterator(FileSystem targetFs, Cop if (configuration.isAbortOnSingleDatasetFailure()) { throw new RuntimeException(ioe); } - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } } diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDataset.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDataset.java index f4db7d4ff78..05f1e265d2a 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDataset.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDataset.java @@ -73,7 +73,7 @@ public class IcebergDataset implements PrioritizedCopyableDataset { private final boolean shouldTolerateMissingSourceFiles = true; // TODO: make parameterizable, if desired /** Destination database name */ - public static final String DESTINATION_DATABASE_KEY = IcebergDatasetFinder.ICEBERG_DATASET_PREFIX + ".copy.destination.database"; + public static final String DESTINATION_DATABASE_KEY = IcebergDatasetFinder.ICEBERG_DATASET_PREFIX + ".destination.database"; public IcebergDataset(String db, String table, IcebergTable srcIcebergTable, IcebergTable destIcebergTable, Properties properties, FileSystem sourceFs) { this.dbName = db; @@ -155,7 +155,8 @@ Collection generateCopyEntities(FileSystem targetFs, CopyConfigurati fileEntity.setDestinationData(getDestinationDataset(targetFs)); copyEntities.add(fileEntity); } - copyEntities.add(createPostPublishStep(this.srcIcebergTable, this.destIcebergTable)); + // TODO: Filter properties specific to iceberg registration and avoid serializing every global property + copyEntities.add(createPostPublishStep(this.dbName, this.inputTableName, this.properties)); log.info("~{}.{}~ generated {} copy entities", dbName, inputTableName, copyEntities.size()); return copyEntities; } @@ -316,8 +317,8 @@ protected DatasetDescriptor getDestinationDataset(FileSystem targetFs) { return this.destIcebergTable.getDatasetDescriptor(targetFs); } - private PostPublishStep createPostPublishStep(IcebergTable srcIcebergTable, IcebergTable dstIcebergTable) { - IcebergRegisterStep icebergRegisterStep = new IcebergRegisterStep(srcIcebergTable, dstIcebergTable); + private PostPublishStep createPostPublishStep(String dbName, String inputTableName, Properties properties) { + IcebergRegisterStep icebergRegisterStep = new IcebergRegisterStep(dbName, inputTableName, properties); return new PostPublishStep(getFileSetId(), Maps.newHashMap(), icebergRegisterStep, 0); } } diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDatasetFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDatasetFinder.java index b20a1bc292c..beded5a723a 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDatasetFinder.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDatasetFinder.java @@ -23,7 +23,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import org.apache.commons.lang.StringUtils; @@ -33,9 +32,13 @@ import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigValue; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.gobblin.config.ConfigBuilder; import org.apache.gobblin.dataset.DatasetConstants; import org.apache.gobblin.dataset.IterableDatasetFinder; import org.apache.gobblin.util.HadoopUtils; @@ -49,20 +52,26 @@ @RequiredArgsConstructor public class IcebergDatasetFinder implements IterableDatasetFinder { public static final String ICEBERG_DATASET_PREFIX = DatasetConstants.PLATFORM_ICEBERG + ".dataset"; - public static final String ICEBERG_CLUSTER_KEY = "cluster"; - public static final String ICEBERG_SRC_CATALOG_CLASS_KEY = ICEBERG_DATASET_PREFIX + ".source.catalog.class"; public static final String DEFAULT_ICEBERG_CATALOG_CLASS = "org.apache.gobblin.data.management.copy.iceberg.IcebergHiveCatalog"; - public static final String ICEBERG_SRC_CATALOG_URI_KEY = ICEBERG_DATASET_PREFIX + ".source.catalog.uri"; - public static final String ICEBERG_SRC_CLUSTER_NAME = ICEBERG_DATASET_PREFIX + ".source.cluster.name"; - public static final String ICEBERG_DEST_CATALOG_CLASS_KEY = ICEBERG_DATASET_PREFIX + ".destination.catalog.class"; - public static final String ICEBERG_DEST_CATALOG_URI_KEY = ICEBERG_DATASET_PREFIX + ".copy.destination.catalog.uri"; - public static final String ICEBERG_DEST_CLUSTER_NAME = ICEBERG_DATASET_PREFIX + ".destination.cluster.name"; + public static final String ICEBERG_CATALOG_KEY = "catalog"; + /** + * This is used with a prefix: "{@link IcebergDatasetFinder#ICEBERG_DATASET_PREFIX}" + "." + "(source or destination)" + "." + "{@link IcebergDatasetFinder#ICEBERG_CATALOG_KEY}" + "..." + * It is an open-ended pattern used to pass arbitrary catalog specific properties + */ + public static final String ICEBERG_CATALOG_CLASS_KEY = "class"; public static final String ICEBERG_DB_NAME = ICEBERG_DATASET_PREFIX + ".database.name"; public static final String ICEBERG_TABLE_NAME = ICEBERG_DATASET_PREFIX + ".table.name"; public enum CatalogLocation { SOURCE, - DESTINATION + DESTINATION; + + /** + * Provides prefix for configs based on the catalog location to filter catalog specific properties + */ + public String getConfigPrefix() { + return ICEBERG_DATASET_PREFIX + "." + this.toString().toLowerCase() + "." + ICEBERG_CATALOG_KEY + "."; + } } protected final FileSystem sourceFs; @@ -88,7 +97,7 @@ public List findDatasets() throws IOException { IcebergCatalog sourceIcebergCatalog = createIcebergCatalog(this.properties, CatalogLocation.SOURCE); IcebergCatalog destinationIcebergCatalog = createIcebergCatalog(this.properties, CatalogLocation.DESTINATION); /* Each Iceberg dataset maps to an Iceberg table */ - matchingDatasets.add(createIcebergDataset(dbName, tblName, sourceIcebergCatalog, destinationIcebergCatalog, properties, sourceFs)); + matchingDatasets.add(createIcebergDataset(dbName, tblName, sourceIcebergCatalog, destinationIcebergCatalog, this.properties, this.sourceFs)); log.info("Found {} matching datasets: {} for the database name: {} and table name: {}", matchingDatasets.size(), matchingDatasets, dbName, tblName); // until future support added to specify multiple icebergs, count expected always to be one return matchingDatasets; @@ -118,30 +127,26 @@ protected IcebergDataset createIcebergDataset(String dbName, String tblName, Ice return new IcebergDataset(dbName, tblName, srcIcebergTable, destIcebergTable, properties, fs); } - protected IcebergCatalog createIcebergCatalog(Properties properties, CatalogLocation location) throws IOException { - Map catalogProperties = new HashMap<>(); + protected static IcebergCatalog createIcebergCatalog(Properties properties, CatalogLocation location) throws IOException { + String prefix = location.getConfigPrefix(); + Map catalogProperties = buildMapFromPrefixChildren(properties, prefix); + // TODO: Filter properties specific to Hadoop Configuration configuration = HadoopUtils.getConfFromProperties(properties); - String catalogUri; - String icebergCatalogClassName; - switch (location) { - case SOURCE: - catalogUri = properties.getProperty(ICEBERG_SRC_CATALOG_URI_KEY); - Preconditions.checkNotNull(catalogUri, "Provide: {%s} as Source Catalog Table Service URI is required", ICEBERG_SRC_CATALOG_URI_KEY); - // introducing an optional property for catalogs requiring cluster specific properties - Optional.ofNullable(properties.getProperty(ICEBERG_SRC_CLUSTER_NAME)).ifPresent(value -> catalogProperties.put(ICEBERG_CLUSTER_KEY, value)); - icebergCatalogClassName = properties.getProperty(ICEBERG_SRC_CATALOG_CLASS_KEY, DEFAULT_ICEBERG_CATALOG_CLASS); - break; - case DESTINATION: - catalogUri = properties.getProperty(ICEBERG_DEST_CATALOG_URI_KEY); - Preconditions.checkNotNull(catalogUri, "Provide: {%s} as Destination Catalog Table Service URI is required", ICEBERG_DEST_CATALOG_URI_KEY); - // introducing an optional property for catalogs requiring cluster specific properties - Optional.ofNullable(properties.getProperty(ICEBERG_DEST_CLUSTER_NAME)).ifPresent(value -> catalogProperties.put(ICEBERG_CLUSTER_KEY, value)); - icebergCatalogClassName = properties.getProperty(ICEBERG_DEST_CATALOG_CLASS_KEY, DEFAULT_ICEBERG_CATALOG_CLASS); - break; - default: - throw new UnsupportedOperationException("Incorrect desired location: %s provided for creating Iceberg Catalog" + location); - } - catalogProperties.put(CatalogProperties.URI, catalogUri); + String icebergCatalogClassName = catalogProperties.getOrDefault(ICEBERG_CATALOG_CLASS_KEY, DEFAULT_ICEBERG_CATALOG_CLASS); return IcebergCatalogFactory.create(icebergCatalogClassName, catalogProperties, configuration); } + + /** + * Filters the properties based on a prefix using {@link ConfigBuilder#loadProps(Properties, String)} and creates a {@link Map} + */ + protected static Map buildMapFromPrefixChildren(Properties properties, String configPrefix) { + Map catalogProperties = new HashMap<>(); + Config config = ConfigBuilder.create().loadProps(properties, configPrefix).build(); + for (Map.Entry entry : config.entrySet()) { + catalogProperties.put(entry.getKey(), entry.getValue().unwrapped().toString()); + } + String catalogUri = config.getString(CatalogProperties.URI); + Preconditions.checkNotNull(catalogUri, "Provide: {%s} as Catalog Table Service URI is required", configPrefix + "." + CatalogProperties.URI); + return catalogProperties; + } } diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergRegisterStep.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergRegisterStep.java index 75f26787b09..8f32f8cc039 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergRegisterStep.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergRegisterStep.java @@ -18,6 +18,7 @@ package org.apache.gobblin.data.management.copy.iceberg; import java.io.IOException; +import java.util.Properties; import org.apache.iceberg.TableMetadata; @@ -33,8 +34,9 @@ @AllArgsConstructor public class IcebergRegisterStep implements CommitStep { - private final IcebergTable srcIcebergTable; - private final IcebergTable destIcebergTable; + private final String dbName; + private final String tblName; + private final Properties properties; @Override public boolean isCompleted() throws IOException { @@ -43,12 +45,20 @@ public boolean isCompleted() throws IOException { @Override public void execute() throws IOException { + IcebergTable srcIcebergTable = IcebergDatasetFinder.createIcebergCatalog(this.properties, IcebergDatasetFinder.CatalogLocation.SOURCE) + .openTable(this.dbName, this.tblName); + IcebergTable destIcebergTable = IcebergDatasetFinder.createIcebergCatalog(this.properties, IcebergDatasetFinder.CatalogLocation.DESTINATION) + .openTable(this.dbName, this.tblName); TableMetadata destinationMetadata = null; try { - destinationMetadata = this.destIcebergTable.accessTableMetadata(); + destinationMetadata = destIcebergTable.accessTableMetadata(); } catch (IcebergTable.TableNotFoundException tnfe) { log.warn("Destination TableMetadata doesn't exist because: " , tnfe); } - this.destIcebergTable.registerIcebergTable(this.srcIcebergTable.accessTableMetadata(), destinationMetadata); + destIcebergTable.registerIcebergTable(srcIcebergTable.accessTableMetadata(), destinationMetadata); + } + @Override + public String toString() { + return String.format("Registering Iceberg Table: {%s}.{%s} ", this.dbName, this.tblName); } } diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergTable.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergTable.java index e8d0ee0ac28..6671ebdeb64 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergTable.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergTable.java @@ -198,7 +198,8 @@ protected DatasetDescriptor getDatasetDescriptor(FileSystem fs) { * @param dstMetadata is null if destination {@link IcebergTable} is absent, in which case registration is skipped */ protected void registerIcebergTable(TableMetadata srcMetadata, TableMetadata dstMetadata) { if (dstMetadata != null) { - this.tableOps.commit(srcMetadata, dstMetadata); + // use current destination metadata as 'base metadata' and source as 'updated metadata' while committing + this.tableOps.commit(dstMetadata, srcMetadata.replaceProperties(dstMetadata.properties())); } } } diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/writer/FileAwareInputStreamDataWriter.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/writer/FileAwareInputStreamDataWriter.java index ed6c754d50c..837174c2055 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/writer/FileAwareInputStreamDataWriter.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/writer/FileAwareInputStreamDataWriter.java @@ -42,7 +42,6 @@ import com.google.common.base.Optional; import com.google.common.base.Predicate; import com.google.common.base.Strings; -import com.google.common.collect.Iterators; import lombok.extern.slf4j.Slf4j; @@ -458,7 +457,7 @@ public void commit() setFilePermissions(copyableFile); Iterator ancestorOwnerAndPermissionIt = - copyableFile.getAncestorsOwnerAndPermission() == null ? Iterators.emptyIterator() + copyableFile.getAncestorsOwnerAndPermission() == null ? Collections.emptyIterator() : copyableFile.getAncestorsOwnerAndPermission().iterator(); ensureDirectoryExists(this.fs, outputFilePath.getParent(), ancestorOwnerAndPermissionIt); diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/partition/CopyableDatasetRequestor.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/partition/CopyableDatasetRequestor.java index e5ca9f63b46..692ec99c721 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/partition/CopyableDatasetRequestor.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/partition/CopyableDatasetRequestor.java @@ -90,7 +90,7 @@ public Iterator> iterator() { throw new RuntimeException(String.format("Could not get FileSets for dataset %s", this.dataset.datasetURN()), exc); } log.error(String.format("Could not get FileSets for dataset %s. Skipping.", this.dataset.datasetURN()), exc); - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } } diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/retention/DatasetCleaner.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/retention/DatasetCleaner.java index 30da80fe2f2..13841ce4528 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/retention/DatasetCleaner.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/retention/DatasetCleaner.java @@ -176,7 +176,7 @@ public void onSuccess(Void arg0) { @Override public void close() throws IOException { try { - if (this.finishCleanSignal.isPresent()) { + if (this.finishCleanSignal != null && this.finishCleanSignal.isPresent()) { this.finishCleanSignal.get().await(); } if (!this.throwables.isEmpty()) { diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/source/LoopingDatasetFinderSource.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/source/LoopingDatasetFinderSource.java index 8e39bd20f82..91a1ed94238 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/source/LoopingDatasetFinderSource.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/source/LoopingDatasetFinderSource.java @@ -17,6 +17,7 @@ package org.apache.gobblin.data.management.source; import java.io.IOException; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -31,6 +32,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.PeekingIterator; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + import org.apache.gobblin.configuration.ConfigurationKeys; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.WorkUnitState; @@ -44,9 +48,6 @@ import org.apache.gobblin.source.workunit.WorkUnit; import org.apache.gobblin.source.workunit.WorkUnitStream; -import javax.annotation.Nullable; -import lombok.extern.slf4j.Slf4j; - /** * A source that processes datasets generated by a {@link org.apache.gobblin.dataset.DatasetsFinder}, processing a few of * them each run, and continuing from where it left off in the next run. When it is done processing all the datasets, it @@ -160,7 +161,7 @@ public DeepIterator(Iterator baseIterator, String previousDatasetUrnWat this.currentPartitionIterator = getPartitionIterator((PartitionableDataset) equalDataset); advanceUntilLargerThan(Iterators.peekingIterator(this.currentPartitionIterator), previousPartitionUrnWatermark); } else { - this.currentPartitionIterator = Iterators.emptyIterator(); + this.currentPartitionIterator = Collections.emptyIterator(); } } @@ -188,7 +189,7 @@ private Iterator getPartitionIterator(Par .iterator(); } catch (IOException ioe) { log.error("Failed to get partitions for dataset " + dataset.getUrn()); - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } } diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/version/finder/AbstractHiveDatasetVersionFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/version/finder/AbstractHiveDatasetVersionFinder.java index f12204862bb..a7a933bcf99 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/version/finder/AbstractHiveDatasetVersionFinder.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/version/finder/AbstractHiveDatasetVersionFinder.java @@ -18,11 +18,13 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.ql.metadata.Partition; import com.google.common.base.Function; @@ -56,6 +58,8 @@ public Class versionClass() { * Calls {@link #getDatasetVersion(Partition)} for every {@link Partition} found. *

* Note: If an exception occurs while processing a partition, that partition will be ignored in the returned collection + * Also note that if the dataset passed is a view type, we will return an empty list even if the underlying table is + * partitioned. *

* * @throws IllegalArgumentException if dataset is not a {@link HiveDataset}. Or if {@link HiveDataset#getTable()} @@ -69,7 +73,13 @@ public Collection findDatasetVersions(Dataset dataset) throw final HiveDataset hiveDataset = (HiveDataset) dataset; if (!hiveDataset.getTable().isPartitioned()) { - throw new IllegalArgumentException("HiveDatasetVersionFinder is only compatible with partitioned hive tables"); + if (hiveDataset.getTable().getTableType() == TableType.VIRTUAL_VIEW) { + log.warn("Skipping processing a view type dataset: ", ((HiveDataset) dataset).getTable().getTableName()); + return Collections.emptyList(); + } else { + throw new IllegalArgumentException("HiveDatasetVersionFinder is only compatible with partitioned hive tables. " + + "This is a snapshot hive table."); + } } try (AutoReturnableObject client = hiveDataset.getClientPool().getClient()) { diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/version/finder/LookbackDateTimeDatasetVersionFinder.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/version/finder/LookbackDateTimeDatasetVersionFinder.java new file mode 100644 index 00000000000..dd7f4f70033 --- /dev/null +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/version/finder/LookbackDateTimeDatasetVersionFinder.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.data.management.version.finder; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.joda.time.DateTime; +import org.joda.time.Duration; +import org.joda.time.Instant; +import org.joda.time.Period; +import org.joda.time.format.PeriodFormatter; +import org.joda.time.format.PeriodFormatterBuilder; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.typesafe.config.Config; + +import org.apache.gobblin.data.management.version.TimestampedDatasetVersion; +import org.apache.gobblin.dataset.Dataset; +import org.apache.gobblin.dataset.FileSystemDataset; +import org.apache.gobblin.util.ConfigUtils; + + +/** + * {@link DatasetVersionFinder} that constructs {@link TimestampedDatasetVersion}s without actually checking for existence + * of the version path. The version path is constructed by appending the version partition pattern to the dataset root. + * The versions are found by looking back a specific period of time and finding unique date partitions between that + * time and the current time. Lookback is supported to hourly granularity. + */ +public class LookbackDateTimeDatasetVersionFinder extends DateTimeDatasetVersionFinder { + public static final String VERSION_PATH_PREFIX = "version.path.prefix"; + public static final String VERSION_LOOKBACK_PERIOD = "version.lookback.period"; + + private final Duration stepDuration; + private final Period lookbackPeriod; + private final String pathPrefix; + private final Instant endTime; + + public LookbackDateTimeDatasetVersionFinder(FileSystem fs, Config config) { + this(fs, config, Instant.now()); + } + + @VisibleForTesting + public LookbackDateTimeDatasetVersionFinder(FileSystem fs, Config config, Instant endTime) { + super(fs, config); + Preconditions.checkArgument(config.hasPath(VERSION_LOOKBACK_PERIOD) , "Missing required property " + VERSION_LOOKBACK_PERIOD); + PeriodFormatter periodFormatter = + new PeriodFormatterBuilder().appendYears().appendSuffix("y").appendMonths().appendSuffix("M").appendDays() + .appendSuffix("d").appendHours().appendSuffix("h").toFormatter(); + this.stepDuration = Duration.standardHours(1); + this.pathPrefix = ConfigUtils.getString(config, VERSION_PATH_PREFIX, ""); + this.lookbackPeriod = periodFormatter.parsePeriod(config.getString(VERSION_LOOKBACK_PERIOD)); + this.endTime = endTime; + } + + @Override + public Collection findDatasetVersions(Dataset dataset) throws IOException { + FileSystemDataset fsDataset = (FileSystemDataset) dataset; + Set versions = new HashSet<>(); + Instant startTime = endTime.minus(lookbackPeriod.toStandardDuration()); + + for (Instant time = startTime; !time.isAfter(endTime); time = time.plus(stepDuration)) { + String truncatedTime = formatter.print(time); + DateTime versionTime = formatter.parseDateTime(truncatedTime); + Path versionPath = new Path(fsDataset.datasetRoot(), new Path(pathPrefix, truncatedTime)); + versions.add(new TimestampedDatasetVersion(versionTime, versionPath)); + } + + return versions; + } +} diff --git a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/version/finder/LookbackDateTimeDatasetVersionFinderTest.java b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/version/finder/LookbackDateTimeDatasetVersionFinderTest.java new file mode 100644 index 00000000000..6040ed32567 --- /dev/null +++ b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/version/finder/LookbackDateTimeDatasetVersionFinderTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gobblin.data.management.version.finder; + +import java.util.Collection; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.joda.time.DateTimeZone; +import org.joda.time.Instant; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.testng.Assert; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.data.management.version.TimestampedDatasetVersion; +import org.apache.gobblin.dataset.Dataset; +import org.apache.gobblin.dataset.FileSystemDataset; +import org.apache.gobblin.util.ConfigUtils; + + +@Test(groups = { "gobblin.data.management.version" }) +public class LookbackDateTimeDatasetVersionFinderTest { + + private FileSystem fs; + private DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy/MM/dd/HH").withZone(DateTimeZone.forID(ConfigurationKeys.PST_TIMEZONE_NAME)); + private final Instant fixedTime = Instant.parse("2023-01-01T12:30:00.000-08:00"); + + @Test + public void testHourlyVersions() throws Exception { + Properties properties = new Properties(); + properties.put(DateTimeDatasetVersionFinder.DATE_TIME_PATTERN_KEY, "yyyy/MM/dd/HH"); + properties.put(LookbackDateTimeDatasetVersionFinder.VERSION_PATH_PREFIX, "hourly"); + properties.put(LookbackDateTimeDatasetVersionFinder.VERSION_LOOKBACK_PERIOD, "96h"); + + LookbackDateTimeDatasetVersionFinder versionFinder = new LookbackDateTimeDatasetVersionFinder(FileSystem.getLocal(new Configuration()), + ConfigUtils.propertiesToConfig(properties), fixedTime); + Dataset dataset = new TestDataset(new Path("/data/Dataset1")); + Collection datasetVersions = versionFinder.findDatasetVersions(dataset); + List sortedVersions = datasetVersions.stream().sorted().collect(Collectors.toList()); + Assert.assertEquals(datasetVersions.size(), 97); + Assert.assertEquals(sortedVersions.get(0).getVersion().toString(formatter), "2022/12/28/12"); + Assert.assertEquals(sortedVersions.get(0).getPath().toString(), "/data/Dataset1/hourly/2022/12/28/12"); + Assert.assertEquals(sortedVersions.get(sortedVersions.size() - 1).getVersion().toString(formatter), "2023/01/01/12"); + Assert.assertEquals(sortedVersions.get(sortedVersions.size() - 1).getPath().toString(), "/data/Dataset1/hourly/2023/01/01/12"); + } + + @Test + public void testDailyVersions() throws Exception { + Properties properties = new Properties(); + properties.put(DateTimeDatasetVersionFinder.DATE_TIME_PATTERN_KEY, "yyyy/MM/dd"); + properties.put(LookbackDateTimeDatasetVersionFinder.VERSION_PATH_PREFIX, "daily"); + properties.put(LookbackDateTimeDatasetVersionFinder.VERSION_LOOKBACK_PERIOD, "366d"); + + LookbackDateTimeDatasetVersionFinder versionFinder = new LookbackDateTimeDatasetVersionFinder(FileSystem.getLocal(new Configuration()), + ConfigUtils.propertiesToConfig(properties), fixedTime); + Dataset dataset = new TestDataset(new Path("/data/Dataset1")); + Collection datasetVersions = versionFinder.findDatasetVersions(dataset); + List sortedVersions = datasetVersions.stream().sorted().collect(Collectors.toList()); + Assert.assertEquals(datasetVersions.size(), 367); + Assert.assertEquals(sortedVersions.get(0).getVersion().toString(formatter), "2021/12/31/00"); + Assert.assertEquals(sortedVersions.get(0).getPath().toString(), "/data/Dataset1/daily/2021/12/31"); + Assert.assertEquals(sortedVersions.get(sortedVersions.size() - 1).getVersion().toString(formatter), "2023/01/01/00"); + Assert.assertEquals(sortedVersions.get(sortedVersions.size() - 1).getPath().toString(), "/data/Dataset1/daily/2023/01/01"); + } +} + +class TestDataset implements FileSystemDataset { + private final Path datasetRoot; + + public TestDataset(Path datasetRoot) { + this.datasetRoot = datasetRoot; + } + + public Path datasetRoot() { + return datasetRoot; + } + + public String datasetURN() { + return null; + } +} diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/util/test/RetentionTestDataGenerator.java b/gobblin-data-management/src/test/java/org/apache/gobblin/util/test/RetentionTestDataGenerator.java similarity index 100% rename from gobblin-data-management/src/main/java/org/apache/gobblin/util/test/RetentionTestDataGenerator.java rename to gobblin-data-management/src/test/java/org/apache/gobblin/util/test/RetentionTestDataGenerator.java diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/util/test/RetentionTestHelper.java b/gobblin-data-management/src/test/java/org/apache/gobblin/util/test/RetentionTestHelper.java similarity index 100% rename from gobblin-data-management/src/main/java/org/apache/gobblin/util/test/RetentionTestHelper.java rename to gobblin-data-management/src/test/java/org/apache/gobblin/util/test/RetentionTestHelper.java diff --git a/gobblin-data-management/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/gobblin-data-management/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000000..1f0955d450f --- /dev/null +++ b/gobblin-data-management/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveRegistrationUnit.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveRegistrationUnit.java index adb024b64f6..8e06eda65f9 100644 --- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveRegistrationUnit.java +++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveRegistrationUnit.java @@ -30,13 +30,13 @@ import com.google.common.collect.Lists; import com.google.common.reflect.TypeToken; -import org.apache.gobblin.annotation.Alpha; -import org.apache.gobblin.configuration.State; - import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; +import org.apache.gobblin.annotation.Alpha; +import org.apache.gobblin.configuration.State; + /** * A class that represents a Hive table or partition. @@ -125,14 +125,13 @@ protected void populateSerDeFields(State state) { protected static Optional populateField(State state, String key, TypeToken token) { if (state.contains(key)) { Optional fieldValue; - - if (new TypeToken() {}.isAssignableFrom(token)) { + if (new TypeToken(){}.isSupertypeOf(token)) { fieldValue = (Optional) Optional.of(state.getPropAsBoolean(key)); - } else if (new TypeToken() {}.isAssignableFrom(token)) { + } else if (new TypeToken(){}.isSupertypeOf(token)) { fieldValue = (Optional) Optional.of(state.getPropAsInt(key)); - } else if (new TypeToken() {}.isAssignableFrom(token)) { + } else if (new TypeToken(){}.isSupertypeOf(token)) { fieldValue = (Optional) Optional.of(state.getPropAsLong(key)); - } else if (new TypeToken>() {}.isAssignableFrom(token)) { + } else if (new TypeToken>(){}.isSupertypeOf(token)) { fieldValue = (Optional) Optional.of(state.getPropAsList(key)); } else { fieldValue = (Optional) Optional.of(state.getProp(key)); diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveSerDeWrapper.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveSerDeWrapper.java index fea2e99d1dd..f01faa36fab 100644 --- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveSerDeWrapper.java +++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveSerDeWrapper.java @@ -19,6 +19,7 @@ import java.io.IOException; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedSerde; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat; import org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat; @@ -28,6 +29,7 @@ import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; +import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.avro.AvroSerDe; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; @@ -39,6 +41,7 @@ import org.apache.gobblin.annotation.Alpha; import org.apache.gobblin.configuration.State; +import org.apache.gobblin.util.Either; /** @@ -88,7 +91,7 @@ public String toString() { } } - private Optional serDe = Optional.absent(); + private Optional> serDe = Optional.absent(); private final String serDeClassName; private final String inputFormatClassName; private final String outputFormatClassName; @@ -107,15 +110,21 @@ private HiveSerDeWrapper(String serDeClassName, String inputFormatClassName, Str * Get the {@link SerDe} instance associated with this {@link HiveSerDeWrapper}. * This method performs lazy initialization. */ - public SerDe getSerDe() throws IOException { + public Object getSerDe() throws IOException { if (!this.serDe.isPresent()) { try { - this.serDe = Optional.of(SerDe.class.cast(Class.forName(this.serDeClassName).newInstance())); + Object serde = Class.forName(this.serDeClassName).newInstance(); + if (serde instanceof OrcSerde) { + this.serDe = Optional.of(Either.right(VectorizedSerde.class.cast(serde))); + } else { + this.serDe = Optional.of(Either.left(AbstractSerDe.class.cast(serde))); + } } catch (Throwable t) { throw new IOException("Failed to instantiate SerDe " + this.serDeClassName, t); } } - return this.serDe.get(); + + return this.serDe.get().get(); } /** diff --git a/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/HiveTableTest.java b/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/HiveTableTest.java new file mode 100644 index 00000000000..68b9bad9c00 --- /dev/null +++ b/gobblin-hive-registration/src/test/java/org/apache/gobblin/hive/HiveTableTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.hive; + +import java.util.ArrayList; +import java.util.List; + +import org.testng.annotations.Test; + +import junit.framework.Assert; + +import org.apache.gobblin.configuration.State; + +public class HiveTableTest { + + @Test + public void testPopulateFieldTypeCasting() throws Exception { + // Test one property of each type in HiveRegistrationUnit storageProps + + State props = new State(); + Long lastAccessTime = System.currentTimeMillis(); + props.setProp(HiveConstants.LAST_ACCESS_TIME, String.valueOf(lastAccessTime)); + State storageProps = new State(); + storageProps.setProp(HiveConstants.LOCATION, "/tmp"); + storageProps.setProp(HiveConstants.COMPRESSED, "false"); + storageProps.setProp(HiveConstants.NUM_BUCKETS, "1"); + storageProps.setProp(HiveConstants.BUCKET_COLUMNS, "col1, col2"); + HiveTable.Builder builder = new HiveTable.Builder(); + builder.withTableName("tableName"); + builder.withDbName("dbName"); + builder.withProps(props); + builder.withStorageProps(storageProps); + + HiveTable hiveTable = builder.build(); + + Assert.assertEquals(hiveTable.getLastAccessTime().get().longValue(), lastAccessTime.longValue()); + Assert.assertEquals(hiveTable.getLocation().get(), "/tmp"); + Assert.assertEquals(hiveTable.isCompressed.get().booleanValue(), false); + Assert.assertEquals(hiveTable.getNumBuckets().get().intValue(), 1); + List bucketColumns = new ArrayList<>(); + bucketColumns.add("col1"); + bucketColumns.add("col2"); + Assert.assertEquals(hiveTable.getBucketColumns().get().get(0), bucketColumns.get(0)); + Assert.assertEquals(hiveTable.getBucketColumns().get().get(1), bucketColumns.get(1)); + } +} diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/GobblinMCEProducer.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/GobblinMCEProducer.java index b037cb6354f..0f0061ed29f 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/GobblinMCEProducer.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/GobblinMCEProducer.java @@ -46,6 +46,7 @@ import org.apache.gobblin.metadata.OperationType; import org.apache.gobblin.metadata.SchemaSource; import org.apache.gobblin.metrics.MetricContext; +import org.apache.gobblin.source.extractor.extract.kafka.KafkaSource; import org.apache.gobblin.util.ClustersNames; import org.apache.gobblin.util.reflection.GobblinConstructorUtils; import org.apache.gobblin.writer.PartitionedDataWriter; @@ -149,6 +150,14 @@ private void setBasicInformationForGMCE(GobblinMetadataChangeEvent.Builder gmceB regProperties.put(HiveRegistrationPolicyBase.HIVE_DATABASE_NAME, state.getProp(HiveRegistrationPolicyBase.HIVE_DATABASE_NAME)); } + if (state.contains(HiveRegistrationPolicyBase.HIVE_TABLE_NAME)) { + regProperties.put(HiveRegistrationPolicyBase.HIVE_TABLE_NAME, + state.getProp(HiveRegistrationPolicyBase.HIVE_TABLE_NAME)); + } + if (state.contains(KafkaSource.TOPIC_NAME)) { + regProperties.put(KafkaSource.TOPIC_NAME, + state.getProp(KafkaSource.TOPIC_NAME)); + } if (state.contains(HiveRegistrationPolicyBase.ADDITIONAL_HIVE_DATABASE_NAMES)) { regProperties.put(HiveRegistrationPolicyBase.ADDITIONAL_HIVE_DATABASE_NAMES, state.getProp(HiveRegistrationPolicyBase.ADDITIONAL_HIVE_DATABASE_NAMES)); diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java index 9c2c4fdf168..1e88a83c7f4 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java @@ -48,12 +48,14 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.specific.SpecificData; +import org.apache.commons.lang3.tuple.Pair; import org.apache.gobblin.hive.writer.MetadataWriterKeys; import org.apache.gobblin.source.extractor.extract.LongWatermark; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils; import org.apache.iceberg.AppendFiles; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFiles; @@ -68,12 +70,12 @@ import org.apache.iceberg.Transaction; import org.apache.iceberg.UpdateProperties; import org.apache.iceberg.avro.AvroSchemaUtil; +import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.hive.HiveCatalogs; import org.apache.iceberg.types.Types; import org.joda.time.DateTime; @@ -186,9 +188,9 @@ public class IcebergMetadataWriter implements MetadataWriter { private final Map tableTopicPartitionMap; @Getter private final KafkaSchemaRegistry schemaRegistry; - private final Map tableMetadataMap; + protected final Map tableMetadataMap; @Setter - protected HiveCatalog catalog; + protected Catalog catalog; protected final Configuration conf; protected final ReadWriteLock readWriteLock; private final HiveLock locks; @@ -330,7 +332,7 @@ public void write(GobblinMetadataChangeEvent gmce, Map new TableMetadata()); tableMetadata.newProperties = Optional.of(IcebergUtils.getTableProperties(table)); @@ -449,7 +451,7 @@ private void computeCandidateSchema(GobblinMetadataChangeEvent gmce, TableIdenti .expireAfterAccess(conf.getInt(MetadataWriter.CACHE_EXPIRING_TIME, MetadataWriter.DEFAULT_CACHE_EXPIRING_TIME), TimeUnit.HOURS) .build())); - Cache candidate = tableMetadata.candidateSchemas.get(); + Cache> candidate = tableMetadata.candidateSchemas.get(); try { switch (gmce.getSchemaSource()) { case SCHEMAREGISTRY: { @@ -457,15 +459,15 @@ private void computeCandidateSchema(GobblinMetadataChangeEvent gmce, TableIdenti String createdOn = AvroUtils.getSchemaCreationTime(schema); if (createdOn == null) { candidate.put(DEFAULT_CREATION_TIME, - IcebergUtils.getIcebergSchema(gmce.getTableSchema(), hiveTable).tableSchema); + Pair.of(IcebergUtils.getIcebergSchema(gmce.getTableSchema(), hiveTable).tableSchema, gmce.getTableSchema())); } else if (!createdOn.equals(lastSchemaVersion)) { - candidate.put(createdOn, IcebergUtils.getIcebergSchema(gmce.getTableSchema(), hiveTable).tableSchema); + candidate.put(createdOn, Pair.of(IcebergUtils.getIcebergSchema(gmce.getTableSchema(), hiveTable).tableSchema, gmce.getTableSchema())); } break; } case EVENT: { candidate.put(DEFAULT_CREATION_TIME, - IcebergUtils.getIcebergSchema(gmce.getTableSchema(), hiveTable).tableSchema); + Pair.of(IcebergUtils.getIcebergSchema(gmce.getTableSchema(), hiveTable).tableSchema, gmce.getTableSchema())); break; } case NONE: { @@ -780,6 +782,21 @@ private StructLike getIcebergPartitionVal(Collection specs, String fil return partitionVal; } + /** + * We will firstly try to use datasetOffsetRange to get the topic name, as the pattern for datasetOffsetRange key should be ({topicName}-{partitionNumber}) + * In case there is no datasetOffsetRange, we fall back to the table property that we set previously for "topic.name" + * @return kafka topic name for this table + */ + protected String getTopicName(TableIdentifier tid, TableMetadata tableMetadata) { + if (tableMetadata.dataOffsetRange.isPresent()) { + String topicPartitionString = tableMetadata.dataOffsetRange.get().keySet().iterator().next(); + //In case the topic name is not the table name or the topic name contains '-' + return topicPartitionString.substring(0, topicPartitionString.lastIndexOf('-')); + } + return tableMetadata.newProperties.or( + Maps.newHashMap(tableMetadata.lastProperties.or(getIcebergTable(tid).properties()))).get(TOPIC_NAME_KEY); + } + /** * For flush of each table, we do the following logic: * 1. Commit the appendFiles if it exist @@ -801,12 +818,14 @@ public void flush(String dbName, String tableName) throws IOException { Transaction transaction = tableMetadata.transaction.get(); Map props = tableMetadata.newProperties.or( Maps.newHashMap(tableMetadata.lastProperties.or(getIcebergTable(tid).properties()))); - String topic = props.get(TOPIC_NAME_KEY); + //Set data offset range + setDatasetOffsetRange(tableMetadata, props); + String topicName = getTopicName(tid, tableMetadata); if (tableMetadata.appendFiles.isPresent()) { tableMetadata.appendFiles.get().commit(); - sendAuditCounts(topic, tableMetadata.serializedAuditCountMaps); + sendAuditCounts(topicName, tableMetadata.serializedAuditCountMaps); if (tableMetadata.completenessEnabled) { - checkAndUpdateCompletenessWatermark(tableMetadata, topic, tableMetadata.datePartitions, props); + checkAndUpdateCompletenessWatermark(tableMetadata, topicName, tableMetadata.datePartitions, props); } } if (tableMetadata.deleteFiles.isPresent()) { @@ -817,15 +836,15 @@ public void flush(String dbName, String tableName) throws IOException { if(!tableMetadata.appendFiles.isPresent() && !tableMetadata.deleteFiles.isPresent() && tableMetadata.completenessEnabled) { if (tableMetadata.completionWatermark > DEFAULT_COMPLETION_WATERMARK) { - log.info(String.format("Checking kafka audit for %s on change_property ", topic)); + log.info(String.format("Checking kafka audit for %s on change_property ", topicName)); SortedSet timestamps = new TreeSet<>(); ZonedDateTime prevWatermarkDT = Instant.ofEpochMilli(tableMetadata.completionWatermark).atZone(ZoneId.of(this.timeZone)); timestamps.add(TimeIterator.inc(prevWatermarkDT, TimeIterator.Granularity.valueOf(this.auditCheckGranularity), 1)); - checkAndUpdateCompletenessWatermark(tableMetadata, topic, timestamps, props); + checkAndUpdateCompletenessWatermark(tableMetadata, topicName, timestamps, props); } else { log.info(String.format("Need valid watermark, current watermark is %s, Not checking kafka audit for %s", - tableMetadata.completionWatermark, topic)); + tableMetadata.completionWatermark, topicName)); } } @@ -836,19 +855,11 @@ public void flush(String dbName, String tableName) throws IOException { props.put(String.format(GMCE_LOW_WATERMARK_KEY, tableTopicPartitionMap.get(tid)), tableMetadata.lowWatermark.get().toString()); //Set whether to delete metadata files after commit - props.put(TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED, Boolean.toString( - conf.getBoolean(TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED, - TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT))); - props.put(TableProperties.METADATA_PREVIOUS_VERSIONS_MAX, Integer.toString( - conf.getInt(TableProperties.METADATA_PREVIOUS_VERSIONS_MAX, - TableProperties.METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT))); - //Set data offset range - boolean containOffsetRange = setDatasetOffsetRange(tableMetadata, props); - String topicName = tableName; - if (containOffsetRange) { - String topicPartitionString = tableMetadata.dataOffsetRange.get().keySet().iterator().next(); - //In case the topic name is not the table name or the topic name contains '-' - topicName = topicPartitionString.substring(0, topicPartitionString.lastIndexOf('-')); + if (conf.getBoolean(ICEBERG_ENABLE_CUSTOM_METADATA_RETENTION_POLICY, DEFAULT_ICEBERG_ENABLE_CUSTOM_METADATA_RETENTION_POLICY)) { + props.put(TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED, Boolean.toString( + conf.getBoolean(TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED, TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT))); + props.put(TableProperties.METADATA_PREVIOUS_VERSIONS_MAX, Integer.toString( + conf.getInt(TableProperties.METADATA_PREVIOUS_VERSIONS_MAX, TableProperties.METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT))); } //Update schema(commit) updateSchema(tableMetadata, props, topicName); @@ -882,7 +893,7 @@ public void flush(String dbName, String tableName) throws IOException { @Override public void reset(String dbName, String tableName) throws IOException { - this.tableMetadataMap.remove(TableIdentifier.of(dbName, tableName)); + this.tableMetadataMap.remove(TableIdentifier.of(dbName, tableName)); } /** @@ -952,7 +963,7 @@ private long computeCompletenessWatermark(String catalogDbTableName, String topi long timestampMillis = timestampDT.toInstant().toEpochMilli(); ZonedDateTime auditCountCheckLowerBoundDT = TimeIterator.dec(timestampDT, granularity, 1); if (auditCountVerifier.get().isComplete(topicName, - auditCountCheckLowerBoundDT.toInstant().toEpochMilli(), timestampMillis)) { + auditCountCheckLowerBoundDT.toInstant().toEpochMilli(), timestampMillis)) { completionWatermark = timestampMillis; // Also persist the watermark into State object to share this with other MetadataWriters // we enforce ourselves to always use lower-cased table name here @@ -1026,7 +1037,8 @@ private void updateSchema(TableMetadata tableMetadata, Map props Cache candidates = tableMetadata.candidateSchemas.get(); //Only have default schema, so either we calculate schema from event or the schema does not have creation time, directly update it if (candidates.size() == 1 && candidates.getIfPresent(DEFAULT_CREATION_TIME) != null) { - updateSchemaHelper(DEFAULT_CREATION_TIME, (Schema) candidates.getIfPresent(DEFAULT_CREATION_TIME), props, + updateSchemaHelper(DEFAULT_CREATION_TIME, + (Pair) candidates.getIfPresent(DEFAULT_CREATION_TIME), props, tableMetadata.table.get()); } else { //update schema if candidates contains the schema that has the same creation time with the latest schema @@ -1037,7 +1049,7 @@ private void updateSchema(TableMetadata tableMetadata, Map props log.warn( "Schema from schema registry does not contain creation time, check config for schema registry class"); } else if (candidates.getIfPresent(creationTime) != null) { - updateSchemaHelper(creationTime, (Schema) candidates.getIfPresent(creationTime), props, + updateSchemaHelper(creationTime, (Pair) candidates.getIfPresent(creationTime), props, tableMetadata.table.get()); } } @@ -1047,10 +1059,11 @@ private void updateSchema(TableMetadata tableMetadata, Map props } } - private void updateSchemaHelper(String schemaCreationTime, Schema schema, Map props, Table table) { + private void updateSchemaHelper(String schemaCreationTime, Pair schema, Map props, Table table) { try { - table.updateSchema().unionByNameWith(schema).commit(); + table.updateSchema().unionByNameWith(schema.getLeft()).commit(); props.put(SCHEMA_CREATION_TIME_KEY, schemaCreationTime); + props.put(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName(), schema.getRight()); } catch (Exception e) { log.error("Cannot update schema to " + schema.toString() + "for table " + table.location(), e); } @@ -1122,7 +1135,7 @@ public void close() throws IOException { * * Also note the difference with {@link org.apache.iceberg.TableMetadata}. */ - private class TableMetadata { + public class TableMetadata { Optional table = Optional.absent(); /** @@ -1133,10 +1146,10 @@ private class TableMetadata { Optional transaction = Optional.absent(); private Optional appendFiles = Optional.absent(); private Optional deleteFiles = Optional.absent(); + public Optional> newProperties = Optional.absent(); Optional> lastProperties = Optional.absent(); - Optional> newProperties = Optional.absent(); - Optional> candidateSchemas = Optional.absent(); + Optional>> candidateSchemas = Optional.absent(); Optional>> dataOffsetRange = Optional.absent(); Optional lastSchemaVersion = Optional.absent(); Optional lowWatermark = Optional.absent(); diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterConfigKeys.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterConfigKeys.java index cd51ca3c609..73c206d5f3d 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterConfigKeys.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterConfigKeys.java @@ -41,6 +41,8 @@ public class IcebergMetadataWriterConfigKeys { public static final String ICEBERG_NEW_PARTITION_WHITELIST = "iceberg.new.partition.whitelist"; public static final String ICEBERG_NEW_PARTITION_BLACKLIST = "iceberg.new.partition.blacklist"; public static final String STATE_COMPLETION_WATERMARK_KEY_OF_TABLE = "completion.watermark.%s"; + public static final String ICEBERG_ENABLE_CUSTOM_METADATA_RETENTION_POLICY = "iceberg.enable.custom.metadata.retention.policy"; + public static final boolean DEFAULT_ICEBERG_ENABLE_CUSTOM_METADATA_RETENTION_POLICY = true; } diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java index b6174ec7dba..294ef08ab95 100644 --- a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java +++ b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java @@ -124,11 +124,11 @@ public void setUp() throws Exception { startMetastore(); tmpDir = Files.createTempDir(); - hourlyDataFile_1 = new File(tmpDir, "testDB/testIcebergTable/hourly/2020/03/17/08/data.avro"); + hourlyDataFile_1 = new File(tmpDir, "testDB/testTopic/hourly/2020/03/17/08/data.avro"); Files.createParentDirs(hourlyDataFile_1); - hourlyDataFile_2 = new File(tmpDir, "testDB/testIcebergTable/hourly/2020/03/17/09/data.avro"); + hourlyDataFile_2 = new File(tmpDir, "testDB/testTopic/hourly/2020/03/17/09/data.avro"); Files.createParentDirs(hourlyDataFile_2); - dailyDataFile = new File(tmpDir, "testDB/testIcebergTable/daily/2020/03/17/data.avro"); + dailyDataFile = new File(tmpDir, "testDB/testTopic/daily/2020/03/17/data.avro"); Files.createParentDirs(dailyDataFile); dataDir = new File(hourlyDataFile_1.getParent()); Assert.assertTrue(dataDir.exists()); @@ -139,7 +139,7 @@ public void setUp() throws Exception { .setDatasetIdentifier(DatasetIdentifier.newBuilder() .setDataOrigin(DataOrigin.EI) .setDataPlatformUrn("urn:namespace:dataPlatform:hdfs") - .setNativeName(new File(tmpDir, "testDB/testIcebergTable").getAbsolutePath()) + .setNativeName(new File(tmpDir, "testDB/testTopic").getAbsolutePath()) .build()) .setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopic-1", "0-1000").build()) .setFlowId("testFlow") @@ -221,7 +221,7 @@ public void testWriteAddFileGMCE() throws IOException { Table table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); Assert.assertFalse(table.properties().containsKey("offset.range.testTopic-1")); Assert.assertEquals(table.location(), - new File(tmpDir, "testDB/testIcebergTable/_iceberg_metadata/").getAbsolutePath() + "/" + dbName); + new File(tmpDir, "testDB/testTopic/_iceberg_metadata/").getAbsolutePath() + "/" + dbName); gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopic-1", "1000-2000").build()); GenericRecord genericGmce_1000_2000 = GenericData.get().deepCopy(gmce.getSchema(), gmce); @@ -363,11 +363,11 @@ public void testFaultTolerant() throws Exception { Assert.assertEquals(gobblinMCEWriter.getDatasetErrorMap().size(), 1); Assert.assertEquals(gobblinMCEWriter.getDatasetErrorMap().values().iterator().next().size(), 1); Assert.assertEquals(gobblinMCEWriter.getDatasetErrorMap() - .get(new File(tmpDir, "testDB/testIcebergTable").getAbsolutePath()) - .get("hivedb.testIcebergTable").get(0).lowWatermark, 50L); + .get(new File(tmpDir, "testDB/testTopic").getAbsolutePath()) + .get("hivedb.testTopic").get(0).lowWatermark, 50L); Assert.assertEquals(gobblinMCEWriter.getDatasetErrorMap() - .get(new File(tmpDir, "testDB/testIcebergTable").getAbsolutePath()) - .get("hivedb.testIcebergTable").get(0).highWatermark, 52L); + .get(new File(tmpDir, "testDB/testTopic").getAbsolutePath()) + .get("hivedb.testTopic").get(0).highWatermark, 52L); // No events sent yet since the topic has not been flushed Assert.assertEquals(eventsSent.size(), 0); @@ -378,7 +378,7 @@ public void testFaultTolerant() throws Exception { // Since this topic has been flushed, there should be an event sent for previous failure, and the table // should be removed from the error map Assert.assertEquals(eventsSent.size(), 1); - Assert.assertEquals(eventsSent.get(0).getMetadata().get(MetadataWriterKeys.TABLE_NAME_KEY), "testIcebergTable"); + Assert.assertEquals(eventsSent.get(0).getMetadata().get(MetadataWriterKeys.TABLE_NAME_KEY), "testTopic"); Assert.assertEquals(eventsSent.get(0).getMetadata().get(MetadataWriterKeys.GMCE_LOW_WATERMARK), "50"); Assert.assertEquals(eventsSent.get(0).getMetadata().get(MetadataWriterKeys.GMCE_HIGH_WATERMARK), "52"); Assert.assertEquals(gobblinMCEWriter.getDatasetErrorMap().values().iterator().next().size(), 0); @@ -398,7 +398,7 @@ public void testWriteAddFileGMCECompleteness() throws IOException { // Creating a copy of gmce with static type in GenericRecord to work with writeEnvelop method // without risking running into type cast runtime error. gmce.setOperationType(OperationType.add_files); - File hourlyFile = new File(tmpDir, "testDB/testIcebergTable/hourly/2021/09/16/10/data.avro"); + File hourlyFile = new File(tmpDir, "testDB/testTopic/hourly/2021/09/16/10/data.avro"); long timestampMillis = 1631811600000L; Files.createParentDirs(hourlyFile); writeRecord(hourlyFile); @@ -421,13 +421,13 @@ public void testWriteAddFileGMCECompleteness() throws IOException { // Test when completeness watermark = -1 bootstrap case KafkaAuditCountVerifier verifier = Mockito.mock(TestAuditCountVerifier.class); - Mockito.when(verifier.isComplete("testIcebergTable", timestampMillis - TimeUnit.HOURS.toMillis(1), timestampMillis)).thenReturn(true); + Mockito.when(verifier.isComplete("testTopic", timestampMillis - TimeUnit.HOURS.toMillis(1), timestampMillis)).thenReturn(true); IcebergMetadataWriter imw = (IcebergMetadataWriter) gobblinMCEWriterWithCompletness.metadataWriters.iterator().next(); imw.setAuditCountVerifier(verifier); gobblinMCEWriterWithCompletness.flush(); table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); //completeness watermark = "2020-09-16-10" - Assert.assertEquals(table.properties().get(TOPIC_NAME_KEY), "testIcebergTable"); + Assert.assertEquals(table.properties().get(TOPIC_NAME_KEY), "testTopic"); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_TIMEZONE_KEY), "America/Los_Angeles"); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_KEY), String.valueOf(timestampMillis)); // 1631811600000L correspond to 2020-09-16-10 in PT @@ -437,7 +437,7 @@ public void testWriteAddFileGMCECompleteness() throws IOException { Assert.assertTrue(dfl.hasNext()); // Test when completeness watermark is still "2021-09-16-10" but have a late file for "2021-09-16-09" - File hourlyFile1 = new File(tmpDir, "testDB/testIcebergTable/hourly/2021/09/16/09/data1.avro"); + File hourlyFile1 = new File(tmpDir, "testDB/testTopic/hourly/2021/09/16/09/data1.avro"); Files.createParentDirs(hourlyFile1); writeRecord(hourlyFile1); gmce.setNewFiles(Lists.newArrayList(DataFile.newBuilder() @@ -460,7 +460,7 @@ public void testWriteAddFileGMCECompleteness() throws IOException { Assert.assertEquals((int) dfl.next().partition().get(1, Integer.class), 1); // Test when completeness watermark will advance to "2021-09-16-11" - File hourlyFile2 = new File(tmpDir, "testDB/testIcebergTable/hourly/2021/09/16/11/data.avro"); + File hourlyFile2 = new File(tmpDir, "testDB/testTopic/hourly/2021/09/16/11/data.avro"); long timestampMillis1 = timestampMillis + TimeUnit.HOURS.toMillis(1); Files.createParentDirs(hourlyFile2); writeRecord(hourlyFile2); @@ -476,7 +476,7 @@ public void testWriteAddFileGMCECompleteness() throws IOException { new KafkaPartition.Builder().withTopicName("GobblinMetadataChangeEvent_test").withId(1).build(), new LongWatermark(60L)))); - Mockito.when(verifier.isComplete("testIcebergTable", timestampMillis1 - TimeUnit.HOURS.toMillis(1), timestampMillis1)).thenReturn(true); + Mockito.when(verifier.isComplete("testTopic", timestampMillis1 - TimeUnit.HOURS.toMillis(1), timestampMillis1)).thenReturn(true); gobblinMCEWriterWithCompletness.flush(); table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_KEY), String.valueOf(timestampMillis1)); @@ -495,7 +495,7 @@ public void testChangePropertyGMCECompleteness() throws IOException { Table table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); long watermark = Long.parseLong(table.properties().get(COMPLETION_WATERMARK_KEY)); long expectedWatermark = watermark + TimeUnit.HOURS.toMillis(1); - File hourlyFile2 = new File(tmpDir, "testDB/testIcebergTable/hourly/2021/09/16/11/data.avro"); + File hourlyFile2 = new File(tmpDir, "testDB/testTopic/hourly/2021/09/16/11/data.avro"); gmce.setOldFilePrefixes(null); gmce.setNewFiles(Lists.newArrayList(DataFile.newBuilder() .setFilePath(hourlyFile2.toString()) @@ -511,14 +511,14 @@ public void testChangePropertyGMCECompleteness() throws IOException { new LongWatermark(65L)))); KafkaAuditCountVerifier verifier = Mockito.mock(TestAuditCountVerifier.class); - Mockito.when(verifier.isComplete("testIcebergTable", watermark, expectedWatermark)).thenReturn(true); + Mockito.when(verifier.isComplete("testTopic", watermark, expectedWatermark)).thenReturn(true); ((IcebergMetadataWriter) gobblinMCEWriterWithCompletness.metadataWriters.iterator().next()).setAuditCountVerifier(verifier); gobblinMCEWriterWithCompletness.flush(); table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); Assert.assertEquals(table.properties().get("offset.range.testTopic-1"), "0-7000"); Assert.assertEquals(table.spec().fields().get(1).name(), "late"); - Assert.assertEquals(table.properties().get(TOPIC_NAME_KEY), "testIcebergTable"); + Assert.assertEquals(table.properties().get(TOPIC_NAME_KEY), "testTopic"); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_TIMEZONE_KEY), "America/Los_Angeles"); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_KEY), String.valueOf(expectedWatermark)); @@ -558,7 +558,7 @@ protected Optional getPartition(Path path, HiveTable table) throw partitionValue = "2020-03-17-00"; } return Optional.of(new HivePartition.Builder().withPartitionValues(Lists.newArrayList(partitionValue)) - .withDbName("hivedb").withTableName("testIcebergTable").build()); + .withDbName("hivedb").withTableName("testTopic").build()); } @Override protected List getTables(Path path) throws IOException { @@ -577,7 +577,7 @@ protected List getTableNames(Optional dbPrefix, Path path) { if (path.toString().contains("testFaultTolerant")) { return Lists.newArrayList("testFaultTolerantIcebergTable"); } - return Lists.newArrayList("testIcebergTable"); + return Lists.newArrayList("testTopic"); } } diff --git a/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/GaaSObservabilityEventExperimental.avsc b/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/GaaSObservabilityEventExperimental.avsc index 5de27a7c463..887d322f7d2 100644 --- a/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/GaaSObservabilityEventExperimental.avsc +++ b/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/GaaSObservabilityEventExperimental.avsc @@ -130,6 +130,24 @@ "doc": "The ID of the spec executor that ran or would have ran the job", "compliance": "NONE" }, + { + "name": "gaasId", + "type": [ + "null", + "string" + ], + "default": null, + "doc": "The instance of GaaS that is sending the event (if multiple GaaS instances are running)" + }, + { + "name":"jobProperties", + "type": [ + "null", + "string" + ], + "default": null, + "doc": "The job properties GaaS sends to the job executor. This is a JSON string of the job properties" + }, { "name": "issues", "type": [ @@ -188,6 +206,44 @@ } } ] - }] + }, + { + "name": "datasetsWritten", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "DatasetMetric", + "doc": "DatasetMetric contains bytes and records written by Gobblin writers for the dataset URN.", + "fields": [ + { + "name": "datasetUrn", + "type": "string", + "doc": "URN of the dataset" + }, + { + "name": "bytesWritten", + "type": "long", + "doc": "Number of bytes written for the dataset, can be -1 if unsupported by the writer (e.g. jdbc writer)" + }, + { + "name": "entitiesWritten", + "type": "long", + "doc": "Number of entities written (e.g. files or records) for the dataset by the Gobblin writer" + }, + { + "name": "successfullyCommitted", + "type": "boolean", + "doc": "Whether the dataset was successfully committed by Gobblin and fully successful, useful when users configure pipelines to allow for partial failures or non-atomic writes" + } + ] + } + } + ], + "default": null + } + ] } diff --git a/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/event/TimingEvent.java b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/event/TimingEvent.java index 39bf7de11ff..ff791221ab4 100644 --- a/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/event/TimingEvent.java +++ b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/event/TimingEvent.java @@ -46,6 +46,7 @@ public static class LauncherTimings { public static final String JOB_START = "JobStartTimer"; public static final String JOB_RUN = "JobRunTimer"; public static final String JOB_COMMIT = "JobCommitTimer"; + public static final String JOB_SUMMARY = "JobSummaryTimer"; public static final String JOB_CLEANUP = "JobCleanupTimer"; public static final String JOB_CANCEL = "JobCancelTimer"; public static final String JOB_COMPLETE = "JobCompleteTimer"; @@ -112,6 +113,7 @@ public static class FlowEventConstants { public static final String JOB_END_TIME = "jobEndTime"; public static final String JOB_LAST_PROGRESS_EVENT_TIME = "jobLastProgressEventTime"; public static final String JOB_COMPLETION_PERCENTAGE = "jobCompletionPercentage"; + public static final String DATASET_TASK_SUMMARIES = "datasetTaskSummaries"; @Getter private Long startTime; diff --git a/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/test/MetricsAssert.java b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/test/MetricsAssert.java index 7cf17dc1e8b..7594f425148 100644 --- a/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/test/MetricsAssert.java +++ b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/test/MetricsAssert.java @@ -20,10 +20,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import javax.annotation.Nonnull; - import com.google.common.base.Function; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; + +import javax.annotation.Nonnull; import org.apache.gobblin.metrics.GobblinTrackingEvent; import org.apache.gobblin.metrics.MetricContext; @@ -106,4 +107,7 @@ public static Predicate hasEventMetdata(final String metad }; } + public ImmutableList getEvents() { + return ImmutableList.copyOf(_events); + } } diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/azkaban/AzkabanGobblinYarnAppLauncher.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/azkaban/AzkabanGobblinYarnAppLauncher.java index 52780927b9e..8fe09e5a527 100644 --- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/azkaban/AzkabanGobblinYarnAppLauncher.java +++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/azkaban/AzkabanGobblinYarnAppLauncher.java @@ -58,7 +58,7 @@ public class AzkabanGobblinYarnAppLauncher extends AbstractJob { private final GobblinYarnAppLauncher gobblinYarnAppLauncher; @Getter - private final YarnConfiguration yarnConfiguration; + protected final YarnConfiguration yarnConfiguration; public AzkabanGobblinYarnAppLauncher(String jobId, Properties gobblinProps) throws IOException { @@ -75,7 +75,14 @@ public AzkabanGobblinYarnAppLauncher(String jobId, Properties gobblinProps) gobblinConfig = gobblinConfig.withValue(GobblinYarnAppLauncher.GOBBLIN_YARN_APP_LAUNCHER_MODE, ConfigValueFactory.fromAnyRef(GobblinYarnAppLauncher.AZKABAN_APP_LAUNCHER_MODE_KEY)); - this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(gobblinConfig, this.yarnConfiguration); + this.gobblinYarnAppLauncher = getYarnAppLauncher(gobblinConfig); + } + + protected GobblinYarnAppLauncher getYarnAppLauncher(Config gobblinConfig) + throws IOException { + GobblinYarnAppLauncher gobblinYarnAppLauncher = new GobblinYarnAppLauncher(gobblinConfig, this.yarnConfiguration); + gobblinYarnAppLauncher.initializeYarnClients(gobblinConfig); + return gobblinYarnAppLauncher; } /** diff --git a/gobblin-modules/gobblin-compliance/build.gradle b/gobblin-modules/gobblin-compliance/build.gradle index 706d7e0601e..d6cf3e13e9f 100644 --- a/gobblin-modules/gobblin-compliance/build.gradle +++ b/gobblin-modules/gobblin-compliance/build.gradle @@ -21,6 +21,8 @@ dependencies { compile project(":gobblin-data-management") compile externalDependency.azkaban compile externalDependency.hiveJdbc + + testCompile externalDependency.mockito } test { diff --git a/gobblin-modules/gobblin-elasticsearch-deps/build.gradle b/gobblin-modules/gobblin-elasticsearch-deps/build.gradle index 8b8c04a93bd..a763c1d49da 100644 --- a/gobblin-modules/gobblin-elasticsearch-deps/build.gradle +++ b/gobblin-modules/gobblin-elasticsearch-deps/build.gradle @@ -33,7 +33,7 @@ tasks.remove(tasks.uploadShadow) dependencies { compile "org.elasticsearch.client:transport:5.6.8" compile "org.elasticsearch.client:elasticsearch-rest-high-level-client:5.6.8" - compile "com.google.guava:guava:18.0" + compile externalDependency.guava } diff --git a/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/runtime/KafkaAvroJobStatusMonitorTest.java b/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/runtime/KafkaAvroJobStatusMonitorTest.java index 5c78989a580..e63969b860f 100644 --- a/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/runtime/KafkaAvroJobStatusMonitorTest.java +++ b/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/runtime/KafkaAvroJobStatusMonitorTest.java @@ -689,7 +689,7 @@ private GobblinTrackingEvent createWorkUnitTimingEvent() { Map metadata = Maps.newHashMap(); metadata.put(TimingEvent.METADATA_START_TIME, "2"); metadata.put(TimingEvent.METADATA_END_TIME, "3"); - return createGTE(TimingEvent.RunJobTimings.WORK_UNITS_PREPARATION, metadata); + return createGTE(TimingEvent.LauncherTimings.WORK_UNITS_CREATION, metadata); } diff --git a/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/service/StreamingKafkaSpecExecutorTest.java b/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/service/StreamingKafkaSpecExecutorTest.java index abbdc610ca1..bf7cf071bc9 100644 --- a/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/service/StreamingKafkaSpecExecutorTest.java +++ b/gobblin-modules/gobblin-kafka-09/src/test/java/org/apache/gobblin/service/StreamingKafkaSpecExecutorTest.java @@ -29,25 +29,26 @@ import org.apache.commons.lang3.tuple.Pair; import org.testng.Assert; import org.testng.annotations.AfterSuite; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeSuite; import org.testng.annotations.Test; import com.google.common.io.Closer; import com.typesafe.config.Config; +import lombok.extern.slf4j.Slf4j; + import org.apache.gobblin.configuration.ConfigurationKeys; import org.apache.gobblin.kafka.KafkaTestBase; import org.apache.gobblin.kafka.client.Kafka09ConsumerClient; import org.apache.gobblin.kafka.writer.KafkaWriterConfigurationKeys; import org.apache.gobblin.runtime.api.JobSpec; import org.apache.gobblin.runtime.api.Spec; +import org.apache.gobblin.runtime.api.SpecExecutor; import org.apache.gobblin.runtime.job_catalog.NonObservingFSJobCatalog; import org.apache.gobblin.runtime.job_monitor.KafkaJobMonitor; import org.apache.gobblin.util.ConfigUtils; import org.apache.gobblin.writer.WriteResponse; -import org.apache.gobblin.runtime.api.SpecExecutor; - -import lombok.extern.slf4j.Slf4j; @Slf4j @@ -63,9 +64,12 @@ public class StreamingKafkaSpecExecutorTest extends KafkaTestBase { private String _kafkaBrokers; private static final String _TEST_DIR_PATH = "/tmp/StreamingKafkaSpecExecutorTest"; private static final String _JOBS_DIR_PATH = _TEST_DIR_PATH + "/jobs"; + String flowSpecUriString = "/flowgroup/flowname/spec"; + Spec flowSpec = initJobSpecWithFlowExecutionId(flowSpecUriString, "12345"); String specUriString = "/foo/bar/spec"; Spec spec = initJobSpec(specUriString); + @BeforeSuite public void beforeSuite() { log.info("Process id = " + ManagementFactory.getRuntimeMXBean().getName()); @@ -92,9 +96,8 @@ private void cleanupTestDir() { } } } - - @Test - public void testAddSpec() throws Exception { + @BeforeClass + public void setup() throws Exception { _closer = Closer.create(); _properties = new Properties(); @@ -116,9 +119,6 @@ public void testAddSpec() throws Exception { // SEI Producer _seip = _closer.register(new SimpleKafkaSpecProducer(config)); - WriteResponse writeResponse = (WriteResponse) _seip.addSpec(spec).get(); - log.info("WriteResponse: " + writeResponse); - _jobCatalog = new NonObservingFSJobCatalog(config.getConfig("gobblin.cluster")); _jobCatalog.startAsync().awaitRunning(); @@ -126,6 +126,13 @@ public void testAddSpec() throws Exception { _seic = _closer.register(new StreamingKafkaSpecConsumer(config, _jobCatalog)); _seic.startAsync().awaitRunning(); + } + + @Test + public void testAddSpec() throws Exception { + WriteResponse writeResponse = (WriteResponse) _seip.addSpec(spec).get(); + log.info("WriteResponse: " + writeResponse); + List> consumedEvent = _seic.changedSpecs().get(); Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); @@ -165,6 +172,78 @@ public void testDeleteSpec() throws Exception { Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); } + @Test(dependsOnMethods = "testDeleteSpec") + public void testCancelSpec() throws Exception { + // Cancel an existing spec that was added + _seip.addSpec(spec).get(); + WriteResponse writeResponse = (WriteResponse) _seip.cancelJob(new URI(specUriString), new Properties()).get(); + log.info("WriteResponse: " + writeResponse); + + // Wait for the cancellation to be processed + Thread.sleep(5000); + List> consumedEvent = _seic.changedSpecs().get(); + Assert.assertTrue(consumedEvent.size() == 3, "Consumption did not match production"); + + Map.Entry consumedSpecAction = consumedEvent.get(2); + log.info(consumedSpecAction.getKey().toString()); + Assert.assertTrue(consumedEvent.get(0).getKey().equals(SpecExecutor.Verb.ADD), "Verb did not match"); + Assert.assertTrue(consumedEvent.get(1).getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match"); + Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.CANCEL), "Verb did not match"); + Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(specUriString), "Expected URI did not match"); + Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); + } + + @Test (dependsOnMethods = "testCancelSpec") + public void testCancelSpecNoopDefault() throws Exception { + _seip.addSpec(flowSpec).get(); + Properties props = new Properties(); + props.setProperty(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, "54321"); // Does not match with added jobspec, so should not cancel job + WriteResponse writeResponse = (WriteResponse) _seip.cancelJob(new URI(flowSpecUriString), props).get(); + log.info("WriteResponse: " + writeResponse); + // Wait for the cancellation to be processed, but it should ignore the spec as flow execution IDs do not match + Thread.sleep(5000); + List> consumedEvent = _seic.changedSpecs().get(); + Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production"); + + Map.Entry consumedSpecAction = consumedEvent.get(0); + Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.ADD), "Verb did not match"); + Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(flowSpecUriString), "Expected URI did not match"); + Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); + + _seip.cancelJob(new URI(flowSpecUriString), new Properties()).get(); + Thread.sleep(5000); + consumedEvent = _seic.changedSpecs().get(); + Assert.assertTrue(consumedEvent.size() == 2, "Should emit cancellation event if no flow ID provided"); + consumedSpecAction = consumedEvent.get(1); + Assert.assertTrue(consumedEvent.get(0).getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match"); + Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.CANCEL), "Verb did not match"); + Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(flowSpecUriString), "Expected URI did not match"); + Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); + } + + @Test(dependsOnMethods = "testCancelSpecNoopDefault") + public void testCancelSpecWithFlowExecutionId() throws Exception { + _seip.addSpec(flowSpec).get(); + Properties props = new Properties(); + props.setProperty(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, "12345"); + WriteResponse writeResponse = (WriteResponse) _seip.cancelJob(new URI(flowSpecUriString), props).get(); + log.info("WriteResponse: " + writeResponse); + + // Wait for the cancellation to be processed + Thread.sleep(5000); + List> consumedEvent = _seic.changedSpecs().get(); + Assert.assertTrue(consumedEvent.size() == 3, "Consumption did not match production"); + + Map.Entry consumedSpecAction = consumedEvent.get(2); + log.info(consumedSpecAction.getKey().toString()); + Assert.assertTrue(consumedEvent.get(0).getKey().equals(SpecExecutor.Verb.ADD), "Verb did not match"); + Assert.assertTrue(consumedEvent.get(1).getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match"); + Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.CANCEL), "Verb did not match"); + Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(flowSpecUriString), "Expected URI did not match"); + Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec"); + } + + private static JobSpec initJobSpec(String specUri) { Properties properties = new Properties(); return JobSpec.builder(specUri) @@ -174,6 +253,16 @@ private static JobSpec initJobSpec(String specUri) { .build(); } + private static JobSpec initJobSpecWithFlowExecutionId(String specUri, String flowExecutionId) { + Properties properties = new Properties(); + properties.setProperty(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, flowExecutionId); + return JobSpec.builder(specUri) + .withConfig(ConfigUtils.propertiesToConfig(properties)) + .withVersion("1") + .withDescription("Spec Description") + .build(); + } + @AfterSuite public void after() { try { diff --git a/gobblin-modules/gobblin-service-kafka/src/main/java/org/apache/gobblin/service/SimpleKafkaSpecProducer.java b/gobblin-modules/gobblin-service-kafka/src/main/java/org/apache/gobblin/service/SimpleKafkaSpecProducer.java index 953d841f11a..0d01bd4f91a 100644 --- a/gobblin-modules/gobblin-service-kafka/src/main/java/org/apache/gobblin/service/SimpleKafkaSpecProducer.java +++ b/gobblin-modules/gobblin-service-kafka/src/main/java/org/apache/gobblin/service/SimpleKafkaSpecProducer.java @@ -17,27 +17,30 @@ package org.apache.gobblin.service; -import com.google.common.base.Joiner; import java.io.Closeable; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; import java.util.List; -import java.util.concurrent.Future; import java.util.Properties; +import java.util.concurrent.Future; import org.apache.commons.lang3.reflect.ConstructorUtils; -import org.apache.gobblin.configuration.ConfigurationKeys; import org.slf4j.Logger; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; +import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.typesafe.config.Config; +import javax.annotation.concurrent.NotThreadSafe; +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.configuration.ConfigurationKeys; import org.apache.gobblin.configuration.State; import org.apache.gobblin.instrumented.Instrumented; import org.apache.gobblin.metrics.MetricContext; @@ -54,9 +57,6 @@ import org.apache.gobblin.writer.AsyncDataWriter; import org.apache.gobblin.writer.WriteCallback; -import javax.annotation.concurrent.NotThreadSafe; -import lombok.extern.slf4j.Slf4j; - @Slf4j @NotThreadSafe public class SimpleKafkaSpecProducer implements SpecProducer, Closeable { @@ -105,19 +105,6 @@ private Meter createMeter(String suffix) { return this.metricContext.meter(MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX, getClass().getSimpleName(), suffix)); } - private Spec addExecutionIdToJobSpecUri(Spec spec) { - JobSpec newSpec = (JobSpec)spec; - if (newSpec.getConfig().hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)) { - try { - newSpec.setUri(new URI(Joiner.on("/"). - join(spec.getUri().toString(), newSpec.getConfig().getString(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)))); - } catch (URISyntaxException e) { - log.error("Cannot create job uri to cancel job", e); - } - } - return newSpec; - } - private URI getURIWithExecutionId(URI originalURI, Properties props) { URI result = originalURI; if (props.containsKey(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)) { @@ -133,10 +120,9 @@ private URI getURIWithExecutionId(URI originalURI, Properties props) { @Override public Future addSpec(Spec addedSpec) { - Spec spec = addExecutionIdToJobSpecUri(addedSpec); - AvroJobSpec avroJobSpec = convertToAvroJobSpec(spec, SpecExecutor.Verb.ADD); + AvroJobSpec avroJobSpec = convertToAvroJobSpec(addedSpec, SpecExecutor.Verb.ADD); - log.info("Adding Spec: " + spec + " using Kafka."); + log.info("Adding Spec: " + addedSpec + " using Kafka."); this.addSpecMeter.mark(); return getKafkaProducer().write(_serializer.serializeRecord(avroJobSpec), new KafkaWriteCallback(avroJobSpec)); @@ -144,10 +130,9 @@ public Future addSpec(Spec addedSpec) { @Override public Future updateSpec(Spec updatedSpec) { - Spec spec = addExecutionIdToJobSpecUri(updatedSpec); - AvroJobSpec avroJobSpec = convertToAvroJobSpec(spec, SpecExecutor.Verb.UPDATE); + AvroJobSpec avroJobSpec = convertToAvroJobSpec(updatedSpec, SpecExecutor.Verb.UPDATE); - log.info("Updating Spec: " + spec + " using Kafka."); + log.info("Updating Spec: " + updatedSpec + " using Kafka."); this.updateSpecMeter.mark(); return getKafkaProducer().write(_serializer.serializeRecord(avroJobSpec), new KafkaWriteCallback(avroJobSpec)); @@ -155,13 +140,11 @@ public Future updateSpec(Spec updatedSpec) { @Override public Future deleteSpec(URI deletedSpecURI, Properties headers) { - URI finalDeletedSpecURI = getURIWithExecutionId(deletedSpecURI, headers); - - AvroJobSpec avroJobSpec = AvroJobSpec.newBuilder().setUri(finalDeletedSpecURI.toString()) + AvroJobSpec avroJobSpec = AvroJobSpec.newBuilder().setUri(deletedSpecURI.toString()) .setMetadata(ImmutableMap.of(SpecExecutor.VERB_KEY, SpecExecutor.Verb.DELETE.name())) .setProperties(Maps.fromProperties(headers)).build(); - log.info("Deleting Spec: " + finalDeletedSpecURI + " using Kafka."); + log.info("Deleting Spec: " + deletedSpecURI + " using Kafka."); this.deleteSpecMeter.mark(); return getKafkaProducer().write(_serializer.serializeRecord(avroJobSpec), new KafkaWriteCallback(avroJobSpec)); @@ -169,12 +152,11 @@ public Future deleteSpec(URI deletedSpecURI, Properties headers) { @Override public Future cancelJob(URI deletedSpecURI, Properties properties) { - URI finalDeletedSpecURI = getURIWithExecutionId(deletedSpecURI, properties); - AvroJobSpec avroJobSpec = AvroJobSpec.newBuilder().setUri(finalDeletedSpecURI.toString()) + AvroJobSpec avroJobSpec = AvroJobSpec.newBuilder().setUri(deletedSpecURI.toString()) .setMetadata(ImmutableMap.of(SpecExecutor.VERB_KEY, SpecExecutor.Verb.CANCEL.name())) .setProperties(Maps.fromProperties(properties)).build(); - log.info("Cancelling job: " + finalDeletedSpecURI + " using Kafka."); + log.info("Cancelling job: " + deletedSpecURI + " using Kafka."); this.cancelSpecMeter.mark(); return getKafkaProducer().write(_serializer.serializeRecord(avroJobSpec), new KafkaWriteCallback(avroJobSpec)); diff --git a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowStatusTest.java b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowStatusTest.java index 317bb53f97c..c507b031056 100644 --- a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowStatusTest.java +++ b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-client/src/test/java/org/apache/gobblin/service/FlowStatusTest.java @@ -29,7 +29,6 @@ import org.testng.annotations.Test; import com.google.common.base.Suppliers; -import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.inject.Binder; import com.google.inject.Guice; @@ -67,7 +66,7 @@ public Iterator getJobStatusesF @Override public Iterator getJobStatusesForFlowExecution(String flowName, String flowGroup, long flowExecutionId, String jobGroup, String jobName) { - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } @Override diff --git a/gobblin-runtime/build.gradle b/gobblin-runtime/build.gradle index df86c5cec17..3a83c2f7f24 100644 --- a/gobblin-runtime/build.gradle +++ b/gobblin-runtime/build.gradle @@ -80,7 +80,6 @@ dependencies { compile externalDependency.metricsCore compile externalDependency.metricsJvm compile externalDependency.metricsJmx - compile externalDependency.mockito compile externalDependency.pegasus.data compile externalDependency.quartz compile externalDependency.slf4j @@ -101,6 +100,7 @@ dependencies { testCompile externalDependency.mockito testRuntime externalDependency.derby testCompile externalDependency.jmh + testCompile externalDependency.mockito } // Begin HACK to get around POM being depenendent on the (empty) gobblin-rest-api instead of gobblin-rest-api-rest-client diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java index 217c95b47f7..6e47cc482b9 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java @@ -92,6 +92,7 @@ import org.apache.gobblin.runtime.troubleshooter.AutomaticTroubleshooter; import org.apache.gobblin.runtime.troubleshooter.AutomaticTroubleshooterFactory; import org.apache.gobblin.runtime.troubleshooter.IssueRepository; +import org.apache.gobblin.runtime.util.GsonUtils; import org.apache.gobblin.runtime.util.JobMetrics; import org.apache.gobblin.service.ServiceConfigKeys; import org.apache.gobblin.source.InfiniteSource; @@ -728,6 +729,39 @@ protected void postProcessTaskStates(@SuppressWarnings("unused") List */ protected void postProcessJobState(JobState jobState) { postProcessTaskStates(jobState.getTaskStates()); + if (!GobblinMetrics.isEnabled(this.jobProps)) { + return; + } + List datasetTaskSummaries = new ArrayList<>(); + Map datasetStates = this.jobContext.getDatasetStatesByUrns(); + // Only process successful datasets unless configuration to process failed datasets is set + boolean processFailedTasks = PropertiesUtils.getPropAsBoolean(this.jobProps, ConfigurationKeys.WRITER_COUNT_METRICS_FROM_FAILED_TASKS, "false"); + for (JobState.DatasetState datasetState : datasetStates.values()) { + if (datasetState.getState() == JobState.RunningState.COMMITTED + || (datasetState.getState() == JobState.RunningState.FAILED && this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_SUCCESSFUL_TASKS)) { + long totalBytesWritten = 0; + long totalRecordsWritten = 0; + for (TaskState taskState : datasetState.getTaskStates()) { + // Certain writers may omit these metrics e.g. CompactionLauncherWriter + if ((taskState.getWorkingState() == WorkUnitState.WorkingState.COMMITTED || processFailedTasks)) { + totalBytesWritten += taskState.getPropAsLong(ConfigurationKeys.WRITER_BYTES_WRITTEN, 0); + totalRecordsWritten += taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN, 0); + } + } + LOG.info(String.format("DatasetMetrics for '%s' - (records: %d; bytes: %d)", datasetState.getDatasetUrn(), totalRecordsWritten, totalBytesWritten)); + datasetTaskSummaries.add(new DatasetTaskSummary(datasetState.getDatasetUrn(), totalRecordsWritten, totalBytesWritten, + datasetState.getState() == JobState.RunningState.COMMITTED)); + } else if (datasetState.getState() == JobState.RunningState.FAILED) { + // Check if config is turned on for submitting writer metrics on failure due to non-atomic write semantics + if (this.jobContext.getJobCommitPolicy() == JobCommitPolicy.COMMIT_ON_FULL_SUCCESS) { + LOG.info("Due to task failure, will report that no records or bytes were written for " + datasetState.getDatasetUrn()); + datasetTaskSummaries.add(new DatasetTaskSummary(datasetState.getDatasetUrn(), 0, 0, false)); + } + } + } + TimingEvent jobSummaryTimer = this.eventSubmitter.getTimingEvent(TimingEvent.LauncherTimings.JOB_SUMMARY); + jobSummaryTimer.addMetadata(TimingEvent.DATASET_TASK_SUMMARIES, GsonUtils.GSON_WITH_DATE_HANDLING.toJson(datasetTaskSummaries)); + jobSummaryTimer.stop(); } @Override diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/DatasetTaskSummary.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/DatasetTaskSummary.java new file mode 100644 index 00000000000..282b3cbb91f --- /dev/null +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/DatasetTaskSummary.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.runtime; + +import lombok.Data; + +import org.apache.gobblin.metrics.DatasetMetric; + + +/** + * A class returned by {@link org.apache.gobblin.runtime.SafeDatasetCommit} to provide metrics for the dataset + * that can be reported as a single event in the commit phase. + */ +@Data +public class DatasetTaskSummary { + private final String datasetUrn; + private final long recordsWritten; + private final long bytesWritten; + private final boolean successfullyCommitted; + + /** + * Convert a {@link DatasetTaskSummary} to a {@link DatasetMetric}. + */ + public static DatasetMetric toDatasetMetric(DatasetTaskSummary datasetTaskSummary) { + return new DatasetMetric(datasetTaskSummary.getDatasetUrn(), datasetTaskSummary.getBytesWritten(), datasetTaskSummary.getRecordsWritten(), datasetTaskSummary.isSuccessfullyCommitted()); + } +} diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/job_monitor/KafkaJobMonitor.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/job_monitor/KafkaJobMonitor.java index 8a1bf3161c9..6952dcff896 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/job_monitor/KafkaJobMonitor.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/job_monitor/KafkaJobMonitor.java @@ -28,11 +28,13 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.gobblin.configuration.ConfigurationKeys; import org.apache.gobblin.kafka.client.DecodeableKafkaRecord; import org.apache.gobblin.metastore.DatasetStateStore; import org.apache.gobblin.metrics.ContextAwareMeter; import org.apache.gobblin.runtime.api.JobSpec; import org.apache.gobblin.runtime.api.JobSpecMonitor; +import org.apache.gobblin.runtime.api.JobSpecNotFoundException; import org.apache.gobblin.runtime.api.MutableJobCatalog; import org.apache.gobblin.runtime.api.SpecExecutor; import org.apache.gobblin.runtime.kafka.HighLevelConsumer; @@ -136,14 +138,32 @@ protected void processMessage(DecodeableKafkaRecord message) { break; case DELETE: this.removedSpecs.mark(); - URI jobSpecUri = parsedMessage.getUri(); - this.jobCatalog.remove(jobSpecUri); + this.jobCatalog.remove(parsedMessage.getUri()); // Delete the job state if it is a delete spec request - deleteStateStore(jobSpecUri); + deleteStateStore(parsedMessage.getUri()); break; case CANCEL: - this.cancelledSpecs.mark(); - this.jobCatalog.remove(parsedMessage.getUri(), true); + URI specUri = parsedMessage.getUri(); + try { + JobSpec spec = this.jobCatalog.getJobSpec(specUri); + // If incoming job or existing job does not have an associated flow execution ID, default to cancelling the job + if (!spec.getConfig().hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY) || !parsedMessage.getConfig().hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)) { + this.cancelledSpecs.mark(); + this.jobCatalog.remove(specUri, true); + } else { + // Validate that the flow execution ID of the running flow matches the one in the incoming job spec + String flowIdToCancel = parsedMessage.getConfig().getString(ConfigurationKeys.FLOW_EXECUTION_ID_KEY); + if (spec.getConfig().getString(ConfigurationKeys.FLOW_EXECUTION_ID_KEY).equals(flowIdToCancel)) { + this.cancelledSpecs.mark(); + this.jobCatalog.remove(specUri, true); + } else { + log.warn("Job spec {} that has flow execution ID {} could not be cancelled, incoming request expects to cancel flow execution ID {}", specUri, + spec.getConfig().getString(ConfigurationKeys.FLOW_EXECUTION_ID_KEY), flowIdToCancel); + } + } + } catch (JobSpecNotFoundException e) { + log.warn("Could not find job spec {} to cancel in job catalog", specUri); + } break; default: log.error("Cannot process spec {} with verb {}", parsedMessage.getUri(), verb); diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncher.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncher.java index a35210389b2..bb89f9c6d50 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncher.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncher.java @@ -21,11 +21,14 @@ import java.io.FileInputStream; import java.io.IOException; import java.net.URI; +import java.text.SimpleDateFormat; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -212,7 +215,7 @@ public MRJobLauncher(Properties jobProps, Configuration conf, SharedResourcesBro JobConfigurationUtils.putPropertiesIntoConfiguration(this.jobProps, this.conf); // Let the job and all mappers finish even if some mappers fail - this.conf.set("mapreduce.map.failures.maxpercent", "100"); // For Hadoop 2.x + this.conf.set("mapreduce.map.failures.maxpercent", isMapperFailureFatalEnabled(this.jobProps) ? "0" : "100"); // For Hadoop 2.x // Do not cancel delegation tokens after job has completed (HADOOP-7002) this.conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false); @@ -228,8 +231,18 @@ public MRJobLauncher(Properties jobProps, Configuration conf, SharedResourcesBro this.fs.delete(this.mrJobDir, true); } this.unsharedJarsDir = new Path(this.mrJobDir, JARS_DIR_NAME); - this.jarsDir = this.jobProps.containsKey(ConfigurationKeys.MR_JARS_DIR) ? new Path( - this.jobProps.getProperty(ConfigurationKeys.MR_JARS_DIR)) : this.unsharedJarsDir; + + if (this.jobProps.containsKey(ConfigurationKeys.MR_JARS_BASE_DIR)) { + Path jarsBaseDir = new Path(this.jobProps.getProperty(ConfigurationKeys.MR_JARS_BASE_DIR)); + String monthSuffix = new SimpleDateFormat("yyyy-MM").format(System.currentTimeMillis()); + cleanUpOldJarsDirIfRequired(this.fs, jarsBaseDir); + this.jarsDir = new Path(jarsBaseDir, monthSuffix); + } else if (this.jobProps.containsKey(ConfigurationKeys.MR_JARS_DIR)) { + this.jarsDir = new Path(this.jobProps.getProperty(ConfigurationKeys.MR_JARS_DIR)); + } else { + this.jarsDir = this.unsharedJarsDir; + } + this.fs.mkdirs(this.mrJobDir); this.jobInputPath = new Path(this.mrJobDir, INPUT_DIR_NAME); @@ -264,6 +277,14 @@ public MRJobLauncher(Properties jobProps, Configuration conf, SharedResourcesBro startCancellationExecutor(); } + static void cleanUpOldJarsDirIfRequired(FileSystem fs, Path jarsBaseDir) throws IOException { + List jarDirs = Arrays.stream(fs.exists(jarsBaseDir) + ? fs.listStatus(jarsBaseDir) : new FileStatus[0]).sorted().collect(Collectors.toList()); + if (jarDirs.size() > 2) { + fs.delete(jarDirs.get(0).getPath(), true); + } + } + @Override public void close() throws IOException { try { @@ -504,6 +525,13 @@ static boolean isCustomizedProgressReportEnabled(Properties properties) { && Boolean.parseBoolean(properties.getProperty(ENABLED_CUSTOMIZED_PROGRESS)); } + static boolean isMapperFailureFatalEnabled(Properties properties) { + return ( + properties.containsKey(ConfigurationKeys.MR_JOB_MAPPER_FAILURE_IS_FATAL_KEY) + && Boolean.parseBoolean(properties.getProperty(ConfigurationKeys.MR_JOB_MAPPER_FAILURE_IS_FATAL_KEY))) + || ConfigurationKeys.DEFAULT_MR_JOB_MAPPER_FAILURE_IS_FATAL; + } + @VisibleForTesting static void serializeJobState(FileSystem fs, Path mrJobDir, Configuration conf, JobState jobState, Job job) throws IOException { diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java b/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java index 26fc76fb907..e5845fe373e 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/service/monitoring/JobStatusRetriever.java @@ -30,7 +30,6 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterators; import com.google.common.collect.Ordering; import com.typesafe.config.ConfigFactory; @@ -99,7 +98,7 @@ public long getLatestExecutionIdForFlow(String flowName, String flowGroup) { public Iterator getLatestJobStatusByFlowNameAndGroup(String flowName, String flowGroup) { long latestExecutionId = getLatestExecutionIdForFlow(flowName, flowGroup); - return latestExecutionId == -1L ? Iterators.emptyIterator() + return latestExecutionId == -1L ? Collections.emptyIterator() : getJobStatusesForFlowExecution(flowName, flowGroup, latestExecutionId); } diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/JobLauncherTestHelper.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/JobLauncherTestHelper.java index fff6c77fa32..00f16edb893 100644 --- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/JobLauncherTestHelper.java +++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/JobLauncherTestHelper.java @@ -18,7 +18,10 @@ package org.apache.gobblin.runtime; import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; @@ -30,12 +33,16 @@ import org.testng.Assert; import com.google.common.io.Closer; +import com.google.gson.reflect.TypeToken; import org.apache.gobblin.configuration.ConfigurationKeys; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.configuration.WorkUnitState; import org.apache.gobblin.metastore.StateStore; +import org.apache.gobblin.metrics.GobblinTrackingEvent; +import org.apache.gobblin.metrics.test.MetricsAssert; import org.apache.gobblin.runtime.JobState.DatasetState; +import org.apache.gobblin.runtime.util.GsonUtils; import org.apache.gobblin.source.extractor.Extractor; import org.apache.gobblin.source.workunit.WorkUnit; import org.apache.gobblin.test.TestExtractor; @@ -63,26 +70,36 @@ public JobLauncherTestHelper(Properties launcherProps, StateStore events = metricsAssert.getEvents(); + Assert.assertTrue(events.stream().anyMatch(e -> e.getName().equals("JobStartTimer"))); Assert.assertTrue(jobContext.getJobMetricsOptional().isPresent()); String jobMetricContextTags = jobContext.getJobMetricsOptional().get().getMetricContext().getTags().toString(); Assert.assertTrue(jobMetricContextTags.contains(ClusterNameTags.CLUSTER_IDENTIFIER_TAG_NAME), ClusterNameTags.CLUSTER_IDENTIFIER_TAG_NAME + " tag missing in job metric context tags."); - + Assert.assertTrue(events.stream().anyMatch(e -> e.getName().equals("JobCommitTimer"))); List datasetStateList = this.datasetStateStore.getAll(jobName, sanitizeJobNameForDatasetStore(jobId) + ".jst"); DatasetState datasetState = datasetStateList.get(0); @@ -90,7 +107,7 @@ public JobContext runTest(Properties jobProps) throws Exception { Assert.assertEquals(datasetState.getJobFailures(), 0); int skippedWorkunits = 0; - + int totalRecords = 0; for (TaskState taskState : datasetState.getTaskStates()) { if (Boolean.parseBoolean(jobProps.getProperty(ConfigurationKeys.WORK_UNIT_SKIP_KEY, Boolean.FALSE.toString())) && taskState.getWorkingState() == WorkUnitState.WorkingState.SKIPPED) { @@ -101,6 +118,7 @@ public JobContext runTest(Properties jobProps) throws Exception { Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED); Assert.assertEquals(taskState.getPropAsLong(ConfigurationKeys.WRITER_RECORDS_WRITTEN), TestExtractor.TOTAL_RECORDS); + totalRecords += TestExtractor.TOTAL_RECORDS; // if the addition of the task timestamp is configured then validate that the file name has the expected format if (Boolean.valueOf(taskState.getProp(ConfigurationKeys.WRITER_ADD_TASK_TIMESTAMP, "false"))) { @@ -122,6 +140,15 @@ public JobContext runTest(Properties jobProps) throws Exception { Assert.assertTrue(Long.valueOf(m.group(2)) < currentTime); } } + Optional + summaryEvent = events.stream().filter(e -> e.getName().equals("JobSummaryTimer")).findFirst(); + Assert.assertNotNull(summaryEvent); + Assert.assertTrue(summaryEvent.get().getMetadata().containsKey("datasetTaskSummaries")); + Type datasetTaskSummaryType = new TypeToken>(){}.getType(); + List datasetTaskSummaries = + GsonUtils.GSON_WITH_DATE_HANDLING.fromJson(summaryEvent.get().getMetadata().get("datasetTaskSummaries"), datasetTaskSummaryType); + Assert.assertEquals(datasetTaskSummaries.size(), 1); + Assert.assertEquals(datasetTaskSummaries.get(0).getRecordsWritten(), totalRecords); if (Boolean.parseBoolean(jobProps.getProperty(ConfigurationKeys.WORK_UNIT_SKIP_KEY, Boolean.FALSE.toString()))) { diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/TestWorkUnitStreamSource.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/TestWorkUnitStreamSource.java index a1fd5ed4e6d..42de0e2eb15 100644 --- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/TestWorkUnitStreamSource.java +++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/TestWorkUnitStreamSource.java @@ -31,6 +31,8 @@ import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; +import lombok.extern.slf4j.Slf4j; + import org.apache.gobblin.configuration.ConfigurationKeys; import org.apache.gobblin.configuration.SourceState; import org.apache.gobblin.runtime.api.JobExecutionDriver; @@ -43,8 +45,6 @@ import org.apache.gobblin.task.EventBusPublishingTaskFactory; import org.apache.gobblin.writer.test.TestingEventBuses; -import lombok.extern.slf4j.Slf4j; - @Slf4j public class TestWorkUnitStreamSource { diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncherTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncherTest.java index 11d4df97507..2a45ad30e19 100644 --- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncherTest.java +++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncherTest.java @@ -26,6 +26,9 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.jboss.byteman.contrib.bmunit.BMNGRunner; import org.jboss.byteman.contrib.bmunit.BMRule; import org.jboss.byteman.contrib.bmunit.BMRules; @@ -37,6 +40,7 @@ import org.testng.annotations.Test; import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; @@ -117,6 +121,22 @@ public void testLaunchJob() throws Exception { log.info("out"); } + @Test + public void testCleanUpMrJarsBaseDir() throws Exception { + File tmpDirectory = Files.createTempDir(); + tmpDirectory.deleteOnExit(); + FileSystem fs = FileSystem.get(new Configuration()); + String baseJarDir = tmpDirectory.getAbsolutePath(); + fs.mkdirs(new Path(baseJarDir, "2023-01")); + fs.mkdirs(new Path(baseJarDir, "2022-12")); + fs.mkdirs(new Path(baseJarDir, "2023-02")); + MRJobLauncher.cleanUpOldJarsDirIfRequired(FileSystem.get(new Configuration()), new Path(tmpDirectory.getPath())); + Assert.assertFalse(fs.exists(new Path(baseJarDir, "2022-12"))); + Assert.assertTrue(fs.exists(new Path(baseJarDir, "2023-01"))); + Assert.assertTrue(fs.exists(new Path(baseJarDir, "2023-02"))); + fs.delete(new Path(baseJarDir), true); + } + @Test public void testNumOfWorkunits() throws Exception { Properties jobProps = loadJobProps(); diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/spec_executorInstance/MockedSpecExecutor.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/spec_executorInstance/MockedSpecExecutor.java similarity index 100% rename from gobblin-runtime/src/main/java/org/apache/gobblin/runtime/spec_executorInstance/MockedSpecExecutor.java rename to gobblin-runtime/src/test/java/org/apache/gobblin/runtime/spec_executorInstance/MockedSpecExecutor.java diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java index 6251ac1d2fe..f47cbde507a 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java @@ -1029,7 +1029,8 @@ private void submitJob(DagNode dagNode) { addSpecFuture.get(); jobMetadata.put(TimingEvent.METADATA_MESSAGE, producer.getExecutionLink(addSpecFuture, specExecutorUri)); - + // Add serialized job properties as part of the orchestrated job event metadata + jobMetadata.put(JobExecutionPlan.JOB_PROPS_KEY, dagNode.getValue().toString()); if (jobOrchestrationTimer != null) { jobOrchestrationTimer.stop(jobMetadata); } diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/spec/JobExecutionPlan.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/spec/JobExecutionPlan.java index beccb1a13f5..8a6f5e1d496 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/spec/JobExecutionPlan.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/spec/JobExecutionPlan.java @@ -60,6 +60,7 @@ @EqualsAndHashCode(exclude = {"executionStatus", "currentAttempts", "jobFuture", "flowStartTime"}) public class JobExecutionPlan { public static final String JOB_MAX_ATTEMPTS = "job.maxAttempts"; + public static final String JOB_PROPS_KEY = "job.props"; private static final int MAX_JOB_NAME_LENGTH = 255; private final JobSpec jobSpec; diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/FsJobStatusRetriever.java b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/FsJobStatusRetriever.java index 613dc7f69a1..15fa2eee64c 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/FsJobStatusRetriever.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/FsJobStatusRetriever.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -81,14 +82,14 @@ public Iterator getJobStatusesForFlowExecution(String flowName, Strin for (String tableName: tableNames) { List jobStates = this.stateStore.getAll(storeName, tableName); if (jobStates.isEmpty()) { - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } jobStatuses.add(getJobStatus(jobStates.get(0))); } return jobStatuses.iterator(); } catch (IOException e) { log.error(String.format("IOException encountered when retrieving job statuses for flow: %s,%s,%s", flowGroup, flowName, flowExecutionId), e); - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } } @@ -105,13 +106,13 @@ public Iterator getJobStatusesForFlowExecution(String flowName, Strin String tableName = KafkaJobStatusMonitor.jobStatusTableName(flowExecutionId, jobGroup, jobName); List jobStates = this.stateStore.getAll(storeName, tableName); if (jobStates.isEmpty()) { - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } else { return Iterators.singletonIterator(getJobStatus(jobStates.get(0))); } } catch (IOException e) { log.error(String.format("Exception encountered when listing files for flow: %s,%s,%s;%s,%s", flowGroup, flowName, flowExecutionId, jobGroup, jobName), e); - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } } diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/GaaSObservabilityEventProducer.java b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/GaaSObservabilityEventProducer.java index e3e5ae11df9..2b81378f6ff 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/GaaSObservabilityEventProducer.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/GaaSObservabilityEventProducer.java @@ -19,16 +19,20 @@ import java.io.Closeable; import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import com.codahale.metrics.MetricRegistry; +import com.google.gson.reflect.TypeToken; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.State; import org.apache.gobblin.instrumented.Instrumented; import org.apache.gobblin.metrics.ContextAwareMeter; +import org.apache.gobblin.metrics.DatasetMetric; import org.apache.gobblin.metrics.GaaSObservabilityEventExperimental; import org.apache.gobblin.metrics.Issue; import org.apache.gobblin.metrics.IssueSeverity; @@ -36,11 +40,15 @@ import org.apache.gobblin.metrics.MetricContext; import org.apache.gobblin.metrics.ServiceMetricNames; import org.apache.gobblin.metrics.event.TimingEvent; +import org.apache.gobblin.runtime.DatasetTaskSummary; import org.apache.gobblin.runtime.troubleshooter.MultiContextIssueRepository; import org.apache.gobblin.runtime.troubleshooter.TroubleshooterException; import org.apache.gobblin.runtime.troubleshooter.TroubleshooterUtils; +import org.apache.gobblin.runtime.util.GsonUtils; import org.apache.gobblin.service.ExecutionStatus; +import org.apache.gobblin.service.ServiceConfigKeys; import org.apache.gobblin.service.modules.orchestration.AzkabanProjectConfig; +import org.apache.gobblin.service.modules.spec.JobExecutionPlan; /** @@ -93,6 +101,12 @@ private GaaSObservabilityEventExperimental createGaaSObservabilityEvent(final St Long jobOrchestratedTime = jobState.contains(TimingEvent.JOB_ORCHESTRATED_TIME) ? jobState.getPropAsLong(TimingEvent.JOB_ORCHESTRATED_TIME) : null; Long jobPlanningPhaseStartTime = jobState.contains(TimingEvent.WORKUNIT_PLAN_START_TIME) ? jobState.getPropAsLong(TimingEvent.WORKUNIT_PLAN_START_TIME) : null; Long jobPlanningPhaseEndTime = jobState.contains(TimingEvent.WORKUNIT_PLAN_END_TIME) ? jobState.getPropAsLong(TimingEvent.WORKUNIT_PLAN_END_TIME) : null; + Type datasetTaskSummaryType = new TypeToken>(){}.getType(); + List datasetTaskSummaries = jobState.contains(TimingEvent.DATASET_TASK_SUMMARIES) ? + GsonUtils.GSON_WITH_DATE_HANDLING.fromJson(jobState.getProp(TimingEvent.DATASET_TASK_SUMMARIES), datasetTaskSummaryType) : null; + List datasetMetrics = datasetTaskSummaries != null ? datasetTaskSummaries.stream().map( + DatasetTaskSummary::toDatasetMetric).collect(Collectors.toList()) : null; + GaaSObservabilityEventExperimental.Builder builder = GaaSObservabilityEventExperimental.newBuilder(); List issueList = null; try { @@ -121,7 +135,10 @@ private GaaSObservabilityEventExperimental createGaaSObservabilityEvent(final St .setJobPlanningPhaseEndTime(jobPlanningPhaseEndTime) .setIssues(issueList) .setJobStatus(status) - .setExecutionUserUrn(jobState.getProp(AzkabanProjectConfig.USER_TO_PROXY, null)); + .setExecutionUserUrn(jobState.getProp(AzkabanProjectConfig.USER_TO_PROXY, null)) + .setDatasetsWritten(datasetMetrics) + .setGaasId(this.state.getProp(ServiceConfigKeys.GOBBLIN_SERVICE_INSTANCE_NAME, null)) + .setJobProperties(jobState.getProp(JobExecutionPlan.JOB_PROPS_KEY, null)); return builder.build(); } diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/KafkaAvroJobStatusMonitor.java b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/KafkaAvroJobStatusMonitor.java index 23de601a11a..c152970cb1a 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/KafkaAvroJobStatusMonitor.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/KafkaAvroJobStatusMonitor.java @@ -127,12 +127,14 @@ public org.apache.gobblin.configuration.State parseJobStatus(GobblinTrackingEven case TimingEvent.FlowTimings.FLOW_COMPILED: properties.put(JobStatusRetriever.EVENT_NAME_FIELD, ExecutionStatus.COMPILED.name()); break; - case TimingEvent.LauncherTimings.WORK_UNITS_PREPARATION: + case TimingEvent.LauncherTimings.WORK_UNITS_CREATION: properties.put(TimingEvent.WORKUNIT_PLAN_START_TIME, properties.getProperty(TimingEvent.METADATA_START_TIME)); properties.put(TimingEvent.WORKUNIT_PLAN_END_TIME, properties.getProperty(TimingEvent.METADATA_END_TIME)); break; case TimingEvent.LauncherTimings.JOB_START: case TimingEvent.FlowTimings.FLOW_RUNNING: + case TimingEvent.LauncherTimings.JOB_SUMMARY: + case TimingEvent.LauncherTimings.WORK_UNITS_PREPARATION: properties.put(JobStatusRetriever.EVENT_NAME_FIELD, ExecutionStatus.RUNNING.name()); break; case TimingEvent.LauncherTimings.JOB_PENDING: diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/LocalFsJobStatusRetriever.java b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/LocalFsJobStatusRetriever.java index b5aac35ae7c..84dbb60ce6c 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/LocalFsJobStatusRetriever.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/LocalFsJobStatusRetriever.java @@ -21,11 +21,11 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; import com.google.common.base.Preconditions; -import com.google.common.collect.Iterators; import com.typesafe.config.Config; import javax.inject.Inject; @@ -97,7 +97,7 @@ public Iterator getJobStatusesForFlowExecution(String flowName, Strin jobStatus = JobStatus.builder().flowName(flowName).flowGroup(flowGroup).flowExecutionId(flowExecutionId). jobName(jobName).jobGroup(jobGroup).jobExecutionId(flowExecutionId).eventName(ExecutionStatus.PENDING.name()).build(); } else { - return Iterators.emptyIterator(); + return Collections.emptyIterator(); } jobStatuses.add(jobStatus); diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/GaaSObservabilityProducerTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/GaaSObservabilityProducerTest.java index 2055d455e8c..96383e6e4c9 100644 --- a/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/GaaSObservabilityProducerTest.java +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/monitoring/GaaSObservabilityProducerTest.java @@ -18,12 +18,13 @@ package org.apache.gobblin.service.monitoring; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; -import org.junit.Test; +import org.testng.annotations.Test; import org.testng.Assert; import com.google.common.collect.Maps; @@ -35,13 +36,17 @@ import org.apache.gobblin.metrics.reporter.util.AvroBinarySerializer; import org.apache.gobblin.metrics.reporter.util.AvroSerializer; import org.apache.gobblin.metrics.reporter.util.NoopSchemaVersionWriter; +import org.apache.gobblin.runtime.DatasetTaskSummary; import org.apache.gobblin.runtime.troubleshooter.InMemoryMultiContextIssueRepository; import org.apache.gobblin.runtime.troubleshooter.Issue; import org.apache.gobblin.runtime.troubleshooter.IssueSeverity; import org.apache.gobblin.runtime.troubleshooter.MultiContextIssueRepository; import org.apache.gobblin.runtime.troubleshooter.TroubleshooterUtils; +import org.apache.gobblin.runtime.util.GsonUtils; import org.apache.gobblin.service.ExecutionStatus; +import org.apache.gobblin.service.ServiceConfigKeys; import org.apache.gobblin.service.modules.orchestration.AzkabanProjectConfig; +import org.apache.gobblin.service.modules.spec.JobExecutionPlan; public class GaaSObservabilityProducerTest { @@ -58,7 +63,15 @@ public void testCreateGaaSObservabilityEventWithFullMetadata() throws Exception TroubleshooterUtils.getContextIdForJob(flowGroup, flowName, flowExecutionId, jobName), createTestIssue("issueSummary", "issueCode", IssueSeverity.INFO) ); - MockGaaSObservabilityEventProducer producer = new MockGaaSObservabilityEventProducer(new State(), this.issueRepository); + List summaries = new ArrayList<>(); + DatasetTaskSummary dataset1 = new DatasetTaskSummary("/testFolder", 100, 1000, true); + DatasetTaskSummary dataset2 = new DatasetTaskSummary("/testFolder2", 1000, 10000, false); + summaries.add(dataset1); + summaries.add(dataset2); + + State state = new State(); + state.setProp(ServiceConfigKeys.GOBBLIN_SERVICE_INSTANCE_NAME, "testCluster"); + MockGaaSObservabilityEventProducer producer = new MockGaaSObservabilityEventProducer(state, this.issueRepository); Map gteEventMetadata = Maps.newHashMap(); gteEventMetadata.put(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD, flowGroup); gteEventMetadata.put(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD, flowName); @@ -74,7 +87,8 @@ public void testCreateGaaSObservabilityEventWithFullMetadata() throws Exception gteEventMetadata.put(JobStatusRetriever.EVENT_NAME_FIELD, ExecutionStatus.COMPLETE.name()); gteEventMetadata.put(TimingEvent.JOB_ORCHESTRATED_TIME, "1"); gteEventMetadata.put(TimingEvent.FlowEventConstants.FLOW_MODIFICATION_TIME_FIELD, "20"); - + gteEventMetadata.put(TimingEvent.DATASET_TASK_SUMMARIES, GsonUtils.GSON_WITH_DATE_HANDLING.toJson(summaries)); + gteEventMetadata.put(JobExecutionPlan.JOB_PROPS_KEY, "{\"flow\":{\"executionId\":1681242538558},\"user\":{\"to\":{\"proxy\":\"newUser\"}}}"); Properties jobStatusProps = new Properties(); jobStatusProps.putAll(gteEventMetadata); producer.emitObservabilityEvent(new State(jobStatusProps)); @@ -98,7 +112,17 @@ public void testCreateGaaSObservabilityEventWithFullMetadata() throws Exception Assert.assertEquals(event.getLastFlowModificationTime(), Long.valueOf(20)); Assert.assertEquals(event.getJobStartTime(), Long.valueOf(20)); Assert.assertEquals(event.getJobEndTime(), Long.valueOf(100)); - + Assert.assertEquals(event.getDatasetsWritten().size(), 2); + Assert.assertEquals(event.getDatasetsWritten().get(0).getDatasetUrn(), dataset1.getDatasetUrn()); + Assert.assertEquals(event.getDatasetsWritten().get(0).getEntitiesWritten(), Long.valueOf(dataset1.getRecordsWritten())); + Assert.assertEquals(event.getDatasetsWritten().get(0).getBytesWritten(), Long.valueOf(dataset1.getBytesWritten())); + Assert.assertEquals(event.getDatasetsWritten().get(0).getSuccessfullyCommitted(), Boolean.valueOf(dataset1.isSuccessfullyCommitted())); + Assert.assertEquals(event.getDatasetsWritten().get(1).getDatasetUrn(), dataset2.getDatasetUrn()); + Assert.assertEquals(event.getDatasetsWritten().get(1).getEntitiesWritten(), Long.valueOf(dataset2.getRecordsWritten())); + Assert.assertEquals(event.getDatasetsWritten().get(1).getBytesWritten(), Long.valueOf(dataset2.getBytesWritten())); + Assert.assertEquals(event.getDatasetsWritten().get(1).getSuccessfullyCommitted(), Boolean.valueOf(dataset2.isSuccessfullyCommitted())); + Assert.assertEquals(event.getJobProperties(), "{\"flow\":{\"executionId\":1681242538558},\"user\":{\"to\":{\"proxy\":\"newUser\"}}}"); + Assert.assertEquals(event.getGaasId(), "testCluster"); AvroSerializer serializer = new AvroBinarySerializer<>( GaaSObservabilityEventExperimental.SCHEMA$, new NoopSchemaVersionWriter() ); diff --git a/gobblin-utility/build.gradle b/gobblin-utility/build.gradle index 7bba0ec2bd3..b916f214cdc 100644 --- a/gobblin-utility/build.gradle +++ b/gobblin-utility/build.gradle @@ -49,6 +49,7 @@ dependencies { compile externalDependency.opencsv compile externalDependency.hadoopHdfs compile externalDependency.groovy + compile externalDependency.errorProne runtimeOnly externalDependency.hadoopCommon runtimeOnly externalDependency.hadoopClientCore diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroFlattener.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroFlattener.java index f81ce38780e..38c2a9aa69b 100644 --- a/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroFlattener.java +++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/AvroFlattener.java @@ -23,13 +23,13 @@ import org.apache.avro.AvroRuntimeException; import org.apache.avro.Schema; -import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; /*** * This class provides methods to flatten an Avro Schema to make it more optimal for ORC @@ -402,6 +402,10 @@ private List flattenField(Schema.Field f, ImmutableList pa } // Wrap the Union, since parent Union is an option else { + // If the field within the parent Union has a non-null default value, then null should not be the first member + if (f.hasDefaultValue() && f.defaultVal() != null) { + isNullFirstMember = false; + } if (isNullFirstMember) { flattenedFieldSchema = Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), flattenedFieldSchema)); diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/JobLauncherUtils.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/JobLauncherUtils.java index 401d159be82..42fae521e8e 100644 --- a/gobblin-utility/src/main/java/org/apache/gobblin/util/JobLauncherUtils.java +++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/JobLauncherUtils.java @@ -68,6 +68,16 @@ public static String newJobId(String jobName) { return Id.Job.create(jobName, System.currentTimeMillis()).toString(); } + /** + * Create a new job ID from a flow execution ID. + * + * @param jobName job name + * @return new job ID + */ + public static String newJobId(String jobName, long executionId) { + return Id.Job.create(jobName, executionId).toString(); + } + /** * Create a new task ID for the job with the given job ID. * diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/request_allocation/PriorityIterableBasedRequestAllocator.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/request_allocation/PriorityIterableBasedRequestAllocator.java index be58e6d14de..b24bfd399d6 100644 --- a/gobblin-utility/src/main/java/org/apache/gobblin/util/request_allocation/PriorityIterableBasedRequestAllocator.java +++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/request_allocation/PriorityIterableBasedRequestAllocator.java @@ -17,20 +17,21 @@ package org.apache.gobblin.util.request_allocation; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; -import lombok.AccessLevel; -import lombok.Getter; - import org.slf4j.Logger; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.collect.Iterators; +import lombok.AccessLevel; +import lombok.Getter; + import org.apache.gobblin.util.Either; import org.apache.gobblin.util.ExecutorsUtils; import org.apache.gobblin.util.executors.IteratorExecutor; @@ -96,7 +97,7 @@ public Void call() } catch (InterruptedException ie) { log.error("Request allocation was interrupted."); return new AllocatedRequestsIteratorBase<>( - Iterators.>emptyIterator(), resourcePool); + Collections.emptyIterator(), resourcePool); } } diff --git a/gobblin-utility/src/test/java/org/apache/gobblin/util/AvroFlattenerTest.java b/gobblin-utility/src/test/java/org/apache/gobblin/util/AvroFlattenerTest.java index c6889da19d7..a4c9477b5c9 100644 --- a/gobblin-utility/src/test/java/org/apache/gobblin/util/AvroFlattenerTest.java +++ b/gobblin-utility/src/test/java/org/apache/gobblin/util/AvroFlattenerTest.java @@ -22,6 +22,8 @@ import org.testng.Assert; import org.testng.annotations.Test; +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; + public class AvroFlattenerTest { @@ -188,4 +190,30 @@ public void testRecordWithinMapWithinMap () throws IOException { } + /** + * Test flattening for non-null default within an Option within another Record + * Record R1 { + * fields: { + * Union [ null, + * Record 2 { + * field: type + * default: type + * } + * ] + * } + * } + */ + @Test + public void testNonNullDefaultWithinOptionWithinRecord () throws IOException { + + Schema originalSchema = readSchemaFromJsonFile("nonNullDefaultWithinOptionWithinRecord_original.json"); + Schema expectedSchema = readSchemaFromJsonFile("nonNullDefaultWithinOptionWithinRecord_flattened.json"); + Schema flattenedSchema = new AvroFlattener().flatten(originalSchema, false); + Assert.assertEquals(AvroCompatibilityHelper.getSpecificDefaultValue( + flattenedSchema.getField("parentFieldUnion__unionRecordMemberFieldUnion__superNestedFieldString1")).toString(), + "defaultString1"); + Assert.assertEquals(flattenedSchema.toString(), expectedSchema.toString()); + } + + } diff --git a/gobblin-utility/src/test/resources/flattenAvro/nonNullDefaultWithinOptionWithinRecord_flattened.json b/gobblin-utility/src/test/resources/flattenAvro/nonNullDefaultWithinOptionWithinRecord_flattened.json new file mode 100644 index 00000000000..1ef5899093a --- /dev/null +++ b/gobblin-utility/src/test/resources/flattenAvro/nonNullDefaultWithinOptionWithinRecord_flattened.json @@ -0,0 +1,37 @@ +{ + "type":"record", + "name":"parentRecordName", + "fields":[ + { + "name":"parentFieldUnion__unionRecordMemberFieldUnion__superNestedFieldString1", + "type":[ + "string", + "null" + ], + "default":"defaultString1", + "flatten_source":"parentFieldUnion.unionRecordMemberFieldUnion.superNestedFieldString1" + }, + { + "name":"parentFieldUnion__unionRecordMemberFieldUnion__superNestedFieldString2", + "type":[ + "string", + "null" + ], + "default":"defaultString2", + "flatten_source":"parentFieldUnion.unionRecordMemberFieldUnion.superNestedFieldString2" + }, + { + "name":"parentFieldUnion__unionRecordMemberFieldString", + "type":[ + "null", + "string" + ], + "default":null, + "flatten_source":"parentFieldUnion.unionRecordMemberFieldString" + }, + { + "name":"parentFieldInt", + "type":"int" + } + ] +} \ No newline at end of file diff --git a/gobblin-utility/src/test/resources/flattenAvro/nonNullDefaultWithinOptionWithinRecord_original.json b/gobblin-utility/src/test/resources/flattenAvro/nonNullDefaultWithinOptionWithinRecord_original.json new file mode 100644 index 00000000000..ae66543f13a --- /dev/null +++ b/gobblin-utility/src/test/resources/flattenAvro/nonNullDefaultWithinOptionWithinRecord_original.json @@ -0,0 +1,33 @@ +{ + "type" : "record", + "name" : "parentRecordName", + "fields" : [ { + "name" : "parentFieldUnion", + "type" : [ "null", { + "type" : "record", + "name" : "unionRecordMember", + "fields" : [ { + "name" : "unionRecordMemberFieldUnion", + "type" : [ "null", { + "type" : "record", + "name" : "superNestedRecord", + "fields" : [ { + "name" : "superNestedFieldString1", + "type" : "string", + "default": "defaultString1" + }, { + "name" : "superNestedFieldString2", + "type" : "string", + "default": "defaultString2" + } ] + } ] + }, { + "name" : "unionRecordMemberFieldString", + "type" : "string" + } ] + } ] + }, { + "name" : "parentFieldInt", + "type" : "int" + } ] +} \ No newline at end of file diff --git a/gobblin-yarn/build.gradle b/gobblin-yarn/build.gradle index 8cfb22cede0..0221c01b4ef 100644 --- a/gobblin-yarn/build.gradle +++ b/gobblin-yarn/build.gradle @@ -64,6 +64,7 @@ dependencies { testCompile project(":gobblin-example") testCompile externalDependency.testng + testCompile externalDependency.mockito testCompile externalDependency.hadoopYarnMiniCluster testCompile externalDependency.curatorFramework testCompile externalDependency.curatorTest diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java index c0cde5140d0..99c4094df50 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java @@ -157,6 +157,10 @@ * this count exceeds the maximum number allowed, it will initiate a shutdown. *

* + *

+ * Users of {@link GobblinYarnAppLauncher} need to call {@link #initializeYarnClients} which a child class can override. + *

+ * * @author Yinan Li */ public class GobblinYarnAppLauncher { @@ -194,7 +198,7 @@ public class GobblinYarnAppLauncher { private final HelixManager helixManager; - private final Configuration yarnConfiguration; + protected final Configuration yarnConfiguration; private final FileSystem fs; private final EventBus eventBus = new EventBus(GobblinYarnAppLauncher.class.getSimpleName()); @@ -238,8 +242,8 @@ public class GobblinYarnAppLauncher { private final String containerTimezone; private final String appLauncherMode; - private final String originalYarnRMAddress; - private final Map potentialYarnClients; + protected final String originalYarnRMAddress; + protected final Map potentialYarnClients = new HashMap<>(); private YarnClient yarnClient; public GobblinYarnAppLauncher(Config config, YarnConfiguration yarnConfiguration) throws IOException { @@ -260,15 +264,6 @@ public GobblinYarnAppLauncher(Config config, YarnConfiguration yarnConfiguration YarnHelixUtils.setAdditionalYarnClassPath(config, this.yarnConfiguration); this.yarnConfiguration.set("fs.automatic.close", "false"); this.originalYarnRMAddress = this.yarnConfiguration.get(GobblinYarnConfigurationKeys.YARN_RESOURCE_MANAGER_ADDRESS); - this.potentialYarnClients = new HashMap(); - Set potentialRMAddresses = new HashSet<>(ConfigUtils.getStringList(config, GobblinYarnConfigurationKeys.OTHER_YARN_RESOURCE_MANAGER_ADDRESSES)); - potentialRMAddresses.add(originalYarnRMAddress); - for (String rmAddress : potentialRMAddresses) { - YarnClient tmpYarnClient = YarnClient.createYarnClient(); - this.yarnConfiguration.set(GobblinYarnConfigurationKeys.YARN_RESOURCE_MANAGER_ADDRESS, rmAddress); - tmpYarnClient.init(new YarnConfiguration(this.yarnConfiguration)); - potentialYarnClients.put(rmAddress, tmpYarnClient); - } this.fs = GobblinClusterUtils.buildFileSystem(config, this.yarnConfiguration); this.closer.register(this.fs); @@ -331,6 +326,17 @@ public GobblinYarnAppLauncher(Config config, YarnConfiguration yarnConfiguration } } + public void initializeYarnClients(Config config) { + Set potentialRMAddresses = new HashSet<>(ConfigUtils.getStringList(config, GobblinYarnConfigurationKeys.OTHER_YARN_RESOURCE_MANAGER_ADDRESSES)); + potentialRMAddresses.add(originalYarnRMAddress); + for (String rmAddress : potentialRMAddresses) { + YarnClient tmpYarnClient = YarnClient.createYarnClient(); + this.yarnConfiguration.set(GobblinYarnConfigurationKeys.YARN_RESOURCE_MANAGER_ADDRESS, rmAddress); + tmpYarnClient.init(new YarnConfiguration(this.yarnConfiguration)); + this.potentialYarnClients.put(rmAddress, tmpYarnClient); + } + } + /** * Launch a new Gobblin instance on Yarn. * @@ -1066,6 +1072,8 @@ static Config addMetricReportingDynamicConfig(Config config, KafkaAvroSchemaRegi public static void main(String[] args) throws Exception { final GobblinYarnAppLauncher gobblinYarnAppLauncher = new GobblinYarnAppLauncher(ConfigFactory.load(), new YarnConfiguration()); + gobblinYarnAppLauncher.initializeYarnClients(ConfigFactory.load()); + Runtime.getRuntime().addShutdownHook(new Thread() { @Override diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java index 9799395dfc9..7088dfa996c 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java @@ -40,7 +40,10 @@ public class GobblinYarnConfigurationKeys { public static final int DEFAULT_RELEASED_CONTAINERS_CACHE_EXPIRY_SECS = 300; public static final String APP_VIEW_ACL = GOBBLIN_YARN_PREFIX + "appViewAcl"; public static final String DEFAULT_APP_VIEW_ACL = "*"; - public static final String YARN_RESOURCE_MANAGER_ADDRESS = "yarn.resourcemanager.address"; + + public static final String YARN_RESOURCE_MANAGER_PREFIX = "yarn.resourcemanager."; + public static final String YARN_RESOURCE_MANAGER_ADDRESS = YARN_RESOURCE_MANAGER_PREFIX + "address"; + public static final String YARN_RESOURCE_MANAGER_IDS = YARN_RESOURCE_MANAGER_PREFIX + "ids"; public static final String OTHER_YARN_RESOURCE_MANAGER_ADDRESSES= "other.yarn.resourcemanager.addresses"; // Gobblin Yarn ApplicationMaster configuration properties. diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/HelixInstancePurgerWithMetrics.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/HelixInstancePurgerWithMetrics.java index efb6644b489..9ac0ccad700 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/HelixInstancePurgerWithMetrics.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/HelixInstancePurgerWithMetrics.java @@ -42,9 +42,9 @@ public class HelixInstancePurgerWithMetrics { /** - * Blocking call for purging all offline helix instances. Provides boiler plate code for providing periodic updates + * Blocking call for purging all offline helix instances. Provides boilerplate code for providing periodic updates * and sending a GTE if it's an unexpected amount of time. - * + *

* All previous helix instances should be purged on startup. Gobblin task runners are stateless from helix * perspective because all important state is persisted separately in Workunit State Store or Watermark store. */ diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java index 7c4da8fd8b9..5f5c872c67e 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java @@ -17,7 +17,6 @@ package org.apache.gobblin.yarn; -import com.google.common.base.Strings; import java.util.ArrayDeque; import java.util.Comparator; import java.util.HashMap; @@ -31,7 +30,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import org.apache.gobblin.cluster.GobblinClusterConfigurationKeys; + +import org.apache.commons.compress.utils.Sets; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.helix.HelixDataAccessor; import org.apache.helix.HelixManager; @@ -40,6 +40,7 @@ import org.apache.helix.task.JobContext; import org.apache.helix.task.JobDag; import org.apache.helix.task.TaskDriver; +import org.apache.helix.task.TaskPartitionState; import org.apache.helix.task.TaskState; import org.apache.helix.task.WorkflowConfig; import org.apache.helix.task.WorkflowContext; @@ -47,12 +48,14 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.util.concurrent.AbstractIdleService; import com.typesafe.config.Config; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.gobblin.cluster.GobblinClusterConfigurationKeys; import org.apache.gobblin.util.ConfigUtils; import org.apache.gobblin.util.ExecutorsUtils; @@ -68,6 +71,7 @@ public class YarnAutoScalingManager extends AbstractIdleService { private final String AUTO_SCALING_PREFIX = GobblinYarnConfigurationKeys.GOBBLIN_YARN_PREFIX + "autoScaling."; private final String AUTO_SCALING_POLLING_INTERVAL_SECS = AUTO_SCALING_PREFIX + "pollingIntervalSeconds"; + private static final int THRESHOLD_NUMBER_OF_ATTEMPTS_FOR_LOGGING = 20; private final int DEFAULT_AUTO_SCALING_POLLING_INTERVAL_SECS = 60; // Only one container will be requested for each N partitions of work private final String AUTO_SCALING_PARTITIONS_PER_CONTAINER = AUTO_SCALING_PREFIX + "partitionsPerContainer"; @@ -94,6 +98,8 @@ public class YarnAutoScalingManager extends AbstractIdleService { private final double overProvisionFactor; private final SlidingWindowReservoir slidingFixedSizeWindow; private static int maxIdleTimeInMinutesBeforeScalingDown = DEFAULT_MAX_IDLE_TIME_BEFORE_SCALING_DOWN_MINUTES; + private static final HashSet + UNUSUAL_HELIX_TASK_STATES = Sets.newHashSet(TaskPartitionState.ERROR, TaskPartitionState.DROPPED); public YarnAutoScalingManager(GobblinApplicationMaster appMaster) { this.config = appMaster.getConfig(); @@ -189,6 +195,21 @@ private Set getParticipants(String filterString) { .keySet().stream().filter(x -> filterString.isEmpty() || x.contains(filterString)).collect(Collectors.toSet()); } + private String getInuseParticipantForHelixPartition(JobContext jobContext, int partition) { + if (jobContext.getPartitionNumAttempts(partition) > THRESHOLD_NUMBER_OF_ATTEMPTS_FOR_LOGGING) { + log.warn("Helix task {} has been retried for {} times, please check the config to see how we can handle this task better", + jobContext.getTaskIdForPartition(partition), jobContext.getPartitionNumAttempts(partition)); + } + if (!UNUSUAL_HELIX_TASK_STATES.contains(jobContext.getPartitionState(partition))) { + return jobContext.getAssignedParticipant(partition); + } + // adding log here now for debugging + //todo: if this happens frequently, we should reset to status to retriable or at least report the error earlier + log.info("Helix task {} is in {} state which is unexpected, please watch out to see if this get recovered", + jobContext.getTaskIdForPartition(partition), jobContext.getPartitionState(partition)); + return null; + } + /** * Iterate through the workflows configured in Helix to figure out the number of required partitions * and request the {@link YarnService} to scale to the desired number of containers. @@ -222,7 +243,8 @@ void runInternal() { if (jobContext != null) { log.debug("JobContext {} num partitions {}", jobContext, jobContext.getPartitionSet().size()); - inUseInstances.addAll(jobContext.getPartitionSet().stream().map(jobContext::getAssignedParticipant) + inUseInstances.addAll(jobContext.getPartitionSet().stream() + .map(i -> getInuseParticipantForHelixPartition(jobContext, i)) .filter(Objects::nonNull).collect(Collectors.toSet())); numPartitions = jobContext.getPartitionSet().size(); @@ -244,6 +266,8 @@ void runInternal() { // per partition. Scale the result by a constant overprovision factor. int containerCount = (int) Math.ceil(((double)numPartitions / this.partitionsPerContainer) * this.overProvisionFactor); yarnContainerRequestBundle.add(jobTag, containerCount, resource); + log.info("jobName={}, jobTag={}, numPartitions={}, targetNumContainers={}", + jobName, jobTag, numPartitions, containerCount); } } // Find all participants appearing in this cluster. Note that Helix instances can contain cluster-manager diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java index 7955c791abe..fe0d93d2dfb 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java @@ -333,7 +333,7 @@ public void handleContainerReleaseRequest(ContainerReleaseRequest containerRelea // Record that this container was explicitly released so that a new one is not spawned to replace it // Put the container id in the releasedContainerCache before releasing it so that handleContainerCompletion() // can check for the container id and skip spawning a replacement container. - // Note that this is best effort since these are asynchronous operations and a container may abort concurrently + // Note that this is the best effort since these are asynchronous operations and a container may abort concurrently // with the release call. So in some cases a replacement container may have already been spawned before // the container is put into the black list. this.releasedContainerCache.put(container.getId(), ""); @@ -453,7 +453,7 @@ private EventSubmitter buildEventSubmitter() { /** * Request an allocation of containers. If numTargetContainers is larger than the max of current and expected number * of containers then additional containers are requested. - * + *

* If numTargetContainers is less than the current number of allocated containers then release free containers. * Shrinking is relative to the number of currently allocated containers since it takes time for containers * to be allocated and assigned work and we want to avoid releasing a container prematurely before it is assigned @@ -709,7 +709,7 @@ private boolean shouldStickToTheSameNode(int containerExitStatus) { * Handle the completion of a container. A new container will be requested to replace the one * that just exited. Depending on the exit status and if container host affinity is enabled, * the new container may or may not try to be started on the same node. - * + *

* A container completes in either of the following conditions: 1) some error happens in the * container and caused the container to exit, 2) the container gets killed due to some reason, * for example, if it runs over the allowed amount of virtual or physical memory, 3) the gets @@ -860,11 +860,11 @@ private ImmutableMap.Builder buildContainerStatusEventMetadata(C * Get the number of matching container requests for the specified resource memory and cores. * Due to YARN-1902 and YARN-660, this API is not 100% accurate. {@link AMRMClientCallbackHandler#onContainersAllocated(List)} * contains logic for best effort clean up of requests, and the resource tend to match the allocated container. So in practice the count is pretty accurate. - * + *

* This API call gets the count of container requests for containers that are > resource if there is no request with the exact same resource * The RM can return containers that are larger (because of normalization etc). * Container may be larger by memory or cpu (e.g. container (1000M, 3cpu) can fit request (1000M, 1cpu) or request (500M, 3cpu). - * + *

* Thankfully since each helix tag / resource has a different priority, matching requests for one helix tag / resource * have complete isolation from another helix tag / resource */ @@ -1109,5 +1109,11 @@ public ContainerInfo(Container container, String helixParticipantId, String heli this.helixTag = helixTag; this.startupCommand = YarnService.this.buildContainerCommand(container, helixParticipantId, helixTag); } + + @Override + public String toString() { + return String.format("ContainerInfo{ container=%s, helixParticipantId=%s, helixTag=%s, startupCommand=%s }", + container.getId(), helixParticipantId, helixTag, startupCommand); + } } } diff --git a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java index 95c109cfe6b..2eb8c4f32f5 100644 --- a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java +++ b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java @@ -199,6 +199,7 @@ public void setUp() throws Exception { InstanceType.CONTROLLER, zkConnectionString); this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf); + this.gobblinYarnAppLauncher.initializeYarnClients(this.config); this.configManagedHelix = ConfigFactory.parseURL(url) .withValue("gobblin.cluster.zk.connection.string", @@ -213,6 +214,7 @@ public void setUp() throws Exception { InstanceType.PARTICIPANT, zkConnectionString); this.gobblinYarnAppLauncherManagedHelix = new GobblinYarnAppLauncher(this.configManagedHelix, clusterConf); + this.gobblinYarnAppLauncherManagedHelix.initializeYarnClients(this.configManagedHelix); } @Test diff --git a/gradle/scripts/dependencyDefinitions.gradle b/gradle/scripts/dependencyDefinitions.gradle index 56511ff0516..95f8f6c3287 100644 --- a/gradle/scripts/dependencyDefinitions.gradle +++ b/gradle/scripts/dependencyDefinitions.gradle @@ -49,8 +49,9 @@ ext.externalDependency = [ "commonsPool": "org.apache.commons:commons-pool2:2.4.2", "datanucleusCore": "org.datanucleus:datanucleus-core:3.2.10", "datanucleusRdbms": "org.datanucleus:datanucleus-rdbms:3.2.9", + "errorProne": "com.google.errorprone:error_prone_annotations:2.0.15", "eventhub": "com.microsoft.azure:azure-eventhubs:0.9.0", - "guava": "com.google.guava:guava:15.0", + "guava": "com.google.guava:guava:20.0", "groovy": "org.codehaus.groovy:groovy:2.4.8", "gson": "com.google.code.gson:gson:2.6.2", "gsonJavatimeSerialisers": "com.fatboyindustrial.gson-javatime-serialisers:gson-javatime-serialisers:1.1.1", @@ -123,7 +124,10 @@ ext.externalDependency = [ "guiceMultibindings": "com.google.inject.extensions:guice-multibindings:4.0", "guiceServlet": "com.google.inject.extensions:guice-servlet:4.0", "derby": "org.apache.derby:derby:10.12.1.1", - "mockito": "org.mockito:mockito-inline:4.11.0", // upgraded to allow mocking for constructors, static and final methods; specifically for iceberg distcp + // NOTE: To use features from mockito-inline, follow the wiki as opposed to changing this + // dependency directly to mockito-inline + // https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#unmockable + "mockito": "org.mockito:mockito-core:4.11.0", "salesforceWsc": "com.force.api:force-wsc:" + salesforceVersion, "salesforcePartner": "com.force.api:force-partner-api:" + salesforceVersion, "scala": "org.scala-lang:scala-library:2.11.8", diff --git a/gradle/scripts/javadoc.gradle b/gradle/scripts/javadoc.gradle index a2be5524222..8a21bc01def 100644 --- a/gradle/scripts/javadoc.gradle +++ b/gradle/scripts/javadoc.gradle @@ -81,7 +81,7 @@ subprojects { // Add standard javadoc repositories so we can reference classes in them using @link tasks.javadoc.options.links "http://typesafehub.github.io/config/latest/api/", "https://docs.oracle.com/javase/7/docs/api/", - "http://google.github.io/guava/releases/15.0/api/docs/", + "http://google.github.io/guava/releases/20.0/api/docs/", "http://hadoop.apache.org/docs/r${rootProject.ext.hadoopVersion}/api/", "https://hive.apache.org/javadocs/r${rootProject.ext.hiveVersion}/api/", "http://avro.apache.org/docs/${avroVersion}/api/java/",