replace org.apache.avro in generated source code

use temp version of hadoop-shaded-avro Update JobQueueChangeEvent.java more avro related code that needs changes more exclusions use thirdparty avro staging jar use thirdparty release
apache · Feb 9, 2024 · 0137b04 · 0137b04
1 parent 22d5d20
commit 0137b04
Show file tree

Hide file tree

Showing 14 changed files with 76 additions and 38 deletions.
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -136,6 +136,10 @@
           <groupId>org.apache.avro</groupId>
           <artifactId>avro</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop.thirdparty</groupId>
+          <artifactId>hadoop-shaded-avro_1_11</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>org.apache.curator</groupId>
           <artifactId>curator-client</artifactId>

diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml
@@ -114,6 +114,10 @@
           <groupId>org.apache.avro</groupId>
           <artifactId>avro</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop.thirdparty</groupId>
+          <artifactId>hadoop-shaded-avro_1_11</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-server</artifactId>
@@ -166,6 +170,10 @@
           <groupId>org.apache.avro</groupId>
           <artifactId>avro</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop.thirdparty</groupId>
+          <artifactId>hadoop-shaded-avro_1_11</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-core</artifactId>
@@ -278,6 +286,10 @@
           <groupId>org.apache.avro</groupId>
           <artifactId>avro</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop.thirdparty</groupId>
+          <artifactId>hadoop-shaded-avro_1_11</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-annotations</artifactId>
@@ -319,6 +331,10 @@
           <groupId>org.apache.avro</groupId>
           <artifactId>avro</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop.thirdparty</groupId>
+          <artifactId>hadoop-shaded-avro_1_11</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-annotations</artifactId>

diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
@@ -250,7 +250,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop.thirdparty</groupId>
-      <artifactId>hadoop-shaded-avro</artifactId>
+      <artifactId>hadoop-shaded-avro_1_11</artifactId>
       <scope>compile</scope>
     </dependency>
     <dependency>
@@ -617,6 +617,8 @@
                 <copy toDir="${project.build.directory}/test-classes">
                   <fileset dir="${basedir}/src/main/conf"/>
                 </copy>
+
+                <replace dir="${project.build.directory}/generated-test-sources/java" token="org.apache.avro" value="org.apache.hadoop.thirdparty.avro"/>
               </target>
             </configuration>
           </execution>

diff --git a/...doop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java b/...doop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
@@ -33,7 +33,7 @@ public class TestAvroSerialization {
   @Test
   public void testSpecific() throws Exception {
     AvroRecord before = new AvroRecord();
-    before.intField = 5;
+    before.setIntField(5);
     AvroRecord after = SerializationTestUtil.testSerialization(conf, before);
     assertEquals(before, after);
   }

diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -131,6 +131,19 @@
         <artifactId>maven-antrun-plugin</artifactId>
         <executions>
           <execution>
+            <id>generate-sources-ant-replace</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <tasks>
+                <replace dir="${project.build.directory}/generated-sources/avro" token="org.apache.avro" value="org.apache.hadoop.thirdparty.avro"/>
+              </tasks>
+            </configuration>
+          </execution>
+          <execution>
+            <id>pre-site-ant-copy</id>
             <phase>pre-site</phase>
             <goals>
               <goal>run</goal>

diff --git a/...client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java b/...client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
@@ -20,8 +20,8 @@
 
 import java.util.Set;
 
-import org.apache.hadoop.thirdparty.avro.util.Utf8;
 import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.thirdparty.avro.util.Utf8;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
@@ -31,8 +31,8 @@ public class JobQueueChangeEvent implements HistoryEvent {
   private JobQueueChange datum = new JobQueueChange();
 
   public JobQueueChangeEvent(JobID id, String queueName) {
-    datum.jobid = new Utf8(id.toString());
-    datum.jobQueueName = new Utf8(queueName);
+    datum.setJobid(new Utf8(id.toString()));
+    datum.setJobQueueName(new Utf8(queueName));
   }
 
   JobQueueChangeEvent() { }
@@ -54,13 +54,13 @@ public void setDatum(Object datum) {
 
   /** Get the Job ID */
   public JobID getJobId() {
-    return JobID.forName(datum.jobid.toString());
+    return JobID.forName(datum.getJobid().toString());
   }
 
   /** Get the new Job queue name */
   public String getJobQueueName() {
-    if (datum.jobQueueName != null) {
-      return datum.jobQueueName.toString();
+    if (datum.getJobQueueName() != null) {
+      return datum.getJobQueueName().toString();
     }
     return null;
   }

diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -42,7 +42,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop.thirdparty</groupId>
-      <artifactId>hadoop-shaded-avro</artifactId>
+      <artifactId>hadoop-shaded-avro_1_11</artifactId>
       <exclusions>
         <exclusion>
           <groupId>org.slf4j</groupId>

diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
@@ -43,7 +43,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop.thirdparty</groupId>
-      <artifactId>hadoop-shaded-avro</artifactId>
+      <artifactId>hadoop-shaded-avro_1_11</artifactId>
       <exclusions>
         <exclusion>
           <groupId>org.slf4j</groupId>

diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
@@ -57,7 +57,8 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop.thirdparty</groupId>
-      <artifactId>hadoop-shaded-avro_1_11</artifactId>
+      <artifactId>hadoop-shaded-avro_1.11</artifactId>
+      <version>${hadoop-thirdparty.avro.version}</version>
       <exclusions>
         <exclusion>
           <groupId>org.slf4j</groupId>

diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
@@ -62,8 +62,8 @@
     <hadoop.common.build.dir>${basedir}/../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir>
     <java.security.egd>file:///dev/urandom</java.security.egd>
 
-    <!-- avro version -->
-    <hadoop-shaded-avro.version>1.2.0-SNAPSHOT</hadoop-shaded-avro.version>
+    <!-- avro version: avro.version needs to match avro version used in hadoop-shaded-avro -->
+    <avro.version>1.11.3</avro.version>
 
     <!-- jersey version -->
     <jersey.version>1.19.4</jersey.version>
@@ -96,7 +96,9 @@
     <hadoop.protobuf.version>3.21.12</hadoop.protobuf.version>
     <protoc.path>${env.HADOOP_PROTOC_PATH}</protoc.path>
 
+
     <hadoop-thirdparty.version>1.2.0</hadoop-thirdparty.version>
+    <hadoop-thirdparty.avro.version>${hadoop-thirdparty.version}</hadoop-thirdparty.avro.version>
     <hadoop-thirdparty-protobuf.version>${hadoop-thirdparty.version}</hadoop-thirdparty-protobuf.version>
     <hadoop-thirdparty-guava.version>${hadoop-thirdparty.version}</hadoop-thirdparty-guava.version>
     <hadoop-thirdparty-shaded-prefix>org.apache.hadoop.thirdparty</hadoop-thirdparty-shaded-prefix>
@@ -1325,8 +1327,8 @@
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop.thirdparty</groupId>
-        <artifactId>hadoop-shaded-avro</artifactId>
-        <version>${hadoop-shaded-avro.version}</version>
+        <artifactId>hadoop-shaded-avro_1_11</artifactId>
+        <version>${hadoop-thirdparty.avro.version}</version>
       </dependency>
       <dependency>
         <groupId>net.sf.kosmosfs</groupId>
@@ -2086,9 +2088,9 @@
           <version>${maven-source-plugin.version}</version>
         </plugin>
         <plugin>
-          <groupId>org.apache.hadoop.thirdparty</groupId>
-          <artifactId>hadoop-shaded-avro-maven-plugin</artifactId>
-          <version>${hadoop-shaded-avro.version.version}</version>
+          <groupId>org.apache.avroy</groupId>
+          <artifactId>avro-maven-plugin</artifactId>
+          <version>${avro.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>

diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java
@@ -89,7 +89,7 @@ public class JobBuilder {
   private Map<ParsedHost, ParsedHost> allHosts =
       new HashMap<ParsedHost, ParsedHost>();
 
-  private org.apache.hadoop.mapreduce.jobhistory.JhCounters EMPTY_COUNTERS =
+  private final static org.apache.hadoop.mapreduce.jobhistory.JhCounters EMPTY_COUNTERS =
       new org.apache.hadoop.mapreduce.jobhistory.JhCounters();
 
   /**
@@ -460,7 +460,7 @@ private void processTaskFinishedEvent(TaskFinishedEvent event) {
     }
     task.setFinishTime(event.getFinishTime());
     task.setTaskStatus(getPre21Value(event.getTaskStatus()));
-    task.incorporateCounters(((TaskFinished) event.getDatum()).counters);
+    task.incorporateCounters(((TaskFinished) event.getDatum()).getCounters());
   }
 
   private void processTaskFailedEvent(TaskFailedEvent event) {
@@ -472,7 +472,7 @@ private void processTaskFailedEvent(TaskFailedEvent event) {
     task.setFinishTime(event.getFinishTime());
     task.setTaskStatus(getPre21Value(event.getTaskStatus()));
     TaskFailed t = (TaskFailed)(event.getDatum());
-    task.putDiagnosticInfo(t.error.toString());
+    task.putDiagnosticInfo(t.getError().toString());
     // killed task wouldn't have failed attempt.
     if (t.getFailedDueToAttempt() != null) {
       task.putFailedDueToAttemptId(t.getFailedDueToAttempt().toString());
@@ -542,7 +542,7 @@ private void processTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) {
     }
     attempt.setFinishTime(event.getFinishTime());
     attempt
-        .incorporateCounters(((TaskAttemptFinished) event.getDatum()).counters);
+        .incorporateCounters(((TaskAttemptFinished) event.getDatum()).getCounters());
   }
 
   private void processReduceAttemptFinishedEvent(
@@ -568,7 +568,7 @@ private void processReduceAttemptFinishedEvent(
     attempt.setShuffleFinished(event.getShuffleFinishTime());
     attempt.setSortFinished(event.getSortFinishTime());
     attempt
-        .incorporateCounters(((ReduceAttemptFinished) event.getDatum()).counters);
+        .incorporateCounters(((ReduceAttemptFinished) event.getDatum()).getCounters());
     attempt.arraySetClockSplits(event.getClockSplits());
     attempt.arraySetCpuUsages(event.getCpuUsages());
     attempt.arraySetVMemKbytes(event.getVMemKbytes());
@@ -596,7 +596,7 @@ private void processMapAttemptFinishedEvent(MapAttemptFinishedEvent event) {
     // is redundant, but making this will add future-proofing.
     attempt.setFinishTime(event.getFinishTime());
     attempt
-      .incorporateCounters(((MapAttemptFinished) event.getDatum()).counters);
+      .incorporateCounters(((MapAttemptFinished) event.getDatum()).getCounters());
     attempt.arraySetClockSplits(event.getClockSplits());
     attempt.arraySetCpuUsages(event.getCpuUsages());
     attempt.arraySetVMemKbytes(event.getVMemKbytes());
@@ -661,11 +661,11 @@ private void processJobFinishedEvent(JobFinishedEvent event) {
 
     JobFinished job = (JobFinished)event.getDatum();
     Map<String, Long> countersMap =
-        JobHistoryUtils.extractCounters(job.totalCounters);
+        JobHistoryUtils.extractCounters(job.getTotalCounters());
     result.putTotalCounters(countersMap);
-    countersMap = JobHistoryUtils.extractCounters(job.mapCounters);
+    countersMap = JobHistoryUtils.extractCounters(job.getMapCounters());
     result.putMapCounters(countersMap);
-    countersMap = JobHistoryUtils.extractCounters(job.reduceCounters);
+    countersMap = JobHistoryUtils.extractCounters(job.getReduceCounters());
     result.putReduceCounters(countersMap);
   }
 

diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobHistoryUtils.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobHistoryUtils.java
@@ -157,9 +157,9 @@ static boolean isJobConfXml(String fileName) {
   static Map<String, Long> extractCounters(JhCounters counters) {
     Map<String, Long> countersMap = new HashMap<String, Long>();
     if (counters != null) {
-      for (JhCounterGroup group : counters.groups) {
-        for (JhCounter counter : group.counts) {
-          countersMap.put(counter.name.toString(), counter.value);
+      for (JhCounterGroup group : counters.getGroups()) {
+        for (JhCounter counter : group.getCounts()) {
+          countersMap.put(counter.getName().toString(), counter.getValue());
         }
       }
     }

diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTask.java
@@ -268,11 +268,11 @@ private static void incorporateCounter(SetField thunk, JhCounters counters,
       String counterName) {
     counterName = canonicalizeCounterName(counterName);
 
-    for (JhCounterGroup group : counters.groups) {
-      for (JhCounter counter : group.counts) {
+    for (JhCounterGroup group : counters.getGroups()) {
+      for (JhCounter counter : group.getCounts()) {
         if (counterName
-            .equals(canonicalizeCounterName(counter.name.toString()))) {
-          thunk.set(counter.value);
+            .equals(canonicalizeCounterName(counter.getName().toString()))) {
+          thunk.set(counter.getValue());
           return;
         }
       }

diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
@@ -636,11 +636,11 @@ private static void incorporateCounter(SetField thunk, JhCounters counters,
       String counterName) {
     counterName = canonicalizeCounterName(counterName);
 
-    for (JhCounterGroup group : counters.groups) {
-      for (JhCounter counter : group.counts) {
+    for (JhCounterGroup group : counters.getGroups()) {
+      for (JhCounter counter : group.getCounts()) {
         if (counterName
-            .equals(canonicalizeCounterName(counter.name.toString()))) {
-          thunk.set(counter.value);
+            .equals(canonicalizeCounterName(counter.getName().toString()))) {
+          thunk.set(counter.getValue());
           return;
         }
       }