Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@pjfanning
pjfanning committed Feb 9, 2024
1 parent 50d256e commit 22d5d20
Show file tree
Hide file tree
Showing 37 changed files with 112 additions and 105 deletions.
1 change: 0 additions & 1 deletion LICENSE-binary
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,6 @@ javax.inject:javax.inject:1
log4j:log4j:1.2.17
net.java.dev.jna:jna:5.2.0
net.minidev:accessors-smart:1.2
org.apache.avro:avro:1.9.2
org.apache.commons:commons-collections4:4.2
org.apache.commons:commons-compress:1.24.0
org.apache.commons:commons-configuration2:2.8.0
Expand Down
4 changes: 2 additions & 2 deletions hadoop-common-project/hadoop-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,8 +249,8 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-avro</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
Expand Down
2 changes: 1 addition & 1 deletion hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import java.io.Closeable;
import java.io.IOException;

import org.apache.avro.file.SeekableInput;
import org.apache.hadoop.thirdparty.avro.file.SeekableInput;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

Expand Down
2 changes: 1 addition & 1 deletion hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import java.util.Optional;
import java.util.regex.Pattern;

import org.apache.avro.reflect.Stringable;
import org.apache.hadoop.thirdparty.avro.reflect.Stringable;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
Expand Down
16 changes: 8 additions & 8 deletions hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@
import java.util.LinkedList;
import java.util.zip.GZIPInputStream;

import org.apache.avro.Schema;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.FileReader;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.io.JsonEncoder;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.file.DataFileReader;
import org.apache.hadoop.thirdparty.avro.file.FileReader;
import org.apache.hadoop.thirdparty.avro.generic.GenericDatumReader;
import org.apache.hadoop.thirdparty.avro.generic.GenericDatumWriter;
import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
import org.apache.hadoop.thirdparty.avro.io.EncoderFactory;
import org.apache.hadoop.thirdparty.avro.io.JsonEncoder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
Expand Down
2 changes: 1 addition & 1 deletion hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
import java.text.StringCharacterIterator;
import java.util.Arrays;

import org.apache.avro.reflect.Stringable;
import org.apache.hadoop.thirdparty.avro.reflect.Stringable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

Expand Down
12 changes: 6 additions & 6 deletions ...p-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@
import java.util.HashSet;
import java.util.Set;

import org.apache.avro.Schema;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.reflect.ReflectData;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.reflect.ReflectDatumWriter;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.DatumReader;
import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectData;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumReader;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumWriter;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

Expand Down
14 changes: 7 additions & 7 deletions ...t/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@
import java.io.InputStream;
import java.io.OutputStream;

import org.apache.avro.Schema;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.EncoderFactory;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.BinaryDecoder;
import org.apache.hadoop.thirdparty.avro.io.BinaryEncoder;
import org.apache.hadoop.thirdparty.avro.io.DatumReader;
import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
import org.apache.hadoop.thirdparty.avro.io.DecoderFactory;
import org.apache.hadoop.thirdparty.avro.io.EncoderFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configured;
Expand Down
12 changes: 6 additions & 6 deletions ...-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@

package org.apache.hadoop.io.serializer.avro;

import org.apache.avro.Schema;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.avro.specific.SpecificRecord;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.DatumReader;
import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumReader;
import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumWriter;
import org.apache.hadoop.thirdparty.avro.specific.SpecificRecord;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

Expand Down
2 changes: 1 addition & 1 deletion ...adoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationKey.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

import javax.crypto.SecretKey;

import org.apache.avro.reflect.Nullable;
import org.apache.hadoop.thirdparty.avro.reflect.Nullable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
Expand Down
12 changes: 6 additions & 6 deletions hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@
import java.io.ByteArrayOutputStream;
import java.lang.reflect.Type;

import org.apache.avro.Schema;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.reflect.ReflectData;
import org.apache.avro.reflect.ReflectDatumWriter;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.io.DecoderFactory;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.EncoderFactory;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectData;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumWriter;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumReader;
import org.apache.hadoop.thirdparty.avro.io.DecoderFactory;

import static org.junit.Assert.assertEquals;

Expand Down
2 changes: 1 addition & 1 deletion ...duce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.StringUtils;
Expand Down
4 changes: 2 additions & 2 deletions ...duce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import java.util.Iterator;
import java.util.List;

import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.generic.GenericData;

public class AvroArrayUtils {

Expand Down
14 changes: 7 additions & 7 deletions ...preduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@
import java.io.EOFException;
import java.io.IOException;

import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Schema;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.specific.SpecificData;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.hadoop.thirdparty.avro.AvroRuntimeException;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.DatumReader;
import org.apache.hadoop.thirdparty.avro.io.Decoder;
import org.apache.hadoop.thirdparty.avro.io.DecoderFactory;
import org.apache.hadoop.thirdparty.avro.specific.SpecificData;
import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumReader;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileSystem;
Expand Down
12 changes: 6 additions & 6 deletions ...preduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@
import java.io.IOException;
import java.util.ArrayList;

import org.apache.avro.Schema;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
import org.apache.hadoop.thirdparty.avro.io.Encoder;
import org.apache.hadoop.thirdparty.avro.io.EncoderFactory;
import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumWriter;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FSDataOutputStream;
Expand Down
2 changes: 1 addition & 1 deletion ...ce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.Counters;
Expand Down
2 changes: 1 addition & 1 deletion ...-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
Expand Down
2 changes: 1 addition & 1 deletion ...duce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
Expand Down
2 changes: 1 addition & 1 deletion ...ent-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.JobPriority;
Expand Down
2 changes: 1 addition & 1 deletion ...client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
Expand Down
2 changes: 1 addition & 1 deletion ...ient-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
Expand Down
2 changes: 1 addition & 1 deletion ...e-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import java.util.Map.Entry;
import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.JobConf;
Expand Down
2 changes: 1 addition & 1 deletion .../src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import java.util.Collections;
import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
Expand Down
2 changes: 1 addition & 1 deletion ...nt-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.ProgressSplitsBlock;
Expand Down
2 changes: 1 addition & 1 deletion ...core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.ProgressSplitsBlock;
Expand Down
2 changes: 1 addition & 1 deletion ...t-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.Counters;
Expand Down
2 changes: 1 addition & 1 deletion ...nt-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskAttemptID;
Expand Down
2 changes: 1 addition & 1 deletion ...n/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.ProgressSplitsBlock;
Expand Down
2 changes: 1 addition & 1 deletion ...uce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.TaskStatus;
Expand Down
2 changes: 1 addition & 1 deletion ...e-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.TaskStatus;
Expand Down
2 changes: 1 addition & 1 deletion ...ce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskID;
Expand Down
2 changes: 1 addition & 1 deletion ...ce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskID;
Expand Down
2 changes: 1 addition & 1 deletion ...-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

import org.junit.Assert;

import org.apache.avro.AvroRemoteException;
import org.apache.hadoop.thirdparty.avro.AvroRemoteException;
import org.apache.hadoop.mapreduce.SleepJob;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
Expand Down
14 changes: 12 additions & 2 deletions hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,18 @@
<artifactId>hadoop-mapreduce-client-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-avro</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
Loading

0 comments on commit 22d5d20

Please sign in to comment.