Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HADOOP-18950: shaded avro jar #4854

Open
wants to merge 2 commits into
base: trunk
Choose a base branch
from
+174 −129
Open

HADOOP-18950: shaded avro jar #4854

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a8ecfbb
HADOOP-18950. Use shaded avro jar
pjfanning Sep 4, 2022
c916623
Merge branch 'trunk' into avro
pjfanning Mar 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion LICENSE-binary
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,6 @@ io.swagger:swagger-annotations:1.5.4
javax.inject:javax.inject:1
net.java.dev.jna:jna:5.2.0
net.minidev:accessors-smart:1.2
org.apache.avro:avro:1.9.2
org.apache.commons:commons-collections4:4.2
org.apache.commons:commons-compress:1.24.0
org.apache.commons:commons-configuration2:2.8.0
Expand Down
4 changes: 4 additions & 0 deletions hadoop-client-modules/hadoop-client-minicluster/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,10 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-avro_1_11</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
Expand Down
16 changes: 16 additions & 0 deletions hadoop-client-modules/hadoop-client/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,10 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-avro_1_11</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
Expand Down Expand Up @@ -166,6 +170,10 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-avro_1_11</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
Expand Down Expand Up @@ -278,6 +286,10 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-avro_1_11</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
Expand Down Expand Up @@ -314,6 +326,10 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-avro_1_11</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
Expand Down
6 changes: 4 additions & 2 deletions hadoop-common-project/hadoop-common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,8 +249,8 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-avro_1_11</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
Expand Down Expand Up @@ -617,6 +617,8 @@
<copy toDir="${project.build.directory}/test-classes">
<fileset dir="${basedir}/src/main/conf"/>
</copy>

<replace dir="${project.build.directory}/generated-test-sources/java" token="org.apache.avro" value="org.apache.hadoop.thirdparty.avro"/>
</target>
</configuration>
</execution>
Expand Down
2 changes: 1 addition & 1 deletion hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import java.io.Closeable;
import java.io.IOException;

import org.apache.avro.file.SeekableInput;
import org.apache.hadoop.thirdparty.avro.file.SeekableInput;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

again, this is a public class we don't use internally.

Should we actually deprecate it? I don't know what uses it?

import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

Expand Down
2 changes: 1 addition & 1 deletion hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import java.util.Optional;
import java.util.regex.Pattern;

import org.apache.avro.reflect.Stringable;
import org.apache.hadoop.thirdparty.avro.reflect.Stringable;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This could be dangerous, as we are saying that a public class can no longer be serialised through Avro.

Do you think it will be possible for us to retain the unshaded annotation as well as adding the new one? And still have everything to work without Avro on the CP?

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
Expand Down
16 changes: 8 additions & 8 deletions hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@
import java.util.LinkedList;
import java.util.zip.GZIPInputStream;

import org.apache.avro.Schema;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.FileReader;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.io.JsonEncoder;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.file.DataFileReader;
import org.apache.hadoop.thirdparty.avro.file.FileReader;
import org.apache.hadoop.thirdparty.avro.generic.GenericDatumReader;
import org.apache.hadoop.thirdparty.avro.generic.GenericDatumWriter;
import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
import org.apache.hadoop.thirdparty.avro.io.EncoderFactory;
import org.apache.hadoop.thirdparty.avro.io.JsonEncoder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
Expand Down
2 changes: 1 addition & 1 deletion hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
import java.text.StringCharacterIterator;
import java.util.Arrays;

import org.apache.avro.reflect.Stringable;
import org.apache.hadoop.thirdparty.avro.reflect.Stringable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

Expand Down
12 changes: 6 additions & 6 deletions ...p-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@
import java.util.HashSet;
import java.util.Set;

import org.apache.avro.Schema;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.reflect.ReflectData;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.reflect.ReflectDatumWriter;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.DatumReader;
import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectData;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumReader;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumWriter;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

Expand Down
14 changes: 7 additions & 7 deletions ...t/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@
import java.io.InputStream;
import java.io.OutputStream;

import org.apache.avro.Schema;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.EncoderFactory;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.BinaryDecoder;
import org.apache.hadoop.thirdparty.avro.io.BinaryEncoder;
import org.apache.hadoop.thirdparty.avro.io.DatumReader;
import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
import org.apache.hadoop.thirdparty.avro.io.DecoderFactory;
import org.apache.hadoop.thirdparty.avro.io.EncoderFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configured;
Expand Down
12 changes: 6 additions & 6 deletions ...-common/src/main/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@

package org.apache.hadoop.io.serializer.avro;

import org.apache.avro.Schema;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.avro.specific.SpecificRecord;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.DatumReader;
import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumReader;
import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumWriter;
import org.apache.hadoop.thirdparty.avro.specific.SpecificRecord;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;

Expand Down
2 changes: 1 addition & 1 deletion ...adoop-common/src/main/java/org/apache/hadoop/security/token/delegation/DelegationKey.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@

import javax.crypto.SecretKey;

import org.apache.avro.reflect.Nullable;
import org.apache.hadoop.thirdparty.avro.reflect.Nullable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
Expand Down
12 changes: 6 additions & 6 deletions hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/AvroTestUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@
import java.io.ByteArrayOutputStream;
import java.lang.reflect.Type;

import org.apache.avro.Schema;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.reflect.ReflectData;
import org.apache.avro.reflect.ReflectDatumWriter;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.io.DecoderFactory;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.EncoderFactory;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectData;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumWriter;
import org.apache.hadoop.thirdparty.avro.reflect.ReflectDatumReader;
import org.apache.hadoop.thirdparty.avro.io.DecoderFactory;

import static org.junit.Assert.assertEquals;

Expand Down
2 changes: 1 addition & 1 deletion ...doop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public class TestAvroSerialization {
@Test
public void testSpecific() throws Exception {
AvroRecord before = new AvroRecord();
before.intField = 5;
before.setIntField(5);
AvroRecord after = SerializationTestUtil.testSerialization(conf, before);
assertEquals(before, after);
}
Expand Down
13 changes: 13 additions & 0 deletions hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,19 @@
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>generate-sources-ant-replace</id>
<phase>generate-sources</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<replace dir="${project.build.directory}/generated-sources/avro" token="org.apache.avro" value="org.apache.hadoop.thirdparty.avro"/>
</tasks>
</configuration>
</execution>
<execution>
<id>pre-site-ant-copy</id>
<phase>pre-site</phase>
<goals>
<goal>run</goal>
Expand Down
2 changes: 1 addition & 1 deletion ...duce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.StringUtils;
Expand Down
4 changes: 2 additions & 2 deletions ...duce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import java.util.Iterator;
import java.util.List;

import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.generic.GenericData;

public class AvroArrayUtils {

Expand Down
14 changes: 7 additions & 7 deletions ...preduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@
import java.io.EOFException;
import java.io.IOException;

import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Schema;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.specific.SpecificData;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.hadoop.thirdparty.avro.AvroRuntimeException;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.DatumReader;
import org.apache.hadoop.thirdparty.avro.io.Decoder;
import org.apache.hadoop.thirdparty.avro.io.DecoderFactory;
import org.apache.hadoop.thirdparty.avro.specific.SpecificData;
import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumReader;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileSystem;
Expand Down
12 changes: 6 additions & 6 deletions ...preduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@
import java.io.IOException;
import java.util.ArrayList;

import org.apache.avro.Schema;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.Schema;
import org.apache.hadoop.thirdparty.avro.io.DatumWriter;
import org.apache.hadoop.thirdparty.avro.io.Encoder;
import org.apache.hadoop.thirdparty.avro.io.EncoderFactory;
import org.apache.hadoop.thirdparty.avro.specific.SpecificDatumWriter;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FSDataOutputStream;
Expand Down
2 changes: 1 addition & 1 deletion ...ce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.Counters;
Expand Down
2 changes: 1 addition & 1 deletion ...-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
Expand Down
2 changes: 1 addition & 1 deletion ...duce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
Expand Down
2 changes: 1 addition & 1 deletion ...ent-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.JobPriority;
Expand Down
12 changes: 6 additions & 6 deletions ...client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
Expand All @@ -31,8 +31,8 @@ public class JobQueueChangeEvent implements HistoryEvent {
private JobQueueChange datum = new JobQueueChange();

public JobQueueChangeEvent(JobID id, String queueName) {
datum.jobid = new Utf8(id.toString());
datum.jobQueueName = new Utf8(queueName);
datum.setJobid(new Utf8(id.toString()));
datum.setJobQueueName(new Utf8(queueName));
}

JobQueueChangeEvent() { }
Expand All @@ -54,13 +54,13 @@ public void setDatum(Object datum) {

/** Get the Job ID */
public JobID getJobId() {
return JobID.forName(datum.jobid.toString());
return JobID.forName(datum.getJobid().toString());
}

/** Get the new Job queue name */
public String getJobQueueName() {
if (datum.jobQueueName != null) {
return datum.jobQueueName.toString();
if (datum.getJobQueueName() != null) {
return datum.getJobQueueName().toString();
}
return null;
}
Expand Down
2 changes: 1 addition & 1 deletion ...ient-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.Set;

import org.apache.avro.util.Utf8;
import org.apache.hadoop.thirdparty.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
Expand Down
Loading
Loading