[CORE] Introduce custom protobuf dependency with recursion limit enla…

…rged (#3814)
apache · Nov 24, 2023 · 3520c9e · 3520c9e
1 parent 4f63ddc
commit 3520c9e
Show file tree

Hide file tree

Showing 8 changed files with 38 additions and 5 deletions.
diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
@@ -20,6 +20,11 @@
       <version>${project.version}</version>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>io.glutenproject</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>${custom.protobuf.version}</version>
+    </dependency>
     <dependency>
       <groupId>io.glutenproject</groupId>
       <artifactId>gluten-core</artifactId>

diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
@@ -38,6 +38,11 @@
       <version>${project.version}</version>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>io.glutenproject</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>${custom.protobuf.version}</version>
+    </dependency>
     <dependency>
       <groupId>io.glutenproject</groupId>
       <artifactId>gluten-core</artifactId>

diff --git a/gluten-celeborn/pom.xml b/gluten-celeborn/pom.xml
@@ -37,6 +37,12 @@
       <artifactId>spark-sql_${scala.binary.version}</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>${guava.version}</version>
+      <scope>provided</scope>
+    </dependency>
   </dependencies>
 
   <build>

diff --git a/gluten-core/pom.xml b/gluten-core/pom.xml
@@ -174,11 +174,12 @@
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
       <version>${protobuf.version}</version>
+      <scope>provided</scope>
     </dependency>
     <dependency>
-      <groupId>com.google.protobuf</groupId>
+      <groupId>io.glutenproject</groupId>
       <artifactId>protobuf-java-util</artifactId>
-      <version>${protobuf.version}</version>
+      <version>${custom.protobuf.version}</version>
     </dependency>
     <dependency>
       <groupId>commons-io</groupId>

diff --git a/gluten-data/pom.xml b/gluten-data/pom.xml
@@ -51,6 +51,17 @@
       <version>${project.version}</version>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>io.glutenproject</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>${custom.protobuf.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>${guava.version}</version>
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>io.glutenproject</groupId>
       <artifactId>gluten-core</artifactId>

diff --git a/package/pom.xml b/package/pom.xml
@@ -255,7 +255,8 @@
                     <ignoreClass>org.apache.spark.sql.execution.datasources.WriterBucketSpec$</ignoreClass>
                     <ignoreClass>org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand</ignoreClass>
                     <ignoreClass>org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand$</ignoreClass>
-
+                    <!-- protobuf -->
+                    <ignoreClass>com.google.protobuf.*</ignoreClass>
                   </ignoreClasses>
                   <scopes>
                     <scope>compile</scope>

diff --git a/pom.xml b/pom.xml
@@ -65,7 +65,11 @@
     <junit.version>4.13.1</junit.version>
 
     <substrait.version>0.5.0</substrait.version>
+    <!-- The original protobuf version -->
     <protobuf.version>3.23.4</protobuf.version>
+    <!-- The custom protobuf version (based on 3.23.4) with recursion limit enlarged. -->
+    <custom.protobuf.version>3.23.4-0</custom.protobuf.version>
+    <guava.version>32.0.1-jre</guava.version>
     <!--spotless-->
     <spotless.version>2.27.2</spotless.version>
     <spotless.scalafmt.version>3.5.9</spotless.scalafmt.version>

diff --git a/substrait/substrait-spark/pom.xml b/substrait/substrait-spark/pom.xml
@@ -46,9 +46,9 @@
       <artifactId>core</artifactId>
     </dependency>
     <dependency>
-      <groupId>com.google.protobuf</groupId>
+      <groupId>io.glutenproject</groupId>
       <artifactId>protobuf-java</artifactId>
-      <version>${protobuf.version}</version>
+      <version>${custom.protobuf.version}</version>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>