Skip to content

Commit

Permalink
[SYSTEMDS-3600] POM Shader bug
Browse files Browse the repository at this point in the history 
This commit fixes a long running bug in the compiling of our system
where when packaging multiple times in a row there was some
inconsistencies with the packed JAR and the shader plugin.
Where the shader plugin would complain. I found out it happens because
the JAR that the shader puts in Antler and Wink already contained
those from previous packing. This is now fixed with dedicated
Jar creation with the maven-jar-plugin, and we now pack the manifest
correctly from this jar-plugin instead of the 'hacky' way from before
with the shader transform packages.

Also included is the construction of a Test and Performance JAR.
These are commented more on in the following commit.

In the end the build directory jars look like:

systemds-3.2.0-SNAPSHOT-perf.jar
systemds-3.2.0-SNAPSHOT-tests.jar
systemds-3.2.0-SNAPSHOT-unshaded.jar
systemds-3.2.0-SNAPSHOT.jar <- this is shaded correctly always
SystemDS.jar
  • Loading branch information
@Baunsgaard
Baunsgaard committed Jul 19, 2023
1 parent 82d9d13 commit fb828dd
Showing 1 changed file with 117 additions and 134 deletions.
251 changes: 117 additions & 134 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@
<maven-assembly-plugin.version>3.5.0</maven-assembly-plugin.version>
<maven-dependency-plugin.version>3.5.0</maven-dependency-plugin.version>
<maven-jar-plugin.version>3.3.0</maven-jar-plugin.version>
<maven-shade-plugin.version>3.5.0</maven-shade-plugin.version>
<maven-compiler-plugin.version>3.11.0</maven-compiler-plugin.version>
<maven-antrun-plugin.version>3.1.0</maven-antrun-plugin.version>
<!-- Set java compile level via argument, ex: 1.8 1.9 10 11-->
<java.level>11</java.level>
<java.version>{java.level}</java.version>
Expand Down Expand Up @@ -159,7 +162,10 @@
<directory>src/test/config/hadoop_bin_windows/bin</directory>
<targetPath>../lib/hadoop/bin</targetPath>
</resource>

<resource>
<directory>src/assembly/bin</directory>
<targetPath>META-INF/</targetPath>
</resource>
</resources>
<testResources>
<testResource>
Expand All @@ -180,46 +186,82 @@
<version>${maven-dependency-plugin.version}</version>
<executions>
<execution>
<id>unpack</id>
<phase>compile</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<silent>true</silent>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin.version}</version>
<executions>
<execution>
<id>default-jar</id>
<phase>package</phase>
<goals>
<goal>unpack</goal>
<goal>jar</goal>
</goals>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-test</artifactId>
<version>1.2.1</version>
<type>jar</type>
<overWrite>true</overWrite>
<outputDirectory>${project.build.directory}/hadoop-test</outputDirectory>
</artifactItem>
</artifactItems>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>true</overWriteSnapshots>
<forceCreation>true</forceCreation>
<archive>
<manifest>
<mainClass>org.apache.sysds.api.DMLScript</mainClass>
</manifest>
</archive>
</configuration>
</execution>
<execution>
<phase>compile</phase>
<id>perf-jar</id>
<goals>
<goal>copy-dependencies</goal>
<goal>test-jar</goal>
</goals>
<configuration>
<silent>true</silent>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
<classifier>perf</classifier>
<includes>
<include>org/apache/sysds/performance/**</include>
<include>log4j.properties</include>
</includes>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>org.apache.sysds.performance.Main</mainClass>
</manifest>
<manifestEntries>
<Class-Path>SystemDS.jar SystemDS-tests.jar</Class-Path>
</manifestEntries>
</archive>
</configuration>
</execution>
<execution>
<id>test-jar</id>
<goals>
<goal>test-jar</goal>
</goals>
<configuration>
<includes>
<include>org/apache/sysds/test/**</include>
</includes>
</configuration>
</execution>
</executions>
</plugin>

<!-- PLEASE DO NOT REMOVE! NEEDED to "PACKAGE" ANTLR RUNTIME INTO SYSTEMDS.JAR -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.4.1</version>
<version>${maven-shade-plugin.version}</version>
<executions>
<execution>
<id>default-shade</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
Expand All @@ -231,41 +273,31 @@
<include>org.antlr:antlr4-runtime:*</include>
</includes>
</artifactSet>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.apache.sysds.api.DMLScript</mainClass>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>src/assembly/bin/LICENSE</file>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>NOTICE</file>
</transformer>
</transformers>
<createDependencyReducedPom>false</createDependencyReducedPom>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>shaded</shadedClassifierName>
</configuration>
</execution>
</executions>
<configuration>
<!-- Include signature files so that recent versions of Java will run
the resulting jar without complaining about "Invalid signature file digest
for Manifest main attributes".
Furthermore, the excluded notice and license files will be explicitly
added by the resource transformers above -->
<filters>
<filter>
<artifact>*:*</artifact>
<artifact>org.apache.wink:wink-json4j:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/DEPENDENCIES</exclude>
<exclude>META-INF/MANIFEST.MF</exclude>
<exclude>META-INF/LICENSE</exclude>
<exclude>META-INF/NOTICE</exclude>
<exclude>META-INF/MANIFEST.MF</exclude>
<!-- Exclude wink if already shaded -->
<!-- <exclude>org/apache/wink/**</exclude> -->
</excludes>
</filter>
<filter>
<artifact>org.antlr:antlr4-runtime:*</artifact>
<excludes>
<exclude>META-INF/MANIFEST.MF</exclude>
<!-- Exclude antlr4 if already shaded -->
<!-- <exclude>org/antlr/v4/**</exclude>
<exclude>META-INF/maven/org.antlr/**</exclude> -->
</excludes>
</filter>
</filters>
Expand All @@ -275,14 +307,15 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.11.0</version>
<version>${maven-compiler-plugin.version}</version>
<configuration>
<source>${java.level}</source>
<target>${java.level}</target>
<release>${java.level}</release>
</configuration>
</plugin>


<plugin>
<!-- unit tests -->
<groupId>org.apache.maven.plugins</groupId>
Expand All @@ -305,43 +338,55 @@
</plugin>

<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>${maven-clean-plugin.version}</version>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>${maven-antrun-plugin.version}</version>
<executions>
<execution>
<id>clean-original-jar</id>
<id>copy-sysds</id>
<phase>package</phase>
<configuration>
<target name="copy and rename JAR files">
<!-- https://ant.apache.org/manual/Tasks -->
<copy
file="${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar"
tofile="${project.build.directory}/SystemDS.jar" />
<move
file="${project.build.directory}/${project.artifactId}-${project.version}.jar"
tofile="${project.build.directory}/${project.artifactId}-${project.version}-unshaded.jar"/>

</target>
</configuration>
<goals>
<goal>clean</goal>
<goal>run</goal>
</goals>
</execution>
<execution>
<id>move-shaded-jar</id>
<phase>package</phase>
<configuration>
<excludeDefaultDirectories>true</excludeDefaultDirectories>
<filesets>
<fileset>
<directory>${project.build.directory}</directory>
<includes>
<include>original-*.jar</include>
</includes>
</fileset>
</filesets>
<target name="copy and rename JAR files">
<!-- https://ant.apache.org/manual/Tasks -->
<move
file="${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar"
tofile="${project.build.directory}/${project.artifactId}-${project.version}.jar"/>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<id>copy</id>
<id>copy-test-jar</id>
<phase>package</phase>
<configuration>
<target name="copy and rename JAR">
<target name="copy and rename test JAR" unless="maven.test.skip">
<copy
file="${project.build.directory}/${project.artifactId}-${project.version}.jar"
tofile="${project.build.directory}/SystemDS.jar" />
file="${project.build.directory}/${project.artifactId}-${project.version}-tests.jar"
tofile="${project.build.directory}/SystemDS-tests.jar" />
<copy
file="${project.build.directory}/${project.artifactId}-${project.version}-perf.jar"
tofile="${project.build.directory}/SystemDS-perf.jar" />
</target>
</configuration>
<goals>
Expand Down Expand Up @@ -407,31 +452,8 @@
</execution>
</executions>
</plugin>


<!--
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<goals>
<goal>testResources</goal>
</goals>
<phase>process-test-resources</phase>
<configuration>
<skip>true</skip>
</configuration>
</execution>
</executions>
</plugin> -->


</plugins>



</build>

<profiles>
Expand Down Expand Up @@ -634,9 +656,6 @@
<exclude>src/test/scripts/functions/jmlc/tfmtd_frame_example/tfmtd_frame</exclude>
<!-- csv test input not captured by *.csv -->
<exclude>src/test/scripts/functions/io/csv/in/*/*</exclude>
<!-- IOGEN input examples -->
<exclude>src/test/scripts/functions/iogen/in/*/*/*</exclude>
<exclude>src/test/scripts/functions/iogen/in/*/*</exclude>
<!-- Python bindings lineage test result comparison -->
<exclude>src/main/python/tests/lt*.txt</exclude>
<!-- Perftest requirement file -->
Expand Down Expand Up @@ -834,42 +853,6 @@
</plugins>
</build>
</profile>

<profile>
<id>test-jar</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin.version}</version>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>

<profile>
<id>skip-sign</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>${maven-gpg-plugin.version}</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>

<dependencies>
Expand Down

0 comments on commit fb828dd

Please sign in to comment.