add tooling to enable quick release testing

JanusGraph · Feb 21, 2017 · af06612 · af06612
1 parent 236dd93
commit af06612
Show file tree

Hide file tree

Showing 7 changed files with 532 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -13,6 +13,7 @@ target/
 /log/
 /output/
 /scripts/
+/codepipelines-ci/dependency-reduced-pom.xml
 # When executing tests in alphabetical order, Maven generates temporary
 # files with names like this:
 #
@@ -26,6 +27,9 @@ target/
 /.settings/
 /.project
 /.classpath
+/codepipelines-ci/.classpath
+/codepipelines-ci/.project
+/codepipelines-ci/.settings/
 /janusgraph-*/.classpath
 /janusgraph-*/.project
 /janusgraph-*/.settings/

diff --git a/AUTHORS.txt b/AUTHORS.txt
@@ -11,6 +11,7 @@
 
 # Please keep the list sorted.
 
+Amazon
 DataStax
 Dylan Bethune-Waddell <[email protected]>
 Expero

diff --git a/buildspec.yml b/buildspec.yml
@@ -0,0 +1,17 @@
+version: 0.1
+
+phases:
+  build:
+    commands:
+      - echo Build started on `date`
+      - mvn install -DskipTests=true -B -pl $MODULE -am -Dmaven.compiler.showWarnings=false -Dmaven.javadoc.skip=true
+      - mvn verify -pl $MODULE $ARGS
+      - mvn surefire-report:report -pl $MODULE
+  post_build:
+    commands:
+      - echo Build completed on `date`
+      - ln -s ./$MODULE ./module 
+artifacts:
+  base-directory: module
+  files:
+    - target/**/*
diff --git a/codepipelines-ci/README.md b/codepipelines-ci/README.md
@@ -0,0 +1,27 @@
+Follow the procedure below to run all release testing in parallel.
+This procedure requires you to have an AWS account.
+It also requires you to create two service roles in IAM: one for CodePipeline and
+one for CodeBuild.
+
+Steps:
+
+1. Get a personal access token from [GitHub](https://github.com/settings/tokens) with repo and admin:repo_hook scopes.
+2. `mvn package`
+3. Navigate to the [AWS Console](https://console.aws.amazon.com) and create an IAM User with the following managed policies: AmazonS3FullAccess, AWSCodePipelineFullAccess, AWSCodeBuildAdminAccess.
+4. For this user, create security credentials and then register them in the `code-pipelines` profile on your computer with `aws configure --profile code-pipelines`.
+5. Create a service role for CodeBuild as described [here](http://docs.aws.amazon.com/codebuild/latest/userguide/setting-up.html#setting-up-service-role).
+6. Run the program to create the stack and start a build:
+```bash
+java -jar target/janusgraph-codepipelines-ci-0.1.0-SNAPSHOT.jar --region <region close to you> \
+--bucket <bucket> \
+--name <name of ci stack> \
+--github-owner <your github username> \
+--github-repo janusgraph \
+--github-branch <branch you want to test> \
+--profile code-pipelines \
+--environments=./environments.json \
+--codebuild-role-arn <AWS CodeBuild Service Role ARN> \
+--codepipeline-role-arn <AWS CodePipeline Service Role ARN> \
+--github-token <token>
+```
+7. Navigate to the [AWS Console](https://console.aws.amazon.com) and check on the status of your pipeline to see status.
diff --git a/codepipelines-ci/pipelines.json b/codepipelines-ci/pipelines.json
@@ -0,0 +1,28 @@
+{
+    "j1" : {
+	    "bdb" : { "MODULE" : "janusgraph-berkeleyje" },
+	    "cassandra" : { "MODULE" : "janusgraph-cassandra" },
+	    "es" : { "MODULE" : "janusgraph-es", "ARGS" : "-DthreadCount=1" },
+	    "hadoop" : { "MODULE" : "janusgraph-hadoop-parent/janusgraph-hadoop-2" },
+	    "hbase09" : { "MODULE" : "janusgraph-hbase-parent/janusgraph-hbase-098" }
+    },
+    "j2" : {
+	    "hbase10" : { "MODULE" : "janusgraph-hbase-parent/janusgraph-hbase-10" },
+	    "lucene" : { "MODULE" : "janusgraph-lucene", "ARGS" : "-DthreadCount=1" },
+	    "solr" : { "MODULE" : "janusgraph-solr", "ARGS" : "-DthreadCount=1" },
+	    "test" : { "MODULE" : "janusgraph-test" }
+    },
+    "jtp1" : {
+    	"bdb-tp" : { "MODULE" : "janusgraph-berkeleyje", "ARGS" : "-Dtest.skip.tp=false -DskipTests=true" },
+        "cassandra-tp" : { "MODULE" : "janusgraph-cassandra", "ARGS" : "-Dtest.skip.tp=false -DskipTests=true" },
+    	"es-tp" : { "MODULE" : "janusgraph-es", "ARGS" : "-DthreadCount=1 -Dtest.skip.tp=false -DskipTests=true" },
+    	"hadoop-tp" : { "MODULE" : "janusgraph-hadoop-parent/janusgraph-hadoop-2", "ARGS" : "-Dtest.skip.tp=false -DskipTests=true" },
+    	"hbase09-tp" : { "MODULE" : "janusgraph-hbase-parent/janusgraph-hbase-098", "ARGS" : "-Dtest.skip.tp=false -DskipTests=true" }
+    },
+    "jtp2" : {
+	    "hbase10-tp" : { "MODULE" : "janusgraph-hbase-parent/janusgraph-hbase-10", "ARGS" : "-Dtest.skip.tp=false -DskipTests=true" },
+	    "lucene-tp" : { "MODULE" : "janusgraph-lucene", "ARGS" : "-DthreadCount=1 -Dtest.skip.tp=false -DskipTests=true" },
+	    "solr-tp" : { "MODULE" : "janusgraph-solr", "ARGS" : "-DthreadCount=1 -Dtest.skip.tp=false -DskipTests=true" },
+	    "test-tp" : { "MODULE" : "janusgraph-test", "ARGS" : "-Dtest.skip.tp=false -DskipTests=true" }
+    }
+}
diff --git a/codepipelines-ci/pom.xml b/codepipelines-ci/pom.xml
@@ -0,0 +1,165 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.janusgraph</groupId>
+        <artifactId>janusgraph</artifactId>
+        <version>0.1.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <artifactId>janusgraph-codepipelines-ci</artifactId>
+    <name>JanusGraph-CodePipeline CI: Distributed Graph Database</name>
+    <url>http://janusgraph.org</url>
+    <properties>
+        <top.level.basedir>${basedir}/..</top.level.basedir>
+        <aws.sdk.version>1.11.86</aws.sdk.version>
+        <guava.version>21.0</guava.version>
+        <project.http.version>1.22.0</project.http.version>
+        <maven-compiler-plugin.version>3.6.1</maven-compiler-plugin.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-codepipeline</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-codebuild</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-iam</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-sts</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-dynamodb</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.16.14</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+    <build>
+        <resources>
+            <resource>
+                <directory>${basedir}/src/main/resources</directory>
+                <filtering>true</filtering>
+            </resource>
+        </resources>
+        <testResources>
+            <testResource>
+                <directory>${basedir}/src/test/resources</directory>
+            </testResource>
+        </testResources>
+
+        <plugins>
+            <plugin>
+                <artifactId>maven-jar-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>pack-test-jar</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+                  <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>3.0.0</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <transformers>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                  <mainClass>org.janusgraph.codepipelines.AwsCodePipelinesCi</mainClass>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+        </plugins>
+    </build>
+    <profiles>
+        <profile>
+            <id>janusgraph-release</id>
+
+            <build>
+                <plugins>
+                    <!-- Redeclare gpg-plugin after jar-plugin to force the
+                        test jar to be signed. gpg-plugin runs in the package phase, same as jar-plugin,
+                        so the only constraint that guarantees the jar will exist when gpg-plugin
+                        is invoked is declaration order in the pom. -->
+                    <plugin>
+                        <artifactId>maven-jar-plugin</artifactId>
+                    </plugin>
+                    <plugin>
+                        <artifactId>maven-gpg-plugin</artifactId>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+                <version>2.6.6</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-core</artifactId>
+                <version>2.6.6</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-annotations</artifactId>
+                <version>2.6.6</version>
+            </dependency>
+            <dependency>
+                <groupId>joda-time</groupId>
+                <artifactId>joda-time</artifactId>
+                <version>2.8.1</version>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+</project>