Add tooling to enable quick release testing [skip ci]

Signed-off-by: Alexander Patrikalakis <[email protected]>
JanusGraph · Jun 8, 2017 · ccd391c · ccd391c
1 parent 32f468c
commit ccd391c
Show file tree

Hide file tree

Showing 15 changed files with 927 additions and 4 deletions.
diff --git a/buildspec.yml b/buildspec.yml
@@ -0,0 +1,17 @@
+version: 0.1
+
+phases:
+  build:
+    commands:
+      - echo Build started on `date`
+      - mvn install -DskipTests=true -B -pl $MODULE -am -Dmaven.compiler.showWarnings=false -Dmaven.javadoc.skip=true
+      - mvn verify -pl $MODULE $ARGS
+      - mvn surefire-report:report -pl $MODULE
+  post_build:
+    commands:
+      - echo Build completed on `date`
+      - ln -s ./$MODULE ./module 
+artifacts:
+  base-directory: module
+  files:
+    - target/**/*
diff --git a/janusgraph-berkeleyje/src/test/resources/release-test-groups/bdb1 b/janusgraph-berkeleyje/src/test/resources/release-test-groups/bdb1
@@ -0,0 +1 @@
+org.apache.tinkerpop.gremlin.process.TraversalPerformanceTest
diff --git a/janusgraph-berkeleyje/src/test/resources/release-test-groups/bdb2 b/janusgraph-berkeleyje/src/test/resources/release-test-groups/bdb2
@@ -0,0 +1,19 @@
+org.apache.tinkerpop.gremlin.process.traversal.TraversalInterruptionComputerTest
+org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$GreedyMatchTraversals
+org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$GreedyMatchTraversals
+org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$CountMatchTraversals
+org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$CountMatchTraversals
+org.apache.tinkerpop.gremlin.algorithm.generator.DistributionGeneratorTest$DifferentDistributionsTest
+org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTest$Traversals
+org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest$Traversals
+org.janusgraph.graphdb.berkeleyje.BerkeleyGraphTest
+org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals
+org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest$Traversals
+org.apache.tinkerpop.gremlin.structure.PropertyTest$PropertyFeatureSupportTest
+org.apache.tinkerpop.gremlin.structure.io.IoGraphTest
+org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyOrderTest$Traversals
+org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest$Traversals
+org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals
+org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyProfileTest$Traversals
+org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyProfileTest$Traversals
+org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals
diff --git a/janusgraph-berkeleyje/src/test/resources/release-test-groups/bdb3 b/janusgraph-berkeleyje/src/test/resources/release-test-groups/bdb3
diff --git a/janusgraph-codepipelines-ci/README.md b/janusgraph-codepipelines-ci/README.md
@@ -0,0 +1,36 @@
+#JanusGraph CodePipelines CI
+CodePipelines CI is a mechanism JanusGraph can use to do release testing in massively
+parallel fashion (to the extent of AWS CodePipelines and CodeBuild service limits).
+
+## Steps to kick off a build
+Follow the procedure below to run all release testing in parallel.
+This procedure requires you to have an AWS account.
+It also requires you to create two service roles in IAM: one for CodePipeline and
+one for CodeBuild.
+
+1. Get a personal access token from [GitHub](https://github.com/settings/tokens) with repo and admin:repo_hook scopes.
+2. Navigate to the [AWS Console](https://console.aws.amazon.com) and create an IAM User with the following managed policies: AmazonS3FullAccess, AWSCodePipelineFullAccess, AWSCodeBuildAdminAccess.
+3. For this user, create security credentials and then register them in the `code-pipelines` profile on your computer with `aws configure --profile code-pipelines`.
+4. Create a service role for CodeBuild as described [here](http://docs.aws.amazon.com/codebuild/latest/userguide/setting-up.html#setting-up-service-role).
+5. Create an IAM policy for a CodePipelines service role as described [here](http://docs.aws.amazon.com/codepipeline/latest/userguide/iam-identity-based-access-control.html#view-default-service-role-policy).
+6. Create a service role for CodePipelines by using the policy you created in step 5 and then set the STS trust to codepipeline.amazonaws.com, like what you did for CodeBuild.
+7. Build and run the program to create the stack and start a build:
+```bash
+mvn clean && mvn package
+java -jar target/janusgraph-codepipelines-ci-0.1.0-SNAPSHOT.jar \
+--region region_close_to_you \
+--bucket bucket_name \
+--github-owner your_github_username \
+--github-repo janusgraph \
+--github-branch branch_to_test \
+--profile code-pipelines \
+--codebuild-role-arn AWS_CodeBuild_Service_Role_ARN \
+--codepipeline-role-arn AWS_CodePipeline_Service_Role_ARN \
+--github-token token \
+--pipelines=./environments.json
+```
+8. Navigate to the [AWS Console](https://console.aws.amazon.com) and check on the status of your pipeline to see status.
+
+## Future ideas
+I would like to figure out a way to orchestrate builds but at the same time remove
+buildspec.yml from the root of the repository.
diff --git a/janusgraph-codepipelines-ci/pipe.json b/janusgraph-codepipelines-ci/pipe.json
@@ -0,0 +1,15 @@
+{
+    "j1pass" : {
+        "bdb" : { "MODULE" : "janusgraph-berkeleyje" },
+        "lucene" : { "MODULE" : "janusgraph-lucene" },
+        "solr" : { "MODULE" : "janusgraph-solr" },
+        "test" : { "MODULE" : "janusgraph-test" }
+    },
+    "j2fail" : {
+        "hbase10" : { "MODULE" : "janusgraph-hbase-parent/janusgraph-hbase-10" },
+        "cassandra" : { "MODULE" : "janusgraph-cassandra" },
+        "es" : { "MODULE" : "janusgraph-es" },
+        "hadoop" : { "MODULE" : "janusgraph-hadoop-parent/janusgraph-hadoop-2" },
+        "hbase098" : { "MODULE" : "janusgraph-hbase-parent/janusgraph-hbase-098" }
+    }
+}
diff --git a/janusgraph-codepipelines-ci/pom.xml b/janusgraph-codepipelines-ci/pom.xml
@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.janusgraph</groupId>
+        <artifactId>janusgraph</artifactId>
+        <version>0.1.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <artifactId>janusgraph-codepipelines-ci</artifactId>
+    <name>JanusGraph CodePipelines CI: Use AWS CodeBuild and CodePipelines to run release tests quickly.</name>
+    <url>http://janusgraph.org</url>
+    <properties>
+        <top.level.basedir>${basedir}/..</top.level.basedir>
+        <aws.sdk.version>1.11.86</aws.sdk.version>
+        <lombok.version>1.16.16</lombok.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-codepipeline</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-codebuild</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-iam</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-dynamodb</artifactId>
+            <version>${aws.sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>${lombok.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.janusgraph</groupId>
+            <artifactId>janusgraph-test</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.0.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.janusgraph.codepipelines.AwsCodePipelinesCi</mainClass>
+                                </transformer>
+                            </transformers>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+                <version>2.6.6</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-core</artifactId>
+                <version>2.6.6</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-annotations</artifactId>
+                <version>2.6.6</version>
+            </dependency>
+            <dependency>
+                <groupId>joda-time</groupId>
+                <artifactId>joda-time</artifactId>
+                <version>2.8.1</version>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+</project>