Merge branch 'feature/0.3.0'

.gitignore

@@ -15,3 +15,6 @@ project/plugins/project/
 # Scala-IDE specific
 .scala_dependencies
 .worksheet
+
+# Vagrant
+.vagrant

CHANGELOG

@@ -1,3 +1,9 @@
+Version 0.3.0 (2015-08-11)
+--------------------------
+Started maintaining a cache for missing schemas (#30)
+Made ExceptionUtils.getRootCause(e).getMessage calls safe (#24)
+Added dedicated Vagrant setup (#26)
+
 Version 0.2.0 (2014-12-07)
 --------------------------
 Added SchemaCriterion class for comparing schema versions (#21)

README.md

@@ -6,6 +6,18 @@ Iglu Scala Client is used extensively in **[Snowplow] [snowplow-repo]** to valid
 
 ![scala-client-img] [scala-client-img]
 
+## Developer quickstart
+
+Assuming git, **[Vagrant] [vagrant-install]** and **[VirtualBox] [virtualbox-install]** installed:
+
+```bash
+ host> git clone https://github.com/snowplow/iglu-scala-client
+ host> cd iglu-scala-client
+ host> vagrant up && vagrant ssh
+guest> cd /vagrant
+guest> sbt compile
+```
+
 ## Find out more
 
 | **[Technical Docs] [techdocs]**     | **[Setup Guide] [setup]**     | **[Roadmap] [roadmap]**           | **[Contributing] [contributing]**           |
@@ -33,6 +45,9 @@ limitations under the License.
 [snowplow-repo]: https://github.com/snowplow/snowplow
 [snowplow-website]: http://snowplowanalytics.com
 
+[vagrant-install]: http://docs.vagrantup.com/v2/installation/index.html
+[virtualbox-install]: https://www.virtualbox.org/wiki/Downloads
+
 [techdocs-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/techdocs.png
 [setup-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/setup.png
 [roadmap-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/roadmap.png

Vagrantfile

@@ -0,0 +1,19 @@
+Vagrant.configure("2") do |config|
+
+  config.vm.box = "ubuntu/trusty64"
+  config.vm.hostname = "iglu-scala-client"
+  config.ssh.forward_agent = true
+
+  config.vm.provider :virtualbox do |vb|
+    vb.name = Dir.pwd().split("/")[-1] + "-" + Time.now.to_f.to_i.to_s
+    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+    vb.customize [ "guestproperty", "set", :id, "--timesync-threshold", 10000 ]
+    # Scala is memory-hungry
+    vb.memory = 5120
+  end
+
+  config.vm.provision :shell do |sh|
+    sh.path = "vagrant/up.bash"
+  end
+
+end

project/BuildSettings.scala

@@ -18,7 +18,7 @@ object BuildSettings {
   // Basic settings for our app
   lazy val basicSettings = Seq[Setting[_]](
     organization  := "com.snowplowanalytics",
-    version       := "0.2.0",
+    version       := "0.3.0",
     description   := "Scala client and resolver for Iglu schema repositories",
     scalaVersion  := "2.10.4",
     scalacOptions := Seq("-deprecation", "-encoding", "utf8"),

src/main/scala/com.snowplowanalytics.iglu/client/Resolver.scala

@@ -179,7 +179,7 @@ case class Resolver(
      * @return the schema if found as Some JsonNode or None
      *         if not found, or cache is not enabled.
      */
-    def get(schemaKey: SchemaKey): Option[JsonNode] =
+    def get(schemaKey: SchemaKey): Option[ValidatedNel[JsonNode]] =
       for {
         l <- lru
         k <- l.get(schemaKey)
@@ -193,7 +193,7 @@ case class Resolver(
      * @param schema The provided schema
      * @return the same schema
      */
-    def store(schemaKey: SchemaKey, schema: JsonNode): JsonNode = {
+    def store(schemaKey: SchemaKey, schema: ValidatedNel[JsonNode]): ValidatedNel[JsonNode] = {
       for (l <- lru) {
         l.put(schemaKey, schema)
       }
@@ -231,10 +231,10 @@ case class Resolver(
     @tailrec def recurse(schemaKey: SchemaKey, errors: ProcessingMessages, tried: RepositoryRefs, remaining: RepositoryRefs): ValidatedNel[JsonNode] = {
       remaining match {
         case Nil =>
-          collectErrors(schemaKey, errors, tried).fail
+          cache.store(schemaKey, collectErrors(schemaKey, errors, tried).fail)
         case repo :: repos => {
           repo.lookupSchema(schemaKey) match {
-            case Success(Some(schema)) => cache.store(schemaKey, schema).success
+            case Success(Some(schema)) => cache.store(schemaKey, schema.success)
             case Success(None)         => recurse(schemaKey, errors, tried.::(repo), repos)
             case Failure(e)            => recurse(schemaKey, errors.::(e), tried.::(repo), repos)
           }
@@ -243,7 +243,7 @@ case class Resolver(
     }
 
     cache.get(schemaKey) match {
-      case Some(schema) => schema.success
+      case Some(schema) => schema
       case None         => recurse(schemaKey, Nil, Nil, prioritizeRepos(schemaKey))
     }
   }

src/main/scala/com.snowplowanalytics.iglu/client/repositories/EmbeddedRepositoryRef.scala

@@ -144,7 +144,7 @@ case class EmbeddedRepositoryRef(
       case ioe: IOException =>
         None.success // Schema not found
       case e: Throwable =>
-        s"Unknown problem reading and parsing ${schemaPath} in ${descriptor} Iglu repository ${config.name}: %s".format(ExceptionUtils.getRootCause(e).getMessage).fail.toProcessingMessage
+        s"Unknown problem reading and parsing ${schemaPath} in ${descriptor} Iglu repository ${config.name}: ${VE.getThrowableMessage(e)}".fail.toProcessingMessage
     }
   }
 }

src/main/scala/com.snowplowanalytics.iglu/client/utils/ValidationExceptions.scala

@@ -13,6 +13,9 @@
 package com.snowplowanalytics.iglu.client
 package utils
 
+// Apache Commons
+import org.apache.commons.lang3.exception.ExceptionUtils
+
 /**
  * Provides helpers around converting JVM
  * exceptions to Scalaz Validations.
@@ -41,4 +44,29 @@ object ValidationExceptions {
     .replaceAll("\\p{Cntrl}", "") // Any other control character
     .trim
   }
+
+  /**
+   * Get the message out of a Throwable's root cause
+   * (or failing that the Throwable itself) in a
+   * null-safe fashion.
+   *
+   * @param throwable The throwable to extract a message from
+   * @return the message from either the Throwable or
+   *         preferably its root cause, Option-boxed
+   */
+  def getThrowableMessage(throwable: Throwable): Option[String] = {
+
+    def getRootCauseIfExists(throwable: Throwable): Throwable =
+      Option(ExceptionUtils.getRootCause(throwable)) match {
+        case Some(root) => root
+        case None       => throwable
+      }
+
+    for {
+      t  <- Option(throwable)
+      rc =  getRootCauseIfExists(t)
+      m  <- Option(rc.getMessage)
+    } yield m
+  }
+
 }

src/main/scala/com.snowplowanalytics.iglu/package.scala

@@ -47,7 +47,7 @@ package object client {
   /**
    * Our LRU cache of schemas
    */
-  type SchemaLruMap = LruMap[SchemaKey, JsonNode]
+  type SchemaLruMap = LruMap[SchemaKey, ValidatedNel[JsonNode]]
 
   /**
    * Our List (possibly empty) of Iglu repositories

vagrant/.gitignore

@@ -0,0 +1,3 @@
+.peru
+oss-playbooks
+ansible

vagrant/ansible.hosts

@@ -0,0 +1,2 @@
+[vagrant]
+127.0.0.1:2222

vagrant/peru.yaml

@@ -0,0 +1,14 @@
+imports:
+    ansible: ansible
+    ansible_playbooks: oss-playbooks
+
+curl module ansible:
+    # Equivalent of git cloning tags/v1.6.6 but much, much faster
+    url: https://codeload.github.com/ansible/ansible/zip/69d85c22c7475ccf8169b6ec9dee3ee28c92a314
+    unpack: zip
+    export: ansible-69d85c22c7475ccf8169b6ec9dee3ee28c92a314
+
+git module ansible_playbooks:
+    url: https://github.com/snowplow/ansible-playbooks.git
+    # Comment out to fetch a specific rev instead of master:
+    # rev: xxx

vagrant/up.bash

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e
+
+vagrant_dir=/vagrant/vagrant
+bashrc=/home/vagrant/.bashrc
+
+echo "========================================"
+echo "INSTALLING PERU AND ANSIBLE DEPENDENCIES"
+echo "----------------------------------------"
+apt-get update
+apt-get install -y language-pack-en git unzip libyaml-dev python3-pip python-yaml python-paramiko python-jinja2
+
+echo "==============="
+echo "INSTALLING PERU"
+echo "---------------"
+sudo pip3 install peru
+
+echo "======================================="
+echo "CLONING ANSIBLE AND PLAYBOOKS WITH PERU"
+echo "---------------------------------------"
+cd ${vagrant_dir} && peru sync -v
+echo "... done"
+
+env_setup=${vagrant_dir}/ansible/hacking/env-setup
+hosts=${vagrant_dir}/ansible.hosts
+
+echo "==================="
+echo "CONFIGURING ANSIBLE"
+echo "-------------------"
+touch ${bashrc}
+echo "source ${env_setup}" >> ${bashrc}
+echo "export ANSIBLE_HOSTS=${hosts}" >> ${bashrc}
+echo "... done"
+
+echo "=========================================="
+echo "RUNNING PLAYBOOKS WITH ANSIBLE*"
+echo "* no output while each playbook is running"
+echo "------------------------------------------"
+while read pb; do
+    su - -c "source ${env_setup} && ${vagrant_dir}/ansible/bin/ansible-playbook ${vagrant_dir}/${pb} --connection=local --inventory-file=${hosts}" vagrant
+done <${vagrant_dir}/up.playbooks
+
+guidance=${vagrant_dir}/up.guidance
+
+if [ -f ${guidance} ]; then
+    echo "==========="
+    echo "PLEASE READ"
+    echo "-----------"
+    cat $guidance
+fi

vagrant/up.guidance

@@ -0,0 +1,4 @@
+To get started:
+vagrant ssh
+cd /vagrant
+sbt test

vagrant/up.playbooks

@@ -0,0 +1,3 @@
+oss-playbooks/java7.yml
+oss-playbooks/scala.yml
+oss-playbooks/sbt.yml