Merge pull request #229 from VirtusLab/fetch-with-retry

Fetch dependencies with retry

core/driver/sources/src/main/scala/org/virtuslab/ideprobe/config/DependenciesConfig.scala

@@ -4,7 +4,8 @@ object DependenciesConfig {
   case class Resolvers(
       intellij: IntelliJ = IntelliJ(Seq.empty),
       plugins: Plugins = Plugins(None),
-      jbr: Jbr = Jbr(Seq.empty)
+      jbr: Jbr = Jbr(Seq.empty),
+      retries: Int = 0
   )
 
   case class IntelliJ(repositories: Seq[String])

core/driver/sources/src/main/scala/org/virtuslab/ideprobe/dependencies/ResourceProvider.scala

@@ -5,29 +5,31 @@ import java.net.URI
 import java.nio.file.Files
 import java.nio.file.Path
 
+import scala.annotation.tailrec
+
 import org.virtuslab.ideprobe.Extensions._
 import org.virtuslab.ideprobe.IdeProbePaths
 
 trait ResourceProvider {
-  def get(uri: URI, provider: => InputStream): Path
+  def get(uri: URI, provider: () => InputStream): Path
 
-  def get(uri: URI): Path = get(uri, uri.toURL.openStream())
+  def get(uri: URI): Path = get(uri, () => uri.toURL.openStream())
 }
 
 object ResourceProvider {
-  def fromConfig(paths: IdeProbePaths): ResourceProvider = {
-    new Cached(paths.cache)
+  def fromConfig(paths: IdeProbePaths, retries: Int): ResourceProvider = {
+    new Cached(paths.cache, retries)
   }
 
-  val Default = new Cached(IdeProbePaths.Default.cache)
+  val Default = new Cached(IdeProbePaths.Default.cache, 0)
 
-  final class Cached(directory: Path) extends ResourceProvider {
-    override def get(uri: URI, provider: => InputStream): Path = {
+  final class Cached(directory: Path, retries: Int) extends ResourceProvider {
+    override def get(uri: URI, provider: () => InputStream): Path = {
       Resource.from(uri) match {
         case Resource.Http(uri) =>
-          cacheUri(uri, provider, cached => s"Fetching $uri into $cached")
+          cacheUri(uri, provider, cached => s"Fetching $uri into $cached", retries)
         case Resource.Jar(uri) =>
-          cacheUri(uri, provider, cached => s"Extracting $uri from jar into $cached")
+          cacheUri(uri, provider, cached => s"Extracting $uri from jar into $cached", retries)
         case file: Resource.File if file.path.toFile.exists() =>
           file.path
         case file: Resource.File =>
@@ -37,19 +39,35 @@ object ResourceProvider {
       }
     }
 
+    @tailrec
+    def retry[T](times: Int)(body: () => T): T = {
+      try {
+        body()
+      } catch {
+        case e: Throwable =>
+          if (times > 0) {
+            Thread.sleep(10000)
+            retry(times - 1)(body)
+          } else throw e
+      }
+    }
+
     private def cacheUri(
         uri: URI,
-        createStream: => InputStream,
-        message: Path => String
+        createStream: () => InputStream,
+        message: Path => String,
+        retries: Int
     ): Path = {
       val cachedResource = cached(uri)
       if (!cachedResource.isFile) {
-        val stream = createStream
-        println(message(cachedResource))
-        Files
-          .createTempFile("cached-resource", "-tmp")
-          .append(stream)
-          .moveTo(cachedResource)
+        retry(retries) { () =>
+          val stream = createStream()
+          println(message(cachedResource))
+          Files
+            .createTempFile("cached-resource", "-tmp")
+            .append(stream)
+            .moveTo(cachedResource)
+        }
       }
       cachedResource
     }

core/driver/sources/src/main/scala/org/virtuslab/ideprobe/ide/intellij/IntelliJProvider.scala

@@ -183,7 +183,7 @@ object IntelliJProvider {
     val intelliJResolvers = IntelliJZipResolver.fromConfig(resolversConfig.intellij)
     val pluginResolver = PluginResolver.fromConfig(resolversConfig.plugins)
     val jbrResolvers = JbrResolvers.fromConfig(resolversConfig.jbr)
-    val resourceProvider = ResourceProvider.fromConfig(paths)
+    val resourceProvider = ResourceProvider.fromConfig(paths, resolversConfig.retries)
     val intelliJDependencyProvider = new IntelliJDependencyProvider(intelliJResolvers, resourceProvider)
     val pluginDependencyProvider = new PluginDependencyProvider(Seq(pluginResolver), resourceProvider)
     val jbrDependencyProvider = new JbrDependencyProvider(jbrResolvers, resourceProvider)

extensions/pants/driver/src/main/scala/org/virtuslab/ideprobe/pants/PantsPluginBuilder.scala

@@ -20,7 +20,7 @@ object PantsPluginBuilder extends DependencyBuilder(Id("pants")) {
     val env = config.get[Map[String, String]]("env").getOrElse(Map.empty)
     val hash = GitRepository.commitHash(repository, "HEAD")
     val artifact = repository.path.resolveChild(hash)
-    resources.get(artifact, provider = build(repository, env))
+    resources.get(artifact, provider = () => build(repository, env))
   }
 
   private def build(repository: GitRepository, userEnv: Map[String, String]): InputStream = {

extensions/scala/driver/src/main/scala/org/virtuslab/ideprobe/scala/ScalaPluginBuilder.scala

@@ -26,7 +26,7 @@ object ScalaPluginBuilder extends DependencyBuilder(Id("scala")) {
     val jdkVersion = params.jdk.getOrElse("11")
     val hash = GitRepository.commitHash(repository, "HEAD")
     val artifact = repository.path.resolveChild(hash)
-    resources.get(artifact, provider = build(repository, jdkVersion, resources))
+    resources.get(artifact, provider = () => build(repository, jdkVersion, resources))
   }
 
   private def build(