Adapted to new cli args & not redownloading ruby_ast_gen on compile

joernio · Oct 31, 2024 · 32a3813 · 32a3813
1 parent 1ae6b4f
commit 32a3813
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 12 deletions.
diff --git a/joern-cli/frontends/rubysrc2cpg/build.sbt b/joern-cli/frontends/rubysrc2cpg/build.sbt
@@ -4,7 +4,7 @@ import versionsort.VersionHelper
 
 import java.net.URI
 import scala.sys.process.stringToProcess
-import scala.util.Try
+import scala.util.{Failure, Success, Try}
 
 name := "rubysrc2cpg"
 
@@ -29,24 +29,42 @@ libraryDependencies ++= Seq(
 
 enablePlugins(JavaAppPackaging, LauncherJarPlugin)
 
-lazy val astGenVersion = settingKey[String]("ruby_ast_gen.rb version")
-astGenVersion := appProperties.value.getString("rubysrc2cpg.ruby_ast_gen_version")
-
 libraryDependencies ++= Seq(
   "io.shiftleft"  %% "codepropertygraph" % Versions.cpg,
   "org.scalatest" %% "scalatest"         % Versions.scalatest % Test
 )
 
+lazy val astGenVersion = settingKey[String]("ruby_ast_gen.rb version")
+astGenVersion := appProperties.value.getString("rubysrc2cpg.ruby_ast_gen_version")
+
 lazy val astGenDlUrl = settingKey[String]("astgen download url")
 astGenDlUrl := s"https://github.com/joernio/ruby_ast_gen/releases/download/v${astGenVersion.value}/"
 
+def hasCompatibleAstGenVersion(astGenBaseDir: File, astGenVersion: String): Boolean = {
+  val versionFile = astGenBaseDir / "lib" / "ruby_ast_gen" / "version.rb"
+  if (!versionFile.exists) return false
+  val versionPattern = "VERSION = \"([0-9]+\\.[0-9]+\\.[0-9]+)\"".r
+  versionPattern.findFirstIn(IO.read(versionFile)) match {
+    case Some(versionString) =>
+      // Regex group matching doesn't appear to work in SBT
+      val version = versionString.stripPrefix("VERSION = \"").stripSuffix("\"")
+      version == astGenVersion
+    case _ => false
+  }
+}
+
 lazy val astGenDlTask = taskKey[Unit](s"Download astgen binaries and update bundled resource")
 astGenDlTask := {
   val targetDir           = baseDirectory.value / "src" / "main" / "resources"
   val gemName             = s"ruby_ast_gen_v${astGenVersion.value}.zip"
   val gemFullPath         = targetDir / gemName
   val unpackedGemFullPath = targetDir / gemName.stripSuffix(s"_v${astGenVersion.value}.zip")
-  DownloadHelper.ensureIsAvailable(s"${astGenDlUrl.value}$gemName", gemFullPath)
+  if (!hasCompatibleAstGenVersion(unpackedGemFullPath, astGenVersion.value)) {
+    val url = s"${astGenDlUrl.value}$gemName"
+    sbt.io.Using.urlInputStream(new URI(url).toURL) { inputStream =>
+      sbt.IO.transfer(inputStream, gemFullPath)
+    }
+  }
   if (unpackedGemFullPath.exists()) IO.delete(unpackedGemFullPath)
   IO.unzip(gemFullPath, unpackedGemFullPath)
   IO.delete(gemFullPath)

diff --git a/...i/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyAstGenRunner.scala b/...i/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyAstGenRunner.scala
@@ -42,13 +42,13 @@ class RubyAstGenRunner(config: Config) extends AstGenRunnerBase(config) {
     }
 
     astGenOut.map(_.strip()).foreach {
-      case s"W, [$_]  WARN -- : $reason - $fileName"   => addReason(reason, Option(fileName))
-      case s"E, [$_] ERROR -- : '$fileName' - $reason" => addReason(reason, Option(fileName))
-      case s"E, [$_] ERROR -- : Failed to parse $fileName: $reason" =>
+      case s"[WARN] $reason - $fileName"  => addReason(reason, Option(fileName))
+      case s"[ERR] '$fileName' - $reason" => addReason(reason, Option(fileName))
+      case s"[ERR] Failed to parse $fileName: $reason" =>
         addReason(s"Failed to parse: $reason", Option(fileName))
-      case s"I, [$_]  INFO -- : Processed: $fileName -> $_" => diagnosticMap.put(fileName, Nil)
-      case s"I, [$_]  INFO -- : Excluding: $fileName"       => addReason("Skipped", Option(fileName))
-      case _                                                => // ignore
+      case s"[INFO] Processed: $fileName -> $_" => diagnosticMap.put(fileName, Nil)
+      case s"[INFO] Excluding: $fileName"       => addReason("Skipped", Option(fileName))
+      case _                                    => // ignore
     }
 
     diagnosticMap.flatMap {
@@ -74,7 +74,7 @@ class RubyAstGenRunner(config: Config) extends AstGenRunnerBase(config) {
         val cwd            = env.path.toAbsolutePath.toString
         val excludeCommand = if (exclude.isEmpty) "" else s"-e \"$exclude\""
         val gemPath        = Seq(cwd, "vendor", "bundle", "jruby", "3.1.0").mkString(separator)
-        val rubyArgs       = Array("--log", "info", "-o", out.toString(), "-i", in, excludeCommand)
+        val rubyArgs       = Array("-o", out.toString(), "-i", in, excludeCommand).filterNot(_.isBlank)
         val mainScript     = Seq(cwd, "exe", "ruby_ast_gen.rb").mkString(separator)
         if (config.tryLocalRuby) {
           executeWithNativeRuby(mainScript, cwd, rubyArgs, gemPath)