Install fixed dependency versions for transpilation (#227)

src/main/scala/io/shiftleft/js2cpg/core/Config.scala

@@ -13,26 +13,27 @@ import scala.util.matching.Regex
 
 object Config {
 
-  val SL_IGNORE_FILE: String                    = ".slignore"
-  val DEFAULT_TS_TYPES: Boolean                 = false
-  val DEFAULT_TS_TRANSPILING: Boolean           = true
-  val DEFAULT_BABEL_TRANSPILING: Boolean        = true
-  val DEFAULT_VUE_TRANSPILING: Boolean          = true
-  val DEFAULT_NUXT_TRANSPILING: Boolean         = true
-  val DEFAULT_TEMPLATE_TRANSPILING: Boolean     = true
-  val DEFAULT_CPG_OUT_FILE: String              = "cpg.bin.zip"
-  val DEFAULT_IGNORED_FILES_REGEX: Regex        = "".r
-  val DEFAULT_IGNORED_FILES: Seq[Path]          = Seq.empty
-  val DEFAULT_IGNORE_MINIFIED: Boolean          = true
-  val DEFAULT_IGNORE_TESTS: Boolean             = true
-  val DEFAULT_IGNORE_PRIVATE_DEPS: Boolean      = false
-  val DEFAULT_PRIVATE_DEPS: Seq[String]         = Seq.empty
-  val DEFAULT_INCLUDE_CONFIGS: Boolean          = false
-  val DEFAULT_INCLUDE_HTML: Boolean             = true
-  val DEFAULT_JVM_METRICS: Option[Int]          = None
-  val DEFAULT_MODULE_MODE: Option[String]       = None
-  val DEFAULT_WITH_NODE_MODULES_FOLDER: Boolean = false
-  val DEFAULT_OPTIMIZE_DEPENDENCIES: Boolean    = true
+  val SL_IGNORE_FILE: String                            = ".slignore"
+  val DEFAULT_TS_TYPES: Boolean                         = false
+  val DEFAULT_TS_TRANSPILING: Boolean                   = true
+  val DEFAULT_BABEL_TRANSPILING: Boolean                = true
+  val DEFAULT_VUE_TRANSPILING: Boolean                  = true
+  val DEFAULT_NUXT_TRANSPILING: Boolean                 = true
+  val DEFAULT_TEMPLATE_TRANSPILING: Boolean             = true
+  val DEFAULT_CPG_OUT_FILE: String                      = "cpg.bin.zip"
+  val DEFAULT_IGNORED_FILES_REGEX: Regex                = "".r
+  val DEFAULT_IGNORED_FILES: Seq[Path]                  = Seq.empty
+  val DEFAULT_IGNORE_MINIFIED: Boolean                  = true
+  val DEFAULT_IGNORE_TESTS: Boolean                     = true
+  val DEFAULT_IGNORE_PRIVATE_DEPS: Boolean              = false
+  val DEFAULT_PRIVATE_DEPS: Seq[String]                 = Seq.empty
+  val DEFAULT_INCLUDE_CONFIGS: Boolean                  = false
+  val DEFAULT_INCLUDE_HTML: Boolean                     = true
+  val DEFAULT_JVM_METRICS: Option[Int]                  = None
+  val DEFAULT_MODULE_MODE: Option[String]               = None
+  val DEFAULT_WITH_NODE_MODULES_FOLDER: Boolean         = false
+  val DEFAULT_OPTIMIZE_DEPENDENCIES: Boolean            = true
+  val DEFAULT_FIXED_TRANSPILATION_DEPENDENCIES: Boolean = false
 
 }
 
@@ -57,7 +58,8 @@ case class Config(
   jvmMetrics: Option[Int] = Config.DEFAULT_JVM_METRICS,
   moduleMode: Option[String] = Config.DEFAULT_MODULE_MODE,
   withNodeModuleFolder: Boolean = Config.DEFAULT_WITH_NODE_MODULES_FOLDER,
-  optimizeDependencies: Boolean = Config.DEFAULT_OPTIMIZE_DEPENDENCIES
+  optimizeDependencies: Boolean = Config.DEFAULT_OPTIMIZE_DEPENDENCIES,
+  fixedTranspilationDependencies: Boolean = Config.DEFAULT_FIXED_TRANSPILATION_DEPENDENCIES
 ) {
 
   def createPathForPackageJson(): Path = Paths.get(packageJsonLocation) match {
@@ -91,6 +93,7 @@ case class Config(
       |\t- package.json location: '${createPathForPackageJson()}'
       |\t- Module mode: '${moduleMode.getOrElse(TypescriptTranspiler.DEFAULT_MODULE)}'
       |\t- Optimize dependencies: $optimizeDependencies
+      |\t- Fixed transpilations dependencies: $fixedTranspilationDependencies
       |\t- Typescript transpiling: $tsTranspiling
       |\t- Babel transpiling: $babelTranspiling
       |\t- Vue.js transpiling: $vueTranspiling

src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala

@@ -9,38 +9,39 @@ import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler
 import scopt.OptionParser
 
 object Js2cpgArgumentsParser {
-  val HELP: String                     = "help"
-  val VERSION: String                  = "version"
-  val SRCDIR: String                   = "<src>"
-  val OUTPUT: String                   = "output"
-  val WITH_TS_TYPES: String            = "with-typescript-types"
-  val EXCLUDE: String                  = "exclude"
-  val EXCLUDE_REGEX: String            = "exclude-regex"
-  val PACKAGE_JSON: String             = "package-json"
-  val NO_TS: String                    = "no-ts"
-  val TS: String                       = "ts"
-  val NO_BABEL: String                 = "no-babel"
-  val NO_VUE: String                   = "no-vue-js"
-  val NO_NUXT: String                  = "no-nuxt-js"
-  val NO_TEMPLATES: String             = "no-templates"
-  val BABEL: String                    = "babel"
-  val TRANSPILING: String              = "transpiling"
-  val IGNORE_MINIFIED: String          = "ignore-minified"
-  val WITH_MINIFIED: String            = "with-minified"
-  val INCLUDE_MINIFIED: String         = "include-minified"
-  val WITH_TESTS: String               = "with-tests"
-  val INCLUDE_TESTS: String            = "include-tests"
-  val IGNORE_PRIVATE_DEPS: String      = "ignore-private-deps"
-  val EXCLUDE_PRIVATE_DEPS: String     = "exclude-private-deps"
-  val PRIVATE_DEPS: String             = "private-deps-ns"
-  val INCLUDE_CONFIGS: String          = "include-configs"
-  val INCLUDE_HTML: String             = "include-html"
-  val EXCLUDE_HTML: String             = "exclude-html"
-  val JVM_MONITOR: String              = "enable-jvm-monitor"
-  val MODULE_MODE: String              = "module-mode"
-  val WITH_NODE_MODULES_FOLDER: String = "with-node-modules-folder"
-  val OPTIMIZE_DEPENDENCIES: String    = "optimize-dependencies"
-  val ALL_DEPENDENCIES: String         = "all-dependencies"
+  val HELP: String                             = "help"
+  val VERSION: String                          = "version"
+  val SRCDIR: String                           = "<src>"
+  val OUTPUT: String                           = "output"
+  val WITH_TS_TYPES: String                    = "with-typescript-types"
+  val EXCLUDE: String                          = "exclude"
+  val EXCLUDE_REGEX: String                    = "exclude-regex"
+  val PACKAGE_JSON: String                     = "package-json"
+  val NO_TS: String                            = "no-ts"
+  val TS: String                               = "ts"
+  val NO_BABEL: String                         = "no-babel"
+  val NO_VUE: String                           = "no-vue-js"
+  val NO_NUXT: String                          = "no-nuxt-js"
+  val NO_TEMPLATES: String                     = "no-templates"
+  val BABEL: String                            = "babel"
+  val TRANSPILING: String                      = "transpiling"
+  val IGNORE_MINIFIED: String                  = "ignore-minified"
+  val WITH_MINIFIED: String                    = "with-minified"
+  val INCLUDE_MINIFIED: String                 = "include-minified"
+  val WITH_TESTS: String                       = "with-tests"
+  val INCLUDE_TESTS: String                    = "include-tests"
+  val IGNORE_PRIVATE_DEPS: String              = "ignore-private-deps"
+  val EXCLUDE_PRIVATE_DEPS: String             = "exclude-private-deps"
+  val PRIVATE_DEPS: String                     = "private-deps-ns"
+  val INCLUDE_CONFIGS: String                  = "include-configs"
+  val INCLUDE_HTML: String                     = "include-html"
+  val EXCLUDE_HTML: String                     = "exclude-html"
+  val JVM_MONITOR: String                      = "enable-jvm-monitor"
+  val MODULE_MODE: String                      = "module-mode"
+  val WITH_NODE_MODULES_FOLDER: String         = "with-node-modules-folder"
+  val OPTIMIZE_DEPENDENCIES: String            = "optimize-dependencies"
+  val ALL_DEPENDENCIES: String                 = "all-dependencies"
+  val FIXED_TRANSPILATION_DEPENDENCIES: String = "fixed-transpilation-dependencies"
 }
 
 class Js2cpgArgumentsParser {
@@ -188,6 +189,11 @@ class Js2cpgArgumentsParser {
         s"install all project dependencies during transpilation (defaults to '${!Config.DEFAULT_OPTIMIZE_DEPENDENCIES}')"
       )
       .action((_, c) => c.copy(optimizeDependencies = false))
+    opt[Unit](FIXED_TRANSPILATION_DEPENDENCIES)
+      .text(
+        s"install fixed versions of transpilation dependencies during transpilation (defaults to '${!Config.DEFAULT_FIXED_TRANSPILATION_DEPENDENCIES}')"
+      )
+      .action((_, c) => c.copy(fixedTranspilationDependencies = true))
     opt[Int](JVM_MONITOR)
       .text("enable JVM metrics logging (requires JMX port number)")
       .action((jmxPortNumber, c) => c.copy(jvmMetrics = Some(jmxPortNumber)))

src/main/scala/io/shiftleft/js2cpg/preprocessing/BabelTranspiler.scala

@@ -19,6 +19,12 @@ class BabelTranspiler(
   private val logger = LoggerFactory.getLogger(getClass)
   private val babel  = Paths.get(projectPath.toString, "node_modules", ".bin", "babel").toString
 
+  private val babelPresets =
+    Versions.babelVersions.keySet.filter(_.contains("@babel/preset")).map(n => s"--presets $n ").mkString
+
+  private val babelPlugins =
+    Versions.babelVersions.keySet.filter(_.contains("@babel/plugin")).map(n => s"--plugins $n ").mkString
+
   override def shouldRun(): Boolean =
     config.babelTranspiling && !VueTranspiler.isVueProject(config, projectPath)
 
@@ -39,17 +45,7 @@ class BabelTranspiler(
     val command = s"${ExternalCommand.toOSCommand(babel)} . " +
       "--no-babelrc " +
       s"--source-root '${in.toString}' " +
-      "--source-maps true " +
-      "--presets @babel/preset-env " +
-      "--presets @babel/preset-react " +
-      "--presets @babel/preset-flow " +
-      "--presets @babel/preset-typescript " +
-      "--plugins @babel/plugin-proposal-class-properties " +
-      "--plugins @babel/plugin-proposal-private-methods " +
-      "--plugins @babel/plugin-proposal-object-rest-spread " +
-      "--plugins @babel/plugin-proposal-nullish-coalescing-operator " +
-      "--plugins @babel/plugin-transform-property-mutators " +
-      "--plugins @babel/plugin-transform-runtime " +
+      "--source-maps true " + babelPresets + babelPlugins +
       s"--out-dir $outDir $constructIgnoreDirArgs"
     logger.debug(s"\t+ Babel transpiling $projectPath to $outDir with command '$command'")
     ExternalCommand.run(command, in.toString) match {

src/main/scala/io/shiftleft/js2cpg/preprocessing/PugTranspiler.scala

@@ -11,8 +11,9 @@ import scala.util.{Failure, Success}
 
 class PugTranspiler(override val config: Config, override val projectPath: Path) extends Transpiler {
 
-  private val logger = LoggerFactory.getLogger(getClass)
-  private val pug    = Paths.get(projectPath.toString, "node_modules", ".bin", "pug").toString
+  private val logger        = LoggerFactory.getLogger(getClass)
+  private val pug           = Paths.get(projectPath.toString, "node_modules", ".bin", "pug").toString
+  private val pugAndVersion = Versions.nameAndVersion("pug-cli")
 
   private def hasPugFiles: Boolean =
     FileUtils.getFileTree(projectPath, config, List(PUG_SUFFIX)).nonEmpty
@@ -21,11 +22,11 @@ class PugTranspiler(override val config: Config, override val projectPath: Path)
 
   private def installPugPlugins(): Boolean = {
     val command = if (pnpmAvailable(projectPath)) {
-      s"${TranspilingEnvironment.PNPM_ADD} pug-cli && ${TranspilingEnvironment.PNPM_INSTALL}"
+      s"${TranspilingEnvironment.PNPM_ADD} $pugAndVersion && ${TranspilingEnvironment.PNPM_INSTALL}"
     } else if (yarnAvailable()) {
-      s"${TranspilingEnvironment.YARN_ADD} pug-cli && ${TranspilingEnvironment.YARN_INSTALL}"
+      s"${TranspilingEnvironment.YARN_ADD} $pugAndVersion && ${TranspilingEnvironment.YARN_INSTALL}"
     } else {
-      s"${TranspilingEnvironment.NPM_INSTALL} pug-cli && ${TranspilingEnvironment.NPM_INSTALL}"
+      s"${TranspilingEnvironment.NPM_INSTALL} $pugAndVersion"
     }
     logger.info("Installing Pug dependencies and plugins. That will take a while.")
     logger.debug(s"\t+ Installing Pug plugins with command '$command' in path '$projectPath'")

src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilerGroup.scala

@@ -13,27 +13,15 @@ case class TranspilerGroup(override val config: Config, override val projectPath
 
   private val logger = LoggerFactory.getLogger(getClass)
 
-  private val BABEL_PLUGINS: String =
-    "@babel/core " +
-      "@babel/cli " +
-      "@babel/preset-env " +
-      "@babel/preset-flow " +
-      "@babel/preset-react " +
-      "@babel/preset-typescript " +
-      "@babel/plugin-proposal-class-properties " +
-      "@babel/plugin-proposal-private-methods " +
-      "@babel/plugin-proposal-object-rest-spread " +
-      "@babel/plugin-proposal-nullish-coalescing-operator " +
-      "@babel/plugin-transform-runtime " +
-      "@babel/plugin-transform-property-mutators"
+  private val BABEL_PLUGINS: String = Versions.babelVersions.keySet.map(Versions.nameAndVersion).mkString(" ")
 
   private def installPlugins(): Boolean = {
     val command = if (pnpmAvailable(projectPath)) {
       s"${TranspilingEnvironment.PNPM_ADD} $BABEL_PLUGINS && ${TranspilingEnvironment.PNPM_INSTALL}"
     } else if (yarnAvailable()) {
       s"${TranspilingEnvironment.YARN_ADD} $BABEL_PLUGINS && ${TranspilingEnvironment.YARN_INSTALL}"
     } else {
-      s"${TranspilingEnvironment.NPM_INSTALL} $BABEL_PLUGINS && ${TranspilingEnvironment.NPM_INSTALL}"
+      s"${TranspilingEnvironment.NPM_INSTALL} $BABEL_PLUGINS"
     }
     logger.info("Installing project dependencies and plugins. That will take a while.")
     logger.debug(s"\t+ Installing plugins with command '$command' in path '$projectPath'")

src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala

@@ -24,15 +24,15 @@ object TranspilingEnvironment {
   val NPM: String  = ExternalCommand.toOSCommand("npm")
 
   val YARN_ADD: String =
-    s"$YARN --prefer-offline --ignore-scripts --legacy-peer-deps --dev -W add"
+    s"$YARN --ignore-scripts --legacy-peer-deps --dev -W add"
   val YARN_INSTALL: String =
-    s"$YARN --prefer-offline --ignore-scripts --legacy-peer-deps install"
+    s"$YARN --ignore-scripts --legacy-peer-deps install"
   val PNPM_ADD: String =
-    s"$PNPM --prefer-offline --ignore-scripts add -D"
+    s"$PNPM --ignore-scripts add -D"
   val PNPM_INSTALL: String =
-    s"$PNPM --prefer-offline --ignore-scripts install"
+    s"$PNPM --ignore-scripts install"
   val NPM_INSTALL: String =
-    s"$NPM --prefer-offline --no-audit --progress=false --ignore-scripts --legacy-peer-deps --save-dev install"
+    s"$NPM --no-audit --progress=false --ignore-scripts --legacy-peer-deps --save-dev install"
 }
 
 trait TranspilingEnvironment {
@@ -42,6 +42,35 @@ trait TranspilingEnvironment {
 
   private val logger = LoggerFactory.getLogger(getClass)
 
+  object Versions {
+    val babelVersions: Map[String, String] = Map(
+      "@babel/core"                                        -> "7.20.2",
+      "@babel/cli"                                         -> "7.19.3",
+      "@babel/preset-env"                                  -> "7.20.2",
+      "@babel/preset-flow"                                 -> "7.18.6",
+      "@babel/preset-react"                                -> "7.18.6",
+      "@babel/preset-typescript"                           -> "7.18.6",
+      "@babel/plugin-proposal-class-properties"            -> "7.18.6",
+      "@babel/plugin-proposal-private-methods"             -> "7.18.6",
+      "@babel/plugin-proposal-object-rest-spread"          -> "7.20.2",
+      "@babel/plugin-proposal-nullish-coalescing-operator" -> "7.18.6",
+      "@babel/plugin-transform-runtime"                    -> "7.19.6",
+      "@babel/plugin-transform-property-mutators"          -> "7.18.6"
+    )
+
+    private val versions: Map[String, String] =
+      babelVersions ++ Map("pug-cli" -> "1.0.0-alpha6", "typescript" -> "4.8.4", "@vue/cli-service-global" -> "4.5.19")
+
+    def nameAndVersion(dependencyName: String): String = {
+      if (config.fixedTranspilationDependencies) {
+        val version = versions.get(dependencyName).map(v => s"@$v").getOrElse("")
+        s"$dependencyName$version"
+      } else {
+        dependencyName
+      }
+    }
+  }
+
   private def checkForPnpm(): Boolean = {
     logger.debug(s"\t+ Checking pnpm ...")
     ExternalCommand.run(s"${TranspilingEnvironment.PNPM} -v", projectPath.toString) match {

src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala

@@ -12,6 +12,7 @@ import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler.DEFAULT_MODULE
 import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler.DENO_CONFIG
 import org.slf4j.LoggerFactory
 import org.apache.commons.io.{FileUtils => CommonsFileUtils}
+
 import java.nio.file.{Path, Paths}
 import scala.util.{Failure, Success, Try}
 
@@ -33,7 +34,8 @@ class TypescriptTranspiler(override val config: Config, override val projectPath
 
   private val NODE_OPTIONS: Map[String, String] = Map("NODE_OPTIONS" -> "--max_old_space_size=4096")
 
-  private val tsc = Paths.get(projectPath.toString, "node_modules", ".bin", "tsc").toString
+  private val tsc                  = Paths.get(projectPath.toString, "node_modules", ".bin", "tsc").toString
+  private val typescriptAndVersion = Versions.nameAndVersion("typescript")
 
   private def hasTsFiles: Boolean =
     FileUtils.getFileTree(projectPath, config, List(TS_SUFFIX)).nonEmpty
@@ -91,11 +93,11 @@ class TypescriptTranspiler(override val config: Config, override val projectPath
 
   private def installTsPlugins(): Boolean = {
     val command = if (pnpmAvailable(projectPath)) {
-      s"${TranspilingEnvironment.PNPM_ADD} typescript"
+      s"${TranspilingEnvironment.PNPM_ADD} $typescriptAndVersion"
     } else if (yarnAvailable()) {
-      s"${TranspilingEnvironment.YARN_ADD} typescript"
+      s"${TranspilingEnvironment.YARN_ADD} $typescriptAndVersion"
     } else {
-      s"${TranspilingEnvironment.NPM_INSTALL} typescript"
+      s"${TranspilingEnvironment.NPM_INSTALL} $typescriptAndVersion"
     }
     logger.info("Installing TypeScript dependencies and plugins. That will take a while.")
     logger.debug(s"\t+ Installing Typescript plugins with command '$command' in path '$projectPath'")

src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala

@@ -33,7 +33,8 @@ class VueTranspiler(override val config: Config, override val projectPath: Path)
 
   private lazy val NODE_OPTIONS: Map[String, String] = nodeOptions()
 
-  private val vue = Paths.get(projectPath.toString, "node_modules", ".bin", "vue-cli-service").toString
+  private val vue           = Paths.get(projectPath.toString, "node_modules", ".bin", "vue-cli-service").toString
+  private val vueAndVersion = Versions.nameAndVersion("@vue/cli-service-global")
 
   override def shouldRun(): Boolean = config.vueTranspiling && isVueProject(config, projectPath)
 
@@ -48,11 +49,11 @@ class VueTranspiler(override val config: Config, override val projectPath: Path)
 
   private def installVuePlugins(): Boolean = {
     val command = if (pnpmAvailable(projectPath)) {
-      s"${TranspilingEnvironment.PNPM_ADD} @vue/cli-service-global && ${TranspilingEnvironment.PNPM_INSTALL}"
+      s"${TranspilingEnvironment.PNPM_ADD} $vueAndVersion && ${TranspilingEnvironment.PNPM_INSTALL}"
     } else if (yarnAvailable()) {
-      s"${TranspilingEnvironment.YARN_ADD} @vue/cli-service-global && ${TranspilingEnvironment.YARN_INSTALL}"
+      s"${TranspilingEnvironment.YARN_ADD} $vueAndVersion && ${TranspilingEnvironment.YARN_INSTALL}"
     } else {
-      s"${TranspilingEnvironment.NPM_INSTALL} @vue/cli-service-global && ${TranspilingEnvironment.NPM_INSTALL}"
+      s"${TranspilingEnvironment.NPM_INSTALL} $vueAndVersion"
     }
     logger.info("Installing Vue.js dependencies and plugins. That will take a while.")
     logger.debug(s"\t+ Installing Vue.js plugins with command '$command' in path '$projectPath'")

src/test/scala/io/shiftleft/js2cpg/fixtures/DataFlowCode2CpgFixture.scala

@@ -33,7 +33,7 @@ trait Js2CpgFrontend extends LanguageFrontend {
         outputFile = cpgFile.pathAsString
       )
       js2cpg.run(config)
-      cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgFile.pathAsString))
+      cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgFile.pathAsString))
     }
     cpg
   }

src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala

@@ -43,7 +43,7 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec
           defaultArgs.map(_.toArg).toArray
       )
 
-      val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString))
+      val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
 
       fileNames(cpg) should contain theSameElementsAs expectedFiles.map(_.replace("/", java.io.File.separator))
       cpg.close()