From 61aafea8790b9d32c5215667be55ef7ca7a84b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Wed, 9 Nov 2022 16:21:59 +0100 Subject: [PATCH] Install fixed dependency versions for transpilation (#227) --- .../io/shiftleft/js2cpg/core/Config.scala | 45 ++++++------ .../js2cpg/core/Js2cpgArgumentsParser.scala | 70 ++++++++++--------- .../preprocessing/BabelTranspiler.scala | 18 ++--- .../js2cpg/preprocessing/PugTranspiler.scala | 11 +-- .../preprocessing/TranspilerGroup.scala | 16 +---- .../TranspilingEnvironment.scala | 39 +++++++++-- .../preprocessing/TypescriptTranspiler.scala | 10 +-- .../js2cpg/preprocessing/VueTranspiler.scala | 9 +-- .../fixtures/DataFlowCode2CpgFixture.scala | 2 +- .../io/shiftleft/js2cpg/io/ExcludeTest.scala | 2 +- .../js2cpg/io/PrivateModulesTest.scala | 8 +-- .../TranspilationRunnerTest.scala | 20 +++--- 12 files changed, 138 insertions(+), 112 deletions(-) diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Config.scala b/src/main/scala/io/shiftleft/js2cpg/core/Config.scala index 6a34cbc54..fbe2266c4 100644 --- a/src/main/scala/io/shiftleft/js2cpg/core/Config.scala +++ b/src/main/scala/io/shiftleft/js2cpg/core/Config.scala @@ -13,26 +13,27 @@ import scala.util.matching.Regex object Config { - val SL_IGNORE_FILE: String = ".slignore" - val DEFAULT_TS_TYPES: Boolean = false - val DEFAULT_TS_TRANSPILING: Boolean = true - val DEFAULT_BABEL_TRANSPILING: Boolean = true - val DEFAULT_VUE_TRANSPILING: Boolean = true - val DEFAULT_NUXT_TRANSPILING: Boolean = true - val DEFAULT_TEMPLATE_TRANSPILING: Boolean = true - val DEFAULT_CPG_OUT_FILE: String = "cpg.bin.zip" - val DEFAULT_IGNORED_FILES_REGEX: Regex = "".r - val DEFAULT_IGNORED_FILES: Seq[Path] = Seq.empty - val DEFAULT_IGNORE_MINIFIED: Boolean = true - val DEFAULT_IGNORE_TESTS: Boolean = true - val DEFAULT_IGNORE_PRIVATE_DEPS: Boolean = false - val DEFAULT_PRIVATE_DEPS: Seq[String] = Seq.empty - val DEFAULT_INCLUDE_CONFIGS: Boolean = false - val DEFAULT_INCLUDE_HTML: Boolean = true - val DEFAULT_JVM_METRICS: Option[Int] = None - val DEFAULT_MODULE_MODE: Option[String] = None - val DEFAULT_WITH_NODE_MODULES_FOLDER: Boolean = false - val DEFAULT_OPTIMIZE_DEPENDENCIES: Boolean = true + val SL_IGNORE_FILE: String = ".slignore" + val DEFAULT_TS_TYPES: Boolean = false + val DEFAULT_TS_TRANSPILING: Boolean = true + val DEFAULT_BABEL_TRANSPILING: Boolean = true + val DEFAULT_VUE_TRANSPILING: Boolean = true + val DEFAULT_NUXT_TRANSPILING: Boolean = true + val DEFAULT_TEMPLATE_TRANSPILING: Boolean = true + val DEFAULT_CPG_OUT_FILE: String = "cpg.bin.zip" + val DEFAULT_IGNORED_FILES_REGEX: Regex = "".r + val DEFAULT_IGNORED_FILES: Seq[Path] = Seq.empty + val DEFAULT_IGNORE_MINIFIED: Boolean = true + val DEFAULT_IGNORE_TESTS: Boolean = true + val DEFAULT_IGNORE_PRIVATE_DEPS: Boolean = false + val DEFAULT_PRIVATE_DEPS: Seq[String] = Seq.empty + val DEFAULT_INCLUDE_CONFIGS: Boolean = false + val DEFAULT_INCLUDE_HTML: Boolean = true + val DEFAULT_JVM_METRICS: Option[Int] = None + val DEFAULT_MODULE_MODE: Option[String] = None + val DEFAULT_WITH_NODE_MODULES_FOLDER: Boolean = false + val DEFAULT_OPTIMIZE_DEPENDENCIES: Boolean = true + val DEFAULT_FIXED_TRANSPILATION_DEPENDENCIES: Boolean = false } @@ -57,7 +58,8 @@ case class Config( jvmMetrics: Option[Int] = Config.DEFAULT_JVM_METRICS, moduleMode: Option[String] = Config.DEFAULT_MODULE_MODE, withNodeModuleFolder: Boolean = Config.DEFAULT_WITH_NODE_MODULES_FOLDER, - optimizeDependencies: Boolean = Config.DEFAULT_OPTIMIZE_DEPENDENCIES + optimizeDependencies: Boolean = Config.DEFAULT_OPTIMIZE_DEPENDENCIES, + fixedTranspilationDependencies: Boolean = Config.DEFAULT_FIXED_TRANSPILATION_DEPENDENCIES ) { def createPathForPackageJson(): Path = Paths.get(packageJsonLocation) match { @@ -91,6 +93,7 @@ case class Config( |\t- package.json location: '${createPathForPackageJson()}' |\t- Module mode: '${moduleMode.getOrElse(TypescriptTranspiler.DEFAULT_MODULE)}' |\t- Optimize dependencies: $optimizeDependencies + |\t- Fixed transpilations dependencies: $fixedTranspilationDependencies |\t- Typescript transpiling: $tsTranspiling |\t- Babel transpiling: $babelTranspiling |\t- Vue.js transpiling: $vueTranspiling diff --git a/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala b/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala index d596b67b9..5731a7796 100644 --- a/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala +++ b/src/main/scala/io/shiftleft/js2cpg/core/Js2cpgArgumentsParser.scala @@ -9,38 +9,39 @@ import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler import scopt.OptionParser object Js2cpgArgumentsParser { - val HELP: String = "help" - val VERSION: String = "version" - val SRCDIR: String = "" - val OUTPUT: String = "output" - val WITH_TS_TYPES: String = "with-typescript-types" - val EXCLUDE: String = "exclude" - val EXCLUDE_REGEX: String = "exclude-regex" - val PACKAGE_JSON: String = "package-json" - val NO_TS: String = "no-ts" - val TS: String = "ts" - val NO_BABEL: String = "no-babel" - val NO_VUE: String = "no-vue-js" - val NO_NUXT: String = "no-nuxt-js" - val NO_TEMPLATES: String = "no-templates" - val BABEL: String = "babel" - val TRANSPILING: String = "transpiling" - val IGNORE_MINIFIED: String = "ignore-minified" - val WITH_MINIFIED: String = "with-minified" - val INCLUDE_MINIFIED: String = "include-minified" - val WITH_TESTS: String = "with-tests" - val INCLUDE_TESTS: String = "include-tests" - val IGNORE_PRIVATE_DEPS: String = "ignore-private-deps" - val EXCLUDE_PRIVATE_DEPS: String = "exclude-private-deps" - val PRIVATE_DEPS: String = "private-deps-ns" - val INCLUDE_CONFIGS: String = "include-configs" - val INCLUDE_HTML: String = "include-html" - val EXCLUDE_HTML: String = "exclude-html" - val JVM_MONITOR: String = "enable-jvm-monitor" - val MODULE_MODE: String = "module-mode" - val WITH_NODE_MODULES_FOLDER: String = "with-node-modules-folder" - val OPTIMIZE_DEPENDENCIES: String = "optimize-dependencies" - val ALL_DEPENDENCIES: String = "all-dependencies" + val HELP: String = "help" + val VERSION: String = "version" + val SRCDIR: String = "" + val OUTPUT: String = "output" + val WITH_TS_TYPES: String = "with-typescript-types" + val EXCLUDE: String = "exclude" + val EXCLUDE_REGEX: String = "exclude-regex" + val PACKAGE_JSON: String = "package-json" + val NO_TS: String = "no-ts" + val TS: String = "ts" + val NO_BABEL: String = "no-babel" + val NO_VUE: String = "no-vue-js" + val NO_NUXT: String = "no-nuxt-js" + val NO_TEMPLATES: String = "no-templates" + val BABEL: String = "babel" + val TRANSPILING: String = "transpiling" + val IGNORE_MINIFIED: String = "ignore-minified" + val WITH_MINIFIED: String = "with-minified" + val INCLUDE_MINIFIED: String = "include-minified" + val WITH_TESTS: String = "with-tests" + val INCLUDE_TESTS: String = "include-tests" + val IGNORE_PRIVATE_DEPS: String = "ignore-private-deps" + val EXCLUDE_PRIVATE_DEPS: String = "exclude-private-deps" + val PRIVATE_DEPS: String = "private-deps-ns" + val INCLUDE_CONFIGS: String = "include-configs" + val INCLUDE_HTML: String = "include-html" + val EXCLUDE_HTML: String = "exclude-html" + val JVM_MONITOR: String = "enable-jvm-monitor" + val MODULE_MODE: String = "module-mode" + val WITH_NODE_MODULES_FOLDER: String = "with-node-modules-folder" + val OPTIMIZE_DEPENDENCIES: String = "optimize-dependencies" + val ALL_DEPENDENCIES: String = "all-dependencies" + val FIXED_TRANSPILATION_DEPENDENCIES: String = "fixed-transpilation-dependencies" } class Js2cpgArgumentsParser { @@ -188,6 +189,11 @@ class Js2cpgArgumentsParser { s"install all project dependencies during transpilation (defaults to '${!Config.DEFAULT_OPTIMIZE_DEPENDENCIES}')" ) .action((_, c) => c.copy(optimizeDependencies = false)) + opt[Unit](FIXED_TRANSPILATION_DEPENDENCIES) + .text( + s"install fixed versions of transpilation dependencies during transpilation (defaults to '${!Config.DEFAULT_FIXED_TRANSPILATION_DEPENDENCIES}')" + ) + .action((_, c) => c.copy(fixedTranspilationDependencies = true)) opt[Int](JVM_MONITOR) .text("enable JVM metrics logging (requires JMX port number)") .action((jmxPortNumber, c) => c.copy(jvmMetrics = Some(jmxPortNumber))) diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/BabelTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/BabelTranspiler.scala index b8ee6f2c0..c15dee9fd 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/BabelTranspiler.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/BabelTranspiler.scala @@ -19,6 +19,12 @@ class BabelTranspiler( private val logger = LoggerFactory.getLogger(getClass) private val babel = Paths.get(projectPath.toString, "node_modules", ".bin", "babel").toString + private val babelPresets = + Versions.babelVersions.keySet.filter(_.contains("@babel/preset")).map(n => s"--presets $n ").mkString + + private val babelPlugins = + Versions.babelVersions.keySet.filter(_.contains("@babel/plugin")).map(n => s"--plugins $n ").mkString + override def shouldRun(): Boolean = config.babelTranspiling && !VueTranspiler.isVueProject(config, projectPath) @@ -39,17 +45,7 @@ class BabelTranspiler( val command = s"${ExternalCommand.toOSCommand(babel)} . " + "--no-babelrc " + s"--source-root '${in.toString}' " + - "--source-maps true " + - "--presets @babel/preset-env " + - "--presets @babel/preset-react " + - "--presets @babel/preset-flow " + - "--presets @babel/preset-typescript " + - "--plugins @babel/plugin-proposal-class-properties " + - "--plugins @babel/plugin-proposal-private-methods " + - "--plugins @babel/plugin-proposal-object-rest-spread " + - "--plugins @babel/plugin-proposal-nullish-coalescing-operator " + - "--plugins @babel/plugin-transform-property-mutators " + - "--plugins @babel/plugin-transform-runtime " + + "--source-maps true " + babelPresets + babelPlugins + s"--out-dir $outDir $constructIgnoreDirArgs" logger.debug(s"\t+ Babel transpiling $projectPath to $outDir with command '$command'") ExternalCommand.run(command, in.toString) match { diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/PugTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/PugTranspiler.scala index bd8901e2c..53c96816c 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/PugTranspiler.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/PugTranspiler.scala @@ -11,8 +11,9 @@ import scala.util.{Failure, Success} class PugTranspiler(override val config: Config, override val projectPath: Path) extends Transpiler { - private val logger = LoggerFactory.getLogger(getClass) - private val pug = Paths.get(projectPath.toString, "node_modules", ".bin", "pug").toString + private val logger = LoggerFactory.getLogger(getClass) + private val pug = Paths.get(projectPath.toString, "node_modules", ".bin", "pug").toString + private val pugAndVersion = Versions.nameAndVersion("pug-cli") private def hasPugFiles: Boolean = FileUtils.getFileTree(projectPath, config, List(PUG_SUFFIX)).nonEmpty @@ -21,11 +22,11 @@ class PugTranspiler(override val config: Config, override val projectPath: Path) private def installPugPlugins(): Boolean = { val command = if (pnpmAvailable(projectPath)) { - s"${TranspilingEnvironment.PNPM_ADD} pug-cli && ${TranspilingEnvironment.PNPM_INSTALL}" + s"${TranspilingEnvironment.PNPM_ADD} $pugAndVersion && ${TranspilingEnvironment.PNPM_INSTALL}" } else if (yarnAvailable()) { - s"${TranspilingEnvironment.YARN_ADD} pug-cli && ${TranspilingEnvironment.YARN_INSTALL}" + s"${TranspilingEnvironment.YARN_ADD} $pugAndVersion && ${TranspilingEnvironment.YARN_INSTALL}" } else { - s"${TranspilingEnvironment.NPM_INSTALL} pug-cli && ${TranspilingEnvironment.NPM_INSTALL}" + s"${TranspilingEnvironment.NPM_INSTALL} $pugAndVersion" } logger.info("Installing Pug dependencies and plugins. That will take a while.") logger.debug(s"\t+ Installing Pug plugins with command '$command' in path '$projectPath'") diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilerGroup.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilerGroup.scala index c6057e59d..f17841005 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilerGroup.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilerGroup.scala @@ -13,19 +13,7 @@ case class TranspilerGroup(override val config: Config, override val projectPath private val logger = LoggerFactory.getLogger(getClass) - private val BABEL_PLUGINS: String = - "@babel/core " + - "@babel/cli " + - "@babel/preset-env " + - "@babel/preset-flow " + - "@babel/preset-react " + - "@babel/preset-typescript " + - "@babel/plugin-proposal-class-properties " + - "@babel/plugin-proposal-private-methods " + - "@babel/plugin-proposal-object-rest-spread " + - "@babel/plugin-proposal-nullish-coalescing-operator " + - "@babel/plugin-transform-runtime " + - "@babel/plugin-transform-property-mutators" + private val BABEL_PLUGINS: String = Versions.babelVersions.keySet.map(Versions.nameAndVersion).mkString(" ") private def installPlugins(): Boolean = { val command = if (pnpmAvailable(projectPath)) { @@ -33,7 +21,7 @@ case class TranspilerGroup(override val config: Config, override val projectPath } else if (yarnAvailable()) { s"${TranspilingEnvironment.YARN_ADD} $BABEL_PLUGINS && ${TranspilingEnvironment.YARN_INSTALL}" } else { - s"${TranspilingEnvironment.NPM_INSTALL} $BABEL_PLUGINS && ${TranspilingEnvironment.NPM_INSTALL}" + s"${TranspilingEnvironment.NPM_INSTALL} $BABEL_PLUGINS" } logger.info("Installing project dependencies and plugins. That will take a while.") logger.debug(s"\t+ Installing plugins with command '$command' in path '$projectPath'") diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala index bc3b8efe5..27cdd45e4 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TranspilingEnvironment.scala @@ -24,15 +24,15 @@ object TranspilingEnvironment { val NPM: String = ExternalCommand.toOSCommand("npm") val YARN_ADD: String = - s"$YARN --prefer-offline --ignore-scripts --legacy-peer-deps --dev -W add" + s"$YARN --ignore-scripts --legacy-peer-deps --dev -W add" val YARN_INSTALL: String = - s"$YARN --prefer-offline --ignore-scripts --legacy-peer-deps install" + s"$YARN --ignore-scripts --legacy-peer-deps install" val PNPM_ADD: String = - s"$PNPM --prefer-offline --ignore-scripts add -D" + s"$PNPM --ignore-scripts add -D" val PNPM_INSTALL: String = - s"$PNPM --prefer-offline --ignore-scripts install" + s"$PNPM --ignore-scripts install" val NPM_INSTALL: String = - s"$NPM --prefer-offline --no-audit --progress=false --ignore-scripts --legacy-peer-deps --save-dev install" + s"$NPM --no-audit --progress=false --ignore-scripts --legacy-peer-deps --save-dev install" } trait TranspilingEnvironment { @@ -42,6 +42,35 @@ trait TranspilingEnvironment { private val logger = LoggerFactory.getLogger(getClass) + object Versions { + val babelVersions: Map[String, String] = Map( + "@babel/core" -> "7.20.2", + "@babel/cli" -> "7.19.3", + "@babel/preset-env" -> "7.20.2", + "@babel/preset-flow" -> "7.18.6", + "@babel/preset-react" -> "7.18.6", + "@babel/preset-typescript" -> "7.18.6", + "@babel/plugin-proposal-class-properties" -> "7.18.6", + "@babel/plugin-proposal-private-methods" -> "7.18.6", + "@babel/plugin-proposal-object-rest-spread" -> "7.20.2", + "@babel/plugin-proposal-nullish-coalescing-operator" -> "7.18.6", + "@babel/plugin-transform-runtime" -> "7.19.6", + "@babel/plugin-transform-property-mutators" -> "7.18.6" + ) + + private val versions: Map[String, String] = + babelVersions ++ Map("pug-cli" -> "1.0.0-alpha6", "typescript" -> "4.8.4", "@vue/cli-service-global" -> "4.5.19") + + def nameAndVersion(dependencyName: String): String = { + if (config.fixedTranspilationDependencies) { + val version = versions.get(dependencyName).map(v => s"@$v").getOrElse("") + s"$dependencyName$version" + } else { + dependencyName + } + } + } + private def checkForPnpm(): Boolean = { logger.debug(s"\t+ Checking pnpm ...") ExternalCommand.run(s"${TranspilingEnvironment.PNPM} -v", projectPath.toString) match { diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala index 922c69509..145b19491 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/TypescriptTranspiler.scala @@ -12,6 +12,7 @@ import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler.DEFAULT_MODULE import io.shiftleft.js2cpg.preprocessing.TypescriptTranspiler.DENO_CONFIG import org.slf4j.LoggerFactory import org.apache.commons.io.{FileUtils => CommonsFileUtils} + import java.nio.file.{Path, Paths} import scala.util.{Failure, Success, Try} @@ -33,7 +34,8 @@ class TypescriptTranspiler(override val config: Config, override val projectPath private val NODE_OPTIONS: Map[String, String] = Map("NODE_OPTIONS" -> "--max_old_space_size=4096") - private val tsc = Paths.get(projectPath.toString, "node_modules", ".bin", "tsc").toString + private val tsc = Paths.get(projectPath.toString, "node_modules", ".bin", "tsc").toString + private val typescriptAndVersion = Versions.nameAndVersion("typescript") private def hasTsFiles: Boolean = FileUtils.getFileTree(projectPath, config, List(TS_SUFFIX)).nonEmpty @@ -91,11 +93,11 @@ class TypescriptTranspiler(override val config: Config, override val projectPath private def installTsPlugins(): Boolean = { val command = if (pnpmAvailable(projectPath)) { - s"${TranspilingEnvironment.PNPM_ADD} typescript" + s"${TranspilingEnvironment.PNPM_ADD} $typescriptAndVersion" } else if (yarnAvailable()) { - s"${TranspilingEnvironment.YARN_ADD} typescript" + s"${TranspilingEnvironment.YARN_ADD} $typescriptAndVersion" } else { - s"${TranspilingEnvironment.NPM_INSTALL} typescript" + s"${TranspilingEnvironment.NPM_INSTALL} $typescriptAndVersion" } logger.info("Installing TypeScript dependencies and plugins. That will take a while.") logger.debug(s"\t+ Installing Typescript plugins with command '$command' in path '$projectPath'") diff --git a/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala b/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala index a1eeb15a9..efe0c01fa 100644 --- a/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala +++ b/src/main/scala/io/shiftleft/js2cpg/preprocessing/VueTranspiler.scala @@ -33,7 +33,8 @@ class VueTranspiler(override val config: Config, override val projectPath: Path) private lazy val NODE_OPTIONS: Map[String, String] = nodeOptions() - private val vue = Paths.get(projectPath.toString, "node_modules", ".bin", "vue-cli-service").toString + private val vue = Paths.get(projectPath.toString, "node_modules", ".bin", "vue-cli-service").toString + private val vueAndVersion = Versions.nameAndVersion("@vue/cli-service-global") override def shouldRun(): Boolean = config.vueTranspiling && isVueProject(config, projectPath) @@ -48,11 +49,11 @@ class VueTranspiler(override val config: Config, override val projectPath: Path) private def installVuePlugins(): Boolean = { val command = if (pnpmAvailable(projectPath)) { - s"${TranspilingEnvironment.PNPM_ADD} @vue/cli-service-global && ${TranspilingEnvironment.PNPM_INSTALL}" + s"${TranspilingEnvironment.PNPM_ADD} $vueAndVersion && ${TranspilingEnvironment.PNPM_INSTALL}" } else if (yarnAvailable()) { - s"${TranspilingEnvironment.YARN_ADD} @vue/cli-service-global && ${TranspilingEnvironment.YARN_INSTALL}" + s"${TranspilingEnvironment.YARN_ADD} $vueAndVersion && ${TranspilingEnvironment.YARN_INSTALL}" } else { - s"${TranspilingEnvironment.NPM_INSTALL} @vue/cli-service-global && ${TranspilingEnvironment.NPM_INSTALL}" + s"${TranspilingEnvironment.NPM_INSTALL} $vueAndVersion" } logger.info("Installing Vue.js dependencies and plugins. That will take a while.") logger.debug(s"\t+ Installing Vue.js plugins with command '$command' in path '$projectPath'") diff --git a/src/test/scala/io/shiftleft/js2cpg/fixtures/DataFlowCode2CpgFixture.scala b/src/test/scala/io/shiftleft/js2cpg/fixtures/DataFlowCode2CpgFixture.scala index d432d3f66..bdea5c939 100644 --- a/src/test/scala/io/shiftleft/js2cpg/fixtures/DataFlowCode2CpgFixture.scala +++ b/src/test/scala/io/shiftleft/js2cpg/fixtures/DataFlowCode2CpgFixture.scala @@ -33,7 +33,7 @@ trait Js2CpgFrontend extends LanguageFrontend { outputFile = cpgFile.pathAsString ) js2cpg.run(config) - cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgFile.pathAsString)) + cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgFile.pathAsString)) } cpg } diff --git a/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala index 257c7282a..e5c19854e 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala @@ -43,7 +43,7 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec defaultArgs.map(_.toArg).toArray ) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain theSameElementsAs expectedFiles.map(_.replace("/", java.io.File.separator)) cpg.close() diff --git a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala index 6a536aa45..6291de3d3 100644 --- a/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala @@ -31,7 +31,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpProjectPath.pathAsString, "--output", cpgPath.pathAsString, "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain allElementsOf Set( s"@privateA${java.io.File.separator}a.js", @@ -59,7 +59,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { ) ) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain allElementsOf Set( s"@privateA${java.io.File.separator}a.js", @@ -89,7 +89,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { ) ) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain only s"@privateB${java.io.File.separator}b.js" cpg.close() @@ -105,7 +105,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpProjectPath.pathAsString, "--output", cpgPath.pathAsString, "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain only "index.js" cpg.close() diff --git a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala index c84a09d03..bbfae2635 100644 --- a/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala +++ b/src/test/scala/io/shiftleft/js2cpg/preprocessing/TranspilationRunnerTest.scala @@ -66,7 +66,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-ts")) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain theSameElementsAs List("foo.js") codeFields(cpg) should contain allElementsOf List( @@ -93,7 +93,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain theSameElementsAs List("a.ts", "b.ts") lineNumbers(cpg) should contain allElementsOf List(1, 1) @@ -146,7 +146,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain theSameElementsAs List( "index.js", @@ -169,7 +169,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-babel", "--with-tests")) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain theSameElementsAs List( "a.ts", @@ -187,7 +187,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString)) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain theSameElementsAs List( "a.js", @@ -205,7 +205,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString)) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain theSameElementsAs List( "a.js", @@ -222,7 +222,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString)) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain theSameElementsAs List( s"src${java.io.File.separator}main.js", @@ -238,7 +238,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString)) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain theSameElementsAs List( s"src${java.io.File.separator}views${java.io.File.separator}AboutPage.vue", @@ -256,7 +256,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-ts", "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) fileNames(cpg) should contain only "test.ejs" lineNumbers(cpg) should contain allElementsOf List(5, 7, 15, 16, 21) @@ -270,7 +270,7 @@ class TranspilationRunnerTest extends AnyWordSpec with Matchers { val cpgPath = tmpDir / "cpg.bin.zip" Js2CpgMain.main(Array(tmpDir.pathAsString, "--output", cpgPath.pathAsString, "--no-ts", "--no-babel")) - val cpg = Cpg.withConfig(overflowdb.Config.withDefaults.withStorageLocation(cpgPath.pathAsString)) + val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString)) // Sadly, calling the pug transpiler via pug cli does not support source maps: fileNames(cpg) should contain only "test.js"