From 169011377a4eb207a3a2c52055fe11dde1797b46 Mon Sep 17 00:00:00 2001 From: name_snrl Date: Sat, 29 Jul 2023 16:59:30 +0700 Subject: [PATCH] feat: init version --- .gitignore | 2 + deps/BUILD | 2 + deps/BazelExt.scala | 39 +++++++++ deps/Coordinates.scala | 29 +++++++ deps/Deps.scala | 82 +++++------------- deps/MakeTree.scala | 38 ++++++++ deps/Resolve.scala | 63 ++++++++++++++ deps/Target.scala | 106 +++++++++++++++++++++++ deps/Vars.scala | 18 ++++ deps/package.scala | 35 ++++++++ deps/templates/jar_artifact_callback.bzl | 32 +++++++ 11 files changed, 388 insertions(+), 58 deletions(-) create mode 100644 deps/BazelExt.scala create mode 100644 deps/Coordinates.scala create mode 100644 deps/MakeTree.scala create mode 100644 deps/Resolve.scala create mode 100644 deps/Target.scala create mode 100644 deps/Vars.scala create mode 100644 deps/package.scala create mode 100644 deps/templates/jar_artifact_callback.bzl diff --git a/.gitignore b/.gitignore index b52cc58f..f6adaad9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ bazel-* +.direnv/ *.gen.scala *.swp *.idea @@ -21,3 +22,4 @@ target/ /.settings /.vscode/ /bazel.iml +/.metals/ diff --git a/deps/BUILD b/deps/BUILD index 44c8608d..e501ff97 100644 --- a/deps/BUILD +++ b/deps/BUILD @@ -4,6 +4,8 @@ scala_binary( name = "deps", srcs = glob(["*.scala"]), main_class = "rules_scala3.deps.Deps", + resource_strip_prefix = package_name(), + resources = ["templates/jar_artifact_callback.bzl"], scala = "//scala:bootstrap_3_3", visibility = ["//visibility:public"], # jvm_flags = ["-Djava.security.manager=allow"], diff --git a/deps/BazelExt.scala b/deps/BazelExt.scala new file mode 100644 index 00000000..95fcc251 --- /dev/null +++ b/deps/BazelExt.scala @@ -0,0 +1,39 @@ +package rules_scala3.deps + +import scala.io.Source +import java.io.FileNotFoundException + +object BazelExt: + private def loadResourceToString(path: String): String = + val resourceStream = getClass.getResourceAsStream(path) + if resourceStream == null then throw new FileNotFoundException(s"Resource not found: $path") + try Source.fromInputStream(resourceStream).mkString finally resourceStream.close + + private lazy val jarArtifactCallback = loadResourceToString("/templates/jar_artifact_callback.bzl") + + def apply(targets: Vector[Target]): String = + val dependencyLines: String = targets + .filterNot(_.replacement_label.isDefined) + .map { t => + Vector( + s""" {"artifact": "${t.coordinates.toString}"""", + s""""url": "${t.url}"""", + s""""name": "${t.name}"""", + s""""actual": "${t.actual}"""", + s""""bind": "${t.bind}"},""" + ).mkString(", ") + } + .mkString("\n") + + s"""# Do not edit. rules_scala3 autogenerates this file + |$jarArtifactCallback + | + |def list_dependencies(): + | return [ + |$dependencyLines + | ] + | + |def maven_dependencies(callback = jar_artifact_callback): + | for hash in list_dependencies(): + | callback(hash) + |""".stripMargin diff --git a/deps/Coordinates.scala b/deps/Coordinates.scala new file mode 100644 index 00000000..8e8b0e64 --- /dev/null +++ b/deps/Coordinates.scala @@ -0,0 +1,29 @@ +package rules_scala3.deps + +import sbt.librarymanagement.ModuleID +import scala.util.matching.Regex.Groups + +case class GroupId(grp: String): + override def toString: String = grp + def toUnixPath: String = grp.replaceAll("\\.", "/").replaceAll("-", "_") + +case class Coordinates(groupId: GroupId, artifactId: String, version: Option[String]): + override def toString: String = version match + case Some(v) => s"${groupId}:${artifactId}:${v}" + case None => s"${groupId}:${artifactId}" + + def withCleanName: Coordinates = copy(artifactId = this.cleanName) + + def cleanName: String = + val scalaNameVersion = """^(.*)_([23](?:\.\d{1,2})?(?:[\.-].*)?)$""".r + def mkName: String = artifactId match + case scalaNameVersion(n, _) => n + case _ => artifactId + mkName.replaceAll("[.\\-]", "_") + +object Coordinates: + def apply(groupId: String, artifactId: String, version: String): Coordinates = + Coordinates(GroupId(groupId), artifactId, Some(version)) + + def apply(groupId: String, artifactId: String): Coordinates = + Coordinates(GroupId(groupId), artifactId, None) diff --git a/deps/Deps.scala b/deps/Deps.scala index 2cf7b766..aad41cab 100644 --- a/deps/Deps.scala +++ b/deps/Deps.scala @@ -1,67 +1,33 @@ package rules_scala3.deps import java.io.File - -import lmcoursier.* -import sbt.internal.librarymanagement.cross.CrossVersionUtil -import sbt.internal.util.ConsoleLogger -import sbt.librarymanagement.* -import sbt.librarymanagement.Configurations.Component -import sbt.librarymanagement.Resolver.{DefaultMavenRepository, JCenterRepository, JavaNet2Repository} import sbt.librarymanagement.syntax.* -object Deps: - def main(args: Array[String]): Unit = - // https://github.com/coursier/sbt-coursier/blob/3df025313bf010d80b8b9288c76fa6a3cb13c7d0/modules/lm-coursier/src/test/scala/lmcoursier/ResolutionSpec.scala - - lazy val log = ConsoleLogger() - - val lmEngine = CoursierDependencyResolution(CoursierConfiguration().withResolvers(resolvers)) - - // val module = "commons-io" % "commons-io" % "2.5" - // lm.retrieve(module, scalaModuleInfo = None, File("/tmp/target"), log) - - val stubModule = "com.example" % "foo" % "0.1.0" % "compile" - val dependencies = Vector( - "com.typesafe.scala-logging" % "scala-logging_2.12" % "3.7.2" % "compile", - "org.scalatest" % "scalatest_2.12" % "3.0.4" % "test" - ).map(_.withIsTransitive(false)) - val coursierModule = module(lmEngine, stubModule, dependencies, Some("2.12.4")) - val resolution = - lmEngine.update(coursierModule, UpdateConfiguration(), UnresolvedWarningConfiguration(), log) - val r = resolution.right.get - println(s"r:$r") - end main - - def resolvers = Vector( - DefaultMavenRepository, - JavaNet2Repository, - JCenterRepository, - Resolver.sbtPluginRepo("releases") +@main def Deps(args: String*): Unit = + // parse args + vars = Vars( + projectRoot = new File("/home/name_snrl/work/forks/rules_scala3"), + depsDirName = "3rdparty", + bazelExtFileName = "workspace.bzl", + buildFilesDirName = "jvm", + buildFileName = "BUILD", + scalaVersion = "3.3.0", + buildFileHeader = """load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import")""" ) - def configurations = Vector(Compile, Test, Runtime, Provided, Optional, Component) +// Replacements are not handled by `librarymanagement`. any Scala prefix in the name will be dropped. +// It also doesn't matter whether you use double `%` to get the Scala version or not. + val replacements = Map( + "org.scala-lang" % "scala-compiler" -> "@io_bazel_rules_scala_scala_compiler//:io_bazel_rules_scala_scala_compiler", + "org.scala-lang" % "scala-library" -> "@io_bazel_rules_scala_scala_library//:io_bazel_rules_scala_scala_library", + "org.scala-lang" % "scala-reflect" -> "@io_bazel_rules_scala_scala_reflect//:io_bazel_rules_scala_scala_reflect", + "org.scala-lang.modules" % "scala-parser-combinators" -> "@io_bazel_rules_scala_scala_parser_combinators//:io_bazel_rules_scala_scala_parser_combinators", + "org.scala-lang.modules" % "scala-xml" -> "@io_bazel_rules_scala_scala_xml//:io_bazel_rules_scala_scala_xml", + ) - def module( - lmEngine: DependencyResolution, - moduleId: ModuleID, - deps: Vector[ModuleID], - scalaFullVersion: Option[String], - overrideScalaVersion: Boolean = true - ): ModuleDescriptor = - val scalaModuleInfo = scalaFullVersion map { fv => - ScalaModuleInfo( - scalaFullVersion = fv, - scalaBinaryVersion = CrossVersionUtil.binaryScalaVersion(fv), - configurations = configurations, - checkExplicit = true, - filterImplicit = false, - overrideScalaVersion = overrideScalaVersion - ) - } + val dependencies = Vector( + "org.scala-sbt" % "librarymanagement-core_3" % "2.0.0-alpha12", + "org.scala-sbt" % "librarymanagement-coursier_3" % "2.0.0-alpha6", + ) - val moduleSetting = ModuleDescriptorConfiguration(moduleId, ModuleInfo("foo")) - .withDependencies(deps) - .withConfigurations(configurations) - .withScalaModuleInfo(scalaModuleInfo) - lmEngine.moduleDescriptor(moduleSetting) + MakeTree(dependencies, replacements) diff --git a/deps/MakeTree.scala b/deps/MakeTree.scala new file mode 100644 index 00000000..0a04a9a4 --- /dev/null +++ b/deps/MakeTree.scala @@ -0,0 +1,38 @@ +package rules_scala3.deps + +import sbt.librarymanagement.{DependencyBuilders, ModuleID}, DependencyBuilders.OrganizationArtifactName +import sbt.librarymanagement.syntax.* +import java.io.{File, FileWriter} + +object MakeTree: + def apply(dependencies: Vector[ModuleID], replacements: Map[OrganizationArtifactName, String]): Unit = + val targets = Resolve(dependencies, replacements.map((k, v) => (k % "0.1.0").toUvCoordinates.withCleanName -> v)) + val bazelExtContent = BazelExt(targets) + writeTree(targets, bazelExtContent) + + private def writeToFile(path: File, content: String): Unit = + val dirname = path.getParentFile + if !dirname.exists then dirname.mkdirs + val writer = new FileWriter(path) + try writer.write(content) + finally writer.close + + private def writeTree( + targets: Vector[Target], + bazelExtContent: String + ): Unit = + // create bazel extension file + writeToFile(vars.bazelExtFile, bazelExtContent) + + // create an empty `BUILD` file in the root directory to mark it as + // a package and call the extensions + writeToFile(vars.depsBuildFile, "") + + // make tree of BUILD files + targets + .groupBy(_.coordinates.groupId) + .foreach { (group, targets) => + val file = new File(vars.treeOfBUILDsFile, group.toUnixPath + File.separator + vars.buildFileName) + val content = vars.buildFileHeader + targets.map(_.toBzl).mkString + writeToFile(file, content) + } diff --git a/deps/Resolve.scala b/deps/Resolve.scala new file mode 100644 index 00000000..2546907e --- /dev/null +++ b/deps/Resolve.scala @@ -0,0 +1,63 @@ +package rules_scala3.deps + +import lmcoursier.{CoursierConfiguration, CoursierDependencyResolution} +import sbt.internal.util.ConsoleLogger +import sbt.librarymanagement.syntax.* +import sbt.librarymanagement.{ModuleDescriptorConfiguration, ModuleID, ModuleInfo, UnresolvedWarningConfiguration, UpdateConfiguration} +import scala.collection.mutable.HashMap + +object Resolve: + def apply(dependencies: Vector[ModuleID], replacements: Map[Coordinates, String]): Vector[Target] = + val csConfig = CoursierConfiguration() + .withScalaVersion(Some(vars.scalaVersion)) + .withClasspathOrder(false) // it just gets in the way and creates duplicates. + + val moduleConfig = ModuleDescriptorConfiguration("com.example" % "foo" % "0.1.0", ModuleInfo("foo")) + .withDependencies(dependencies) + + val lmEngine = CoursierDependencyResolution(csConfig) + + val resolution = lmEngine.update( + module = lmEngine.moduleDescriptor(moduleConfig), + configuration = UpdateConfiguration(), + uwconfig = UnresolvedWarningConfiguration(), + log = ConsoleLogger() + ) + // convert librarymanagement modules to our targets + resolution match + case Left(e) => throw e.resolveException + case Right(resolution) => + val modules = resolution.configurations.head.modules + // filter out dependencies of replacement + .map(_.filterCallers(replacements)) + .filterNot { moduleReport => + moduleReport.callers.isEmpty && !dependencies + .map(_.toUvCoordinates.withCleanName) + .contains(moduleReport.module.toUvCoordinates.withCleanName) + } + val uvCoordinates_modules = modules.map { m => m.module.toUvCoordinates -> m.module }.toMap + val modules_deps: HashMap[Coordinates, Vector[Coordinates]] = HashMap.empty + modules.foreach { moduleReport => + moduleReport.callers.foreach { caller => + // dependency can be resolved with a different version, + // so the comparison is done by org-name pair. + val uvCaller = caller.caller.toUvCoordinates + val coordinates = uvCoordinates_modules.get(uvCaller) match + case Some(m) => m.toCoordinates + case None => sys.error(s"""The organization-name pair was not found in the resolved modules. + |${uvCaller.toString} is missing.""".stripMargin) + if modules_deps.contains(coordinates) + then modules_deps(coordinates) = modules_deps(coordinates) :+ moduleReport.module.toCoordinates + else modules_deps += (coordinates -> Vector(moduleReport.module.toCoordinates)) + } + } + modules + .map { moduleReport => + Target( + moduleReport = moduleReport, + replacement_label = replacements.get(moduleReport.module.toUvCoordinates.withCleanName), + isDirect = dependencies.map(_.cleanName).contains(moduleReport.module.cleanName), + module_deps = modules_deps.toMap.getOrElse(moduleReport.module.toCoordinates, Vector.empty).sortBy(_.toString) + ) + } + .sortBy(_.name) diff --git a/deps/Target.scala b/deps/Target.scala new file mode 100644 index 00000000..eac47632 --- /dev/null +++ b/deps/Target.scala @@ -0,0 +1,106 @@ +package rules_scala3.deps + +import sbt.librarymanagement.{Artifact, ModuleID, ModuleReport} + +case class Target( + coordinates: Coordinates, + replacement_label: Option[String], + lang: Language, + name: String, + visibility: Target.Visibility, + actual: String, + bind: String, + jar: String, + url: String, + deps: Vector[Coordinates] +): + def toBzl: String = + replacement_label match + case Some(replacement_label) => + s"""\n${lang.asString}_import( + | name = "${coordinates.cleanName}", + | exports = [ + | "$replacement_label" + | ], + | visibility = [ + | "${visibility.asString}" + | ] + |)\n""".stripMargin + case None => + val runtime_deps = + val deps0 = deps + .map { c => + if coordinates.groupId == c.groupId + then s"\":${c.cleanName}\"" + else s"\"${vars.treeOfBUILDsBazelPath}/${c.groupId.toUnixPath}:${c.cleanName}\"" + } + .sorted + .mkString(",\n ") + if deps0 == "" + then "" + else s""" + | runtime_deps = [ + | $deps0 + | ],""".stripMargin + s"""\n${lang.asString}_import( + | name = "${coordinates.cleanName}", + | jars = [ + | "$jar" + | ],$runtime_deps + | visibility = [ + | "${visibility.asString}" + | ] + |)\n""".stripMargin + +object Target: + enum Visibility(val asString: String): + case Public extends Visibility("//visibility:public") + case SubPackages extends Visibility(s"${vars.treeOfBUILDsBazelPath}:__subpackages__") + + def apply( + moduleReport: ModuleReport, + replacement_label: Option[String], + isDirect: Boolean, + module_deps: Vector[Coordinates] + ): Target = + val coordinates = moduleReport.module.toCoordinates + val lang = moduleReport.module.language + val name: String = s"${coordinates.groupId}_${coordinates.artifactId}".replaceAll("[.\\-]", "_") + val visibility: Target.Visibility = + if isDirect + then Target.Visibility.Public + else Target.Visibility.SubPackages + val actual: String = s"@$name//jar" + val bind: String = s"jar/${coordinates.groupId.toUnixPath}/${coordinates.artifactId}".replaceAll("[.\\-:]", "_") + val jar: String = s"//external:$bind" + val url: String = + def e = sys.error(s"No url in artifact field of ${moduleReport.module.toString}") + moduleReport.artifacts.head.head.url.fold(e) { u => u.toString } // TODO are you sure there can only be one artifact? + + replacement_label match + case Some(replacement_label) => + Target( + coordinates = coordinates, + replacement_label = Some(replacement_label), + lang = lang, + name = name, + visibility = visibility, + actual = "", + bind = "", + jar = "", + url = "", + deps = Vector.empty + ) + case None => + Target( + coordinates = coordinates, + replacement_label = None, + lang = lang, + name = name, + visibility = visibility, + actual = actual, + bind = bind, + jar = jar, + url = url, + deps = module_deps + ) diff --git a/deps/Vars.scala b/deps/Vars.scala new file mode 100644 index 00000000..ba61d956 --- /dev/null +++ b/deps/Vars.scala @@ -0,0 +1,18 @@ +package rules_scala3.deps + +import java.io.File + +case class Vars( + projectRoot: File, + depsDirName: String, + bazelExtFileName: String, + buildFilesDirName: String, + buildFileName: String, + scalaVersion: String, + buildFileHeader: String +): + def depsFile: File = new File(projectRoot, depsDirName) + def bazelExtFile: File = new File(depsFile, bazelExtFileName) + def depsBuildFile: File = new File(depsFile, buildFileName) + def treeOfBUILDsFile: File = new File(depsFile, buildFilesDirName) + def treeOfBUILDsBazelPath: String = s"//$depsDirName/$buildFilesDirName" diff --git a/deps/package.scala b/deps/package.scala new file mode 100644 index 00000000..df0d3a11 --- /dev/null +++ b/deps/package.scala @@ -0,0 +1,35 @@ +package rules_scala3.deps + +import sbt.librarymanagement.{ModuleID, ModuleReport} +import java.io.File + +var vars = Vars( + projectRoot = new File(""), + depsDirName = "", + bazelExtFileName = "", + buildFilesDirName = "", + buildFileName = "", + scalaVersion = "", + buildFileHeader = "" +) + +enum Language(val asString: String): + case Scala extends Language("scala") + case Java extends Language("java") + +extension (m: ModuleID) + def toCoordinates: Coordinates = Coordinates(groupId = m.organization, artifactId = m.name, version = m.revision) + def toUvCoordinates: Coordinates = Coordinates(groupId = m.organization, artifactId = m.name) + def language: Language = + if "_[23]".r.findFirstIn(m.name).isDefined || m.organization == "org.scala-lang" + then Language.Scala + else Language.Java + def cleanName: String = + val scalaNameVersion = """^(.*)_([23](?:\.\d{1,2})?(?:[\.-].*)?)$""".r + m.name match + case scalaNameVersion(n, _) => n + case _ => m.name + +extension (m: ModuleReport) + def filterCallers(replacements: Map[Coordinates, String]): ModuleReport = + m.withCallers(m.callers.filterNot(caller => replacements.contains(caller.caller.toUvCoordinates.withCleanName))) diff --git a/deps/templates/jar_artifact_callback.bzl b/deps/templates/jar_artifact_callback.bzl new file mode 100644 index 00000000..e778630a --- /dev/null +++ b/deps/templates/jar_artifact_callback.bzl @@ -0,0 +1,32 @@ +def _jar_artifact_impl(ctx): + jar_name = "%s.jar" % ctx.name + ctx.download( + output = ctx.path("jar/%s" % jar_name), + url = ctx.attr.urls, + executable = False + ) + build_file_contents = """ +package(default_visibility = ['//visibility:public']) +filegroup( + name = 'jar', + srcs = ['{jar_name}'], + visibility = ['//visibility:public'] +)\n""".format(artifact = ctx.attr.artifact, jar_name = jar_name) + ctx.file(ctx.path("jar/BUILD"), build_file_contents, False) + return None + +jar_artifact = repository_rule( + attrs = { + "artifact": attr.string(mandatory = True), + "urls": attr.string_list(mandatory = True), + }, + implementation = _jar_artifact_impl +) + +def jar_artifact_callback(hash): + jar_artifact( + artifact = hash["artifact"], + name = hash["name"], + urls = [hash["url"]], + ) + native.bind(name = hash["bind"], actual = hash["actual"])