Fix cross versions in mill (#116)

* Mittigate cyclic dependencies in the build plan * Update version to v0.1.2 * Less eager overrides in scaalfix rules for mill * Update config * Filter out tapir-loom - no JDK 19 support yet
VirtusLab · Oct 8, 2022 · 8c3572a · 8c3572a
1 parent c372eee
commit 8c3572a
Show file tree

Hide file tree

Showing 15 changed files with 79 additions and 40 deletions.
diff --git a/README.md b/README.md
@@ -93,19 +93,19 @@ eval $(minikube -p minikube docker-env)
 Most likely you'll need to build the base image only once (it doesn't get modified too often but building it takes quite a lot of time), e.g.:
 
 ```shell
-scripts/build-builder-base.sh v0.1.1
+scripts/build-builder-base.sh v0.1.2
 ```
 
 Build all the remaining images
 
 ```shell
-scripts/build-quick.sh v0.1.1
+scripts/build-quick.sh v0.1.2
 ```
 
 or (re)build each image separately e.g.
 
 ```shell
-scripts/build-mvn-repo.sh v0.1.1
+scripts/build-mvn-repo.sh v0.1.2
 ```
 
 ### Deploying and debugging in k8s

diff --git a/cli/scb-cli.scala b/cli/scb-cli.scala
@@ -25,7 +25,7 @@ given ExecutionContext = ExecutionContext.Implicits.global
 
 class FailedProjectException(msg: String) extends RuntimeException(msg) with NoStackTrace
 
-val communityBuildVersion = sys.props.getOrElse("communitybuild.version", "v0.1.1")
+val communityBuildVersion = sys.props.getOrElse("communitybuild.version", "v0.1.2")
 private val CBRepoName = "VirtusLab/community-build3"
 val projectBuilderUrl = s"https://raw.githubusercontent.com/$CBRepoName/master/project-builder"
 lazy val communityBuildDir = sys.props

diff --git a/coordinator/src/main/scala/buildPlan.scala b/coordinator/src/main/scala/buildPlan.scala
@@ -290,19 +290,24 @@ private given FromString[Seq[Project]] = str =>
     ): List[Set[ProjectBuildDef]] =
       if remaining.isEmpty then acc.reverse
       else
-        val (currentStage, newRemainings) = remaining.partition {
+        var (currentStage, newRemainings) = remaining.partition {
           _.dependencies.forall(done.contains)
         }
         if currentStage.isEmpty then {
           val deps = plan.map(v => (v.name, v)).toMap
-          newRemainings
-            .filter(p => p.dependencies.exists { d => deps(d).dependencies.contains(p.name) })
-            .foreach(v =>
-              println(
-                s"cyclic dependency in  ${v.name} -> ${v.dependencies.toList.filterNot(done.contains)}"
-              )
+          def hasCyclicDependencies(p: ProjectBuildDef) =
+            p.dependencies.exists(deps(_).dependencies.contains(p.name))
+          val (cyclicDeps, remaining) = newRemainings.partition(hasCyclicDependencies)
+          currentStage ++= cyclicDeps
+          newRemainings --= cyclicDeps
+
+          cyclicDeps.foreach(v =>
+            println(
+              s"Mitigated cyclic dependency in  ${v.name} -> ${v.dependencies.toList.filterNot(done.contains)}"
             )
-          sys.error("cyclic dependency")
+          )
+          if remaining.nonEmpty then
+            sys.error(s"Unresolved cyclic dependencies involving: ${remaining.map(_.name)}")
         }
         val names = currentStage.map(_.name)
         groupByDeps(newRemainings, done ++ names, currentStage :: acc)

diff --git a/env/prod/config/filtered-projects.txt b/env/prod/config/filtered-projects.txt
@@ -49,7 +49,7 @@ pme123:camundala-dsl:.*
 
 # Build problems:
 ## Incorrect handling of Scala 3 binary versions
-dmurvihill:courier:.*
+wartremover:.*
 inkytonik:kiama:.*
 kevin-lee:can-equal:.*
 kevin-lee:just-fp:.*
@@ -66,6 +66,24 @@ sciss:swingplus:.*
 ## Uses mill-tpolecat - it does fails to parse RC versions 
 joan38:kubernetes-client:.*
 carlosedp:riscvassembler:.*
+neandertech:jsonrpclib:.*
+
+
+## Missing artifacts in public repo
+# sbt-beangle-parent:0.6.0
+beangle:boot:.*
+beangle:cache:.*
+beangle:cdi:.*
+beangle:commons:.*
+beangle:data:.*
+beangle:security:.*
+beangle:serializer:.*
+beangle:template:.*
+beangle:web:.*
+beangle:webmvc:.*
+
+## Misconfigured + needs ruby on path
+makenowjust-labs:lite:.*
 
 ## Missing artifacts in public repo
 # sbt-beangle-parent:0.6.0
@@ -119,14 +137,14 @@ pjfanning:scala3-reflection:.*
 johnhungerford:generic-schema:1.[0-1].[0.1]
 
 #https://github.com/lampepfl/dotty/issues/15668
-kevin-lee:extras:0.[0-1].*
+kevin-lee:extras:0.*
 
 
 # opaque type overrides
 lorandszakacs:sprout:0.0.[0-5]	
 
 # Changes to Compiler API
-nrinaudo:kantan.repl:1.1.*
+nrinaudo:kantan.repl:1.*
 virtuslab:pretty-stacktraces:0.0.*
 zygfryd:scala-zygf-cement:0.3.0
 
@@ -177,16 +195,6 @@ laserdisc-io:fs2-aws:5.0.*
 ## override opaque type
 tabdulradi:mazboot:.*
 
-# Cyclic dependency
-## reactivemongo/reactivemongo
-reactivemongo:reactivemongo-bson:.*
-
-##japgolly/microlibs-scala
-japgolly:nyaya.*
-
-## errors4s_errors4s-core
-errors4s:errors4s-core-scalacheck:.*
-
 # Unsupported minimal JDK version
 ## JDK 19
 softwaremill:tapir-loom:.*
diff --git a/env/prod/config/projects-config.conf b/env/prod/config/projects-config.conf
@@ -51,6 +51,7 @@ augustjune_canoe {
 }
 avast_datadog4s.java.version=11
 
+babylonhealth_lit-fhir.tests = compile-only
 benhutchison_scalaswingcontrib.tests = disabled // Does not compile with any Scala version
 bilal-fazlani_akka-http-jwt-auth.tests = compile-only
 bilal-fazlani_circe-json-schema.tests = compile-only // Missing test files at runtime
@@ -191,6 +192,7 @@ indoorvivants_subatomic{
       "disableFatalWarnings"
     ]
 }
+indoorvivants_weaver-playwright.tests = compile-only // uses scripts
 irevive_union-derivation.tests = compile-only // fails due to changes in report messages
 itv_quartz4s.tests = compile-only
 
@@ -243,8 +245,11 @@ naoh87_lettucef.tests = compile-only
 noelwelsh_mads.sbt.commands = ["disableFatalWarnings"]
 ohze_couchbase-scala.tests = compile-only
 ohze_php-utils.tests = compile-only
+#outr_profig.tests = compile-only
 outr_scarango.tests = compile-only
 
+palanga_zio-cassandra.tests = compile-only
+palanga_parana.tests = compile-only // usses cassandra instance
 pdal_java.tests = disabled // Needs cmake in tests
 pityka_lamp{
   java.version=17
@@ -321,6 +326,7 @@ scalapb_scalapb{
   }]
 }
 scalapy_scalapy.tests = compile-only
+scalapy_python-native-libs.tests = compile-only
 scalaz_scalaz {
   sbt {
     options = ["-J-Xmx5g"]
@@ -336,6 +342,12 @@ sciss_soundprocesses{
 }
 scodec_scodec-cats.sbt.commands = ["set every Test/classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat"]
 scottweaver_testcontainers-for-zio.tests = compile-only
+scoverage_scalac-scoverage-plugin{
+  projects.overrides {
+    # Fails in runtime times due to changed error messages
+    scalac-scoverage-reporter.tests = compile-only
+  }
+}
 seratch_awscala.tests = compile-only
 simy4_coregex.java.version = 11 // scaldoc/javadoc fails in Java 8  due to incorrect flags
 sirthias_scala-ssh.tests = compile-only
@@ -427,8 +439,8 @@ typelevel_cats-effect {
   source-patches = [{
     // https://github.com/lampepfl/dotty/issues/15920
     path = "core/shared/src/main/scala/cats/effect/IO.scala"
-    pattern      = "Right(t.asLeft\\[IO\\[B\\]\\])"
-    replace-with = "t.asLeft.asRight"
+    pattern      = "Right(t.asLeft\\[IO\\[B\\]\\], limit - 1)"
+    replace-with = "(t.asLeft, limit - 1).asRight"
   }]
 }
 typelevel_cats-mtl {
@@ -463,6 +475,7 @@ typelevel_kittens {
   }]
 }
 
+ua-parser_uap-scala.tests = compile-only 
 unibas-gravis_scalismo.tests = compile-only // Uses native library
 unfiltered_unfiltered.tests = disabled // https://github.com/unfiltered/unfiltered/blob/9432d75e3a0f080fc926236ebc4e83518c884351/build.sbt#L8-L17
 
@@ -509,6 +522,8 @@ wartremover_wartremover {
   memory-request-mb = 7168
 }
 
+xuwei-k_httpz.tests = compile-only
+
 y-yu_slick-bulk-insert.tests = compile-only
 
 zengularity_benji.tests = compile-only

diff --git a/jenkins/seeds/buildCommunityProject.groovy b/jenkins/seeds/buildCommunityProject.groovy
@@ -57,7 +57,7 @@ pipeline {
                   shareProcessNamespace: true
                   containers:
                   - name: project-builder
-                    image: virtuslab/scala-community-build-project-builder:jdk${params.javaVersion?: 11}-v0.1.1
+                    image: virtuslab/scala-community-build-project-builder:jdk${params.javaVersion?: 11}-v0.1.2
                     imagePullPolicy: IfNotPresent
                     volumeMounts:
                     - name: mvn-repo-cert
@@ -164,7 +164,7 @@ pipeline {
               containers: [
                 containerTemplate(
                   name: 'reporter',
-                  image: 'virtuslab/scala-community-build-project-builder:jdk11-v0.1.1',
+                  image: 'virtuslab/scala-community-build-project-builder:jdk11-v0.1.2',
                   command: 'sleep',
                   args: '15m',
                   resourceRequestMemory: '250M',

diff --git a/jenkins/seeds/buildCompiler.groovy b/jenkins/seeds/buildCompiler.groovy
@@ -34,7 +34,7 @@ pipeline {
                               name: mvn-repo-cert
                           containers:
                           - name: compiler-builder
-                            image: virtuslab/scala-community-build-compiler-builder:v0.1.1
+                            image: virtuslab/scala-community-build-compiler-builder:v0.1.2
                             imagePullPolicy: IfNotPresent
                             volumeMounts:
                             - name: mvn-repo-cert

diff --git a/jenkins/seeds/computeBuildPlan.groovy b/jenkins/seeds/computeBuildPlan.groovy
@@ -25,7 +25,7 @@ pipeline {
                         spec:
                           containers:
                           - name: coordinator
-                            image: virtuslab/scala-community-build-coordinator:v0.1.1
+                            image: virtuslab/scala-community-build-coordinator:v0.1.2
                             imagePullPolicy: IfNotPresent
                             command:
                             - cat

diff --git a/jenkins/seeds/runBuild.groovy b/jenkins/seeds/runBuild.groovy
@@ -188,7 +188,7 @@ pipeline {
         podTemplate(
           containers: [
             // Any container having a curl or pre-installed scala-cli would work 
-            containerTemplate(name: 'reporter', image: 'virtuslab/scala-community-build-project-builder:jdk17-v0.1.1', command: 'sleep', args: '15m'),
+            containerTemplate(name: 'reporter', image: 'virtuslab/scala-community-build-project-builder:jdk17-v0.1.2', command: 'sleep', args: '15m'),
           ],
           envVars: [
             envVar(key: 'ELASTIC_USERNAME', value: params.elasticSearchUserName),

diff --git a/jenkins/seeds/runBuildWeekly.groovy b/jenkins/seeds/runBuildWeekly.groovy
@@ -22,7 +22,7 @@ pipeline {
           runBuildJobRef = build(
             job: runBuildJobName,
             parameters: [
-              string(name: "buildName", value: buildName)
+              string(name: "buildName", value: buildName),
               string(name: "publishedScalaVersion", value: compilerVersion),
               string(name: "minStarsCount", value: "-1"),
               string(name: "maxProjectsCount", value: "-1") // unlimited

diff --git a/k8s/mvn-repo.yaml b/k8s/mvn-repo.yaml
@@ -20,7 +20,7 @@ spec:
           secretName: mvn-repo-keystore
       containers:
       - name: mvn-repo
-        image: virtuslab/scala-community-build-mvn-repo:v0.1.1
+        image: virtuslab/scala-community-build-mvn-repo:v0.1.2
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 8081

diff --git a/project-builder/mill/scalafix/input/src/main/scala/fix/MillScalaVersionOverride.scala b/project-builder/mill/scalafix/input/src/main/scala/fix/MillScalaVersionOverride.scala
@@ -49,6 +49,9 @@ object MillScalaVersionOverride {
       def sv = scala213
       def scala213 = "2.13.8"
     }
+    object binVersions{
+      def scala3 = "3"
+    }
   }
 
   val snippet = s"""

diff --git a/project-builder/mill/scalafix/output/src/main/scala/fix/MillScalaVersionOverride.scala b/project-builder/mill/scalafix/output/src/main/scala/fix/MillScalaVersionOverride.scala
@@ -46,6 +46,9 @@ object MillScalaVersionOverride {
       def sv = scala213
       def scala213 = "2.13.8"
     }
+    object binVersions{
+      def scala3 = "3"
+    }
   }
 
   val snippet = s"""

diff --git a/project-builder/mill/scalafix/rules/src/main/scala/fix/Scala3CommunityBuildMillAdapter.scala b/project-builder/mill/scalafix/rules/src/main/scala/fix/Scala3CommunityBuildMillAdapter.scala
@@ -51,6 +51,8 @@ class Scala3CommunityBuildMillAdapter(
     "scala_3"
   )
 
+  val Scala3Literal = raw""""3.\d+.\d+(?:-RC\d+)?"""".r
+
   override def fix(implicit doc: SyntacticDocument): Patch = {
     val headerInject = {
       if (sys.props.contains("communitybuild.noInjects")) Patch.empty
@@ -95,7 +97,6 @@ class Scala3CommunityBuildMillAdapter(
         Patch.replaceTree(name, Replacment.CommunityBuildPublishModule)
 
       case ValOrDefDef(Term.Name("scalaVersion"), tpe, body) =>
-        val Scala3Literal = raw""""3.\d+.\d+(?:-RC\d+)?"""".r
         def replace(isLiteral: Boolean) =
           Patch.replaceTree(
             body,
@@ -108,10 +109,14 @@ class Scala3CommunityBuildMillAdapter(
         }
 
       case ValOrDefDef(Term.Name(id), tpe, body) if scala3Identifiers.contains(id) =>
-        Patch.replaceTree(
-          body,
-          Replacment.ScalaVersion(body.toString, asTarget = shouldWrapInTarget(body, tpe))
-        )
+        body.toString().trim() match {
+          case Scala3Literal() =>
+            Patch.replaceTree(
+              body,
+              Replacment.ScalaVersion(body.toString, asTarget = shouldWrapInTarget(body, tpe))
+            )
+          case _ => Patch.empty
+        }
 
       case ValOrDefDef(Term.Name("publishVersion"), tpe, body) =>
         Patch.replaceTree(

diff --git a/scripts/build-all.sh b/scripts/build-all.sh
@@ -7,7 +7,7 @@ if [ $# -ne 1 ]; then
 fi
 
 VERSION="$1"
-export PREV_CB_VERSION="v0.1.1"
+export PREV_CB_VERSION="v0.1.2"
 
 javaDefault=11
 javaAccessoryVersions=(8 17)