Migrate DSL -debbuilders to gazebo_libs (#1107)

* Migrate -debbuilders to DSL gazebo_libs
* Use single linux generation for -source and -debbuilders
* Change test and use always gz- prefix for -source and -debbuilders

---------

Signed-off-by: Jose Luis Rivero <[email protected]>

check_releasepy.bash

@@ -98,8 +98,8 @@ expect_job_not_run "${ignition_test}" "ignition-foo-source"
 expect_number_of_jobs "${ignition_test}" "1"
 
 ignition_source_tarball_uri_test=$(exec_ignition_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz")
-expect_job_run "${ignition_source_tarball_uri_test}" "ignition-foo-debbuilder"
+expect_job_run "${ignition_source_tarball_uri_test}" "gz-foo-debbuilder"
 expect_job_run "${ignition_source_tarball_uri_test}" "generic-release-homebrew_pull_request_updater"
-expect_job_not_run "${ignition_source_tarball_uri_test}" "ignition-foo-source"
+expect_job_not_run "${ignition_source_tarball_uri_test}" "gz-foo-source"
 expect_number_of_jobs "${ignition_source_tarball_uri_test}" "7"
 expect_param "${ignition_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fgz-foo-1.2.3.tar.gz"

jenkins-scripts/dsl/gazebo_libs.dsl

@@ -84,7 +84,7 @@ boolean are_cmake_warnings_enabled(lib_name, ci_config)
  *   index[gz-cmake][jammy] -> [ branch: gz-cmake3, collection: garden ,
  *                               branch: gz-cmake3, collection: harmonic]
  *
- * # pkgconf_per_src_inde index structure:
+ * # pkgconf_per_src index structure:
  *   pkg_src_name : [ packaging_config_name : [ .lib_name .collection ] ]
  *
  *   The index main keys are package source names (i.e gz-cmake3 or gz-harmonic and associated them
@@ -293,6 +293,33 @@ String generate_brew_install(src_name, lib_name, arch)
   return job_name
 }
 
+def generate_debbuilder_job(src_name, ArrayList pre_setup_script_hooks)
+{
+  def extra_cmd = pre_setup_script_hooks.join('\n')
+  assert extra_cmd instanceof String
+
+  def build_pkg_job = job("${src_name}-debbuilder")
+  OSRFLinuxBuildPkg.create(build_pkg_job)
+  build_pkg_job.with
+  {
+      concurrentBuild(true)
+
+      throttleConcurrentBuilds {
+        maxPerNode(1)
+        maxTotal(8)
+      }
+
+      steps {
+        shell("""\
+              #!/bin/bash -xe
+              ${GLOBAL_SHELL_CMD}
+              ${extra_cmd}
+              /bin/bash -x ./scripts/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash
+              """.stripIndent())
+      }
+  }
+}
+
 def ciconf_per_lib_index = [:].withDefault { [:] }
 def pkgconf_per_src_index = [:].withDefault { [:] }
 generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index)
@@ -448,23 +475,55 @@ ciconf_per_lib_index.each { lib_name, lib_configs ->
 } // end of lib
 
 pkgconf_per_src_index.each { pkg_src, pkg_src_configs ->
-  pkg_src_configs.each { pkg_src_config ->
-    def config_name = pkg_src_config.getKey()
-    def pkg_config = gz_collections_yaml.packaging_configs.find{ it.name == config_name }
-    // lib_names are the same in all the entries
-    def canonical_lib_name = pkg_src_config.getValue()[0].lib_name
-    if (pkg_config.exclude?.contains(canonical_lib_name))
-      return
-    def pkg_system = pkg_config.system
-    // --------------------------------------------------------------
+  // For each entry in the index perform two steps:
+  //  1. Generate Linux builders artifacts (-source and -debbuilders)
+  //  2. Generate all -ci-install- jobs looping in the index entries
+
+  // 1. Generate Linux builders
+  // All entries are the same for canonical_lib_name, pick the first
+  def canonical_lib_name = pkg_src_configs.values()[0].lib_name[0]
+  def linux_ciconfigs = gz_collections_yaml.packaging_configs.findAll{
+    it.name in pkg_src_configs.keySet() &&
+    it.system.so == 'linux'}
+  // Collect the pre_setup_script_hooks defined in all the linux distributions
+  // Each distribution commands are joined in a signle item.
+  // Unique + findall { it } should clean up all null values
+  def pre_setup_script_hooks = linux_ciconfigs.findAll {
+    it.keySet().contains('pre_setup_script_hook')
+  }.collect {
+    it -> it.pre_setup_script_hook.get(canonical_lib_name)
+  }.flatten().unique().findAll { it }
+
+  def exclusion_list = linux_ciconfigs.findAll {
+    it.keySet().contains('exclude')
+  }.collect{
+    it -> it.exclude
+  }
+
+  if (linux_ciconfigs && !exclusion_list.contains(canonical_lib_name))
+  {
+    // - DEBBUILD jobs -------------------------------------------------
+    generate_debbuilder_job(pkg_src,
+      pre_setup_script_hooks
+    )
+    // - SOURCE jobs ---------------------------------------------------
     def gz_source_job = job("${pkg_src}-source")
     OSRFSourceCreation.create(gz_source_job, [
       PACKAGE: pkg_src,
       SOURCE_REPO_URI: "https://github.com/gazebosim/${canonical_lib_name}.git"])
     OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job,
       'repository_uploader_packages',
       '_releasepy')
-    // --------------------------------------------------------------
+  }
+
+  // 2. Generate all -ci-install jobs
+  pkg_src_configs.each { pkg_src_config ->
+    def config_name = pkg_src_config.getKey()
+    def pkg_config = gz_collections_yaml.packaging_configs.find{ it.name == config_name }
+    if (pkg_config.exclude?.contains(canonical_lib_name))
+      return
+    def pkg_system = pkg_config.system
+      // - CI-INSTALL jobs ------------------------------------------------
     pkg_system.arch.each { arch ->
       def linux_install_job_name = generate_linux_install(
         pkg_src,

jenkins-scripts/dsl/gz-collections.yaml

@@ -588,12 +588,18 @@ packaging_configs:
       version: focal
       arch:
         - amd64
+    pre_setup_script_hook:
+      gz-physics:
+        - "export MAKE_JOBS=1"
   - name: jammy
     system:
       so: linux
       distribution: ubuntu
       version: jammy
       arch:
         - amd64
+    pre_setup_script_hook:
+      gz-physics:
+        - "export MAKE_JOBS=1"
     exclude:
         - __upcoming__

jenkins-scripts/dsl/ignition.dsl

@@ -391,45 +391,6 @@ gz_software.each { gz_sw ->
   } // end of arch
 } // end of gz_software
 
-// --------------------------------------------------------------
-// DEBBUILD: linux package builder
-all_debbuilders().each { debbuilder_name ->
-  extra_str = ""
-  if (debbuilder_name.contains("gazebo") || debbuilder_name == "transport7")
-    extra_str="export NEED_C17_COMPILER=true"
-
-  // Gazebo physics consumes huge amount of memory making arm node to FAIL
-  // Force here to use one compilation thread
-  if (debbuilder_name.contains("-physics"))
-    extra_str += '\nexport MAKE_JOBS=1'
-
-  def build_pkg_job = job("${debbuilder_name}-debbuilder")
-  OSRFLinuxBuildPkg.create(build_pkg_job)
-  build_pkg_job.with
-  {
-
-      concurrentBuild(true)
-
-      throttleConcurrentBuilds {
-        maxPerNode(1)
-        maxTotal(8)
-      }
-
-      steps {
-        shell("""\
-              #!/bin/bash -xe
-
-              ${GLOBAL_SHELL_CMD}
-              ${extra_str}
-              /bin/bash -x ./scripts/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash
-              """.stripIndent())
-      }
-  }
-}
-
-// --------------------------------------------------------------
-// WINDOWS: CI job
-
 // Main CI workflow
 gz_software.each { gz_sw ->
   if (gz_sw == 'sim')

release.py

@@ -608,7 +608,7 @@ def go(argv):
                     # "-()" do not work even in the web UI directly. Real
                     # string should be:
                     # f"{args.version}-{args.release_version}({l}/{d}::{a})")
-                    call_jenkins_build(f"{args.package_alias}-debbuilder",
+                    call_jenkins_build(f'{package_alias_force_gz}-debbuilder',
                                        linux_platform_params,
                                        f"{l} {d}/{a}",
                                        f"{args.version}-{args.release_version}")