Adds packaging images release name support into citus package

.github/workflows/citus-package-all-platforms-test.yml

@@ -23,16 +23,14 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-          [
-            el/7,
-            el/8,
-            ol/7,
-            debian/stretch,
-            debian/buster,
-            ubuntu/xenial,
-            ubuntu/bionic,
-            ubuntu/focal,
-          ]
+          - el/7
+          - el/8
+          - ol/7
+          - debian/stretch
+          - debian/buster
+          - debian/bullseye
+          - ubuntu/bionic
+          - ubuntu/focal
     env:
       PLATFORM: ${{ matrix.platform }}
 

packaging_automation/citus_package.py

@@ -19,7 +19,7 @@
 GPG_KEY_NAME = "[email protected]"
 
 supported_platforms = {
-    "debian": ["buster", "stretch", "jessie", "wheezy","bullseye"],
+    "debian": ["bullseye", "buster", "stretch", "jessie", "wheezy"],
     "el": ["8", "7", "6"],
     "ol": ["7", "8"],
     "ubuntu": ["focal", "bionic", "xenial", "trusty"]
@@ -43,6 +43,21 @@ def platform_names() -> List[str]:
     "pgxn": "pgxn"
 }
 
+package_docker_platform_dict = {
+    "centos,8": "el/8",
+    "centos,7": "el/7",
+    "debian,bullseye": "debian/bullseye",
+    "debian,buster": "debian/buster",
+    "debian,stretch": "debian/stretch",
+    "oraclelinux,8": "ol/8",
+    "oraclelinux,7": "ol/7",
+    "oraclelinux,6": "ol/6",
+    "ubuntu,focal": "ubuntu/focal",
+    "ubuntu,bionic": "ubuntu/bionic",
+    "ubuntu,xenial": "ubuntu/xenial",
+    "pgxn": "pgxn"
+}
+
 
 def get_package_type_by_docker_image_name(docker_image_name: str) -> PackageType:
     return PackageType.deb if docker_image_name.startswith(("ubuntu", "debian")) else PackageType.rpm
@@ -243,7 +258,7 @@ def get_release_package_folder_name(os_name: str, os_version: str) -> str:
 
 def get_docker_image_name(platform: str):
     os_name, os_version = decode_os_and_release(platform)
-    return f'{docker_image_names[os_name]}-{os_version}'
+    return f'{docker_image_names[os_name]}-{os_version}' if os_version else f'{docker_image_names[os_name]}'
 
 
 @validate_parameters
@@ -274,10 +289,16 @@ def build_packages(github_token: non_empty(non_blank(str)),
     sign_packages(output_sub_folder, signing_credentials, input_output_parameters)
 
 
+def get_build_platform(packaging_platform: str, packaging_docker_platform: str) -> str:
+    return (
+        package_docker_platform_dict[packaging_docker_platform] if packaging_docker_platform else packaging_platform)
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument('--gh_token', required=True)
-    parser.add_argument('--platform', required=True, choices=platform_names())
+    parser.add_argument('--platform', required=False, choices=platform_names())
+    parser.add_argument('--packaging_docker_platform', required=False, choices=package_docker_platform_dict.keys())
     parser.add_argument('--build_type', choices=[b.name for b in BuildType])
     parser.add_argument('--secret_key', required=True)
     parser.add_argument('--passphrase', required=True)
@@ -287,7 +308,11 @@ def build_packages(github_token: non_empty(non_blank(str)),
 
     args = parser.parse_args()
 
+    if args.platform and args.packaging_docker_platform:
+        raise ValueError("Either platform or packaging_docker_platform should be set.")
+    build_platform = get_build_platform(args.platform, args.packaging_docker_platform)
+
     io_parameters = InputOutputParameters.build(args.input_files_dir, args.output_dir, args.output_validation)
     sign_credentials = SigningCredentials(args.secret_key, args.passphrase)
-    build_packages(args.gh_token, args.platform, BuildType[args.build_type], sign_credentials,
+    build_packages(args.gh_token, build_platform, BuildType[args.build_type], sign_credentials,
                    io_parameters)

packaging_automation/tests/test_citus_package.py

@@ -4,7 +4,7 @@
 
 from .test_utils import generate_new_gpg_key
 from ..citus_package import (build_packages, BuildType, decode_os_and_release, get_release_package_folder_name,
-                             SigningCredentials, InputOutputParameters)
+                             SigningCredentials, InputOutputParameters, get_build_platform)
 from ..common_tool_methods import (run, delete_rpm_key_by_name, get_gpg_fingerprints_by_name,
                                    get_private_key_by_fingerprint_with_passphrase, define_rpm_public_key_to_machine,
                                    transform_key_into_base64_str,
@@ -22,28 +22,32 @@
     "el/7": 4,
     "el/8": 6,
     "ol/7": 4,
+    "ol/8": 6,
     "debian/stretch": 4,
     "debian/buster": 4,
     "debian/bullseye": 4,
     "ubuntu/xenial": 2,
     "ubuntu/bionic": 4,
-    "ubuntu/focal": 4
+    "ubuntu/focal": 4,
+    "pgxn": 1
 }
 
 TEST_GPG_KEY_NAME = "Citus Data <[email protected]>"
 TEST_GPG_KEY_PASSPHRASE = os.getenv("PACKAGING_PASSPHRASE")
 GH_TOKEN = os.getenv("GH_TOKEN")
-PLATFORM = os.getenv("PLATFORM")
 PACKAGE_CLOUD_API_TOKEN = os.getenv("PACKAGE_CLOUD_API_TOKEN")
 REPO_CLIENT_SECRET = os.getenv("REPO_CLIENT_SECRET")
+PLATFORM = get_build_platform(os.getenv("PLATFORM"), os.getenv("PACKAGING_IMAGE_PLATFORM"))
 
 
 def setup_module():
     # Run tests against "all-citus-unit-tests" since we don't want to deal with the changes
     # made to "all-citus" in each release.
-    if not os.path.exists(PACKAGING_SOURCE_FOLDER):
+    packaging_branch_name = "pgxn-citus" if PLATFORM == "pgxn" else "all-citus-unit-tests"
+    if not os.path.exists(PACKAGING_EXEC_FOLDER):
         run(
-            f"git clone --branch all-citus-unit-tests https://github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}")
+            f"git clone --branch {packaging_branch_name} https://github.com/citusdata/packaging.git"
+            f" {PACKAGING_EXEC_FOLDER}")
 
 
 def teardown_module():
@@ -52,8 +56,6 @@ def teardown_module():
 
 
 def test_build_packages():
-    platform = os.getenv("PLATFORM")
-
     delete_all_gpg_keys_by_name(TEST_GPG_KEY_NAME)
     delete_rpm_key_by_name(TEST_GPG_KEY_NAME)
     generate_new_gpg_key(f"{TEST_BASE_PATH}/packaging_automation/tests/files/gpg/packaging_with_passphrase.gpg")
@@ -65,28 +67,34 @@ def test_build_packages():
     signing_credentials = SigningCredentials(secret_key, TEST_GPG_KEY_PASSPHRASE)
     input_output_parameters = InputOutputParameters.build(PACKAGING_EXEC_FOLDER, BASE_OUTPUT_FOLDER,
                                                           output_validation=False)
-    build_packages(GH_TOKEN, platform, BuildType.release, signing_credentials, input_output_parameters)
+    build_packages(GH_TOKEN, PLATFORM, BuildType.release, signing_credentials, input_output_parameters)
     verify_rpm_signature_in_dir(BASE_OUTPUT_FOLDER)
-    os_name, os_version = decode_os_and_release(platform)
+    os_name, os_version = decode_os_and_release(PLATFORM)
     sub_folder = get_release_package_folder_name(os_name, os_version)
     release_output_folder = f"{BASE_OUTPUT_FOLDER}/{sub_folder}"
     delete_all_gpg_keys_by_name(TEST_GPG_KEY_NAME)
-    assert len(os.listdir(release_output_folder)) == package_counts[platform]
+    assert len(os.listdir(release_output_folder)) == package_counts[PLATFORM]
+
+
+def test_decode_os_packages():
+    os, release = decode_os_and_release("el/7")
+    assert os == "el" and release == "7"
 
 
 def test_upload_to_package_cloud():
+    platform = get_build_platform(os.getenv("PLATFORM"), os.getenv("PACKAGING_IMAGE_PLATFORM"))
     current_branch = "all-citus"
     main_branch = "all-citus"
-    output = upload_files_in_directory_to_package_cloud(BASE_OUTPUT_FOLDER, PLATFORM, PACKAGE_CLOUD_API_TOKEN,
+    output = upload_files_in_directory_to_package_cloud(BASE_OUTPUT_FOLDER, platform, PACKAGE_CLOUD_API_TOKEN,
                                                         "citus-bot/sample",
                                                         current_branch, main_branch)
-    distro_parts = PLATFORM.split("/")
+    distro_parts = platform.split("/")
     if len(distro_parts) != 2:
         raise ValueError("Platform should consist of two parts splitted with '/' e.g. el/8")
     for return_value in output.return_values:
         exists = package_exists(PACKAGE_CLOUD_API_TOKEN, "citus-bot", "sample",
                                 os.path.basename(return_value.file_name),
-                                PLATFORM)
+                                platform)
         if not exists:
             raise ValueError(f"{os.path.basename(return_value.file_name)} could not be found on package cloud")