From d53b49f36f7cebb145fc6d03db36c5ef3ed8a611 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Mon, 22 Jan 2024 15:50:29 +0000 Subject: [PATCH] (DiamondLightSource/hyperion#819) Initial attempt at github workflow to pin versions --- .github/workflows/pin_versions.py | 126 +++++++++++++++++++++ .github/workflows/pre_release_workflow.yml | 38 +++++++ 2 files changed, 164 insertions(+) create mode 100755 .github/workflows/pin_versions.py create mode 100644 .github/workflows/pre_release_workflow.yml diff --git a/.github/workflows/pin_versions.py b/.github/workflows/pin_versions.py new file mode 100755 index 000000000..ffe2e84af --- /dev/null +++ b/.github/workflows/pin_versions.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +import argparse +import locale +import os +import re +import subprocess +from functools import partial +from sys import stderr, stdout + +SETUP_CFG_PATTERN = re.compile("(.*?)(@(.*))?\n") +PIP = "pip" + + +def rename_original(suffix): + os.rename("setup.cfg", "setup.cfg" + suffix) + + +def normalize(package_name: str): + return re.sub(r"[-_.]+", "-", package_name).lower() + + +def fetch_pin_versions() -> dict[str, str]: + process = subprocess.run( + [PIP, "freeze"], capture_output=True, encoding=locale.getpreferredencoding() + ) + if process.returncode == 0: + output = process.stdout + lines = output.split("\n") + pin_versions = {} + for line in lines: + kvpair = line.split("==") + if len(kvpair) != 2: + stderr.write(f"Unable to parse {line} - ignored\n") + else: + pin_versions[normalize(kvpair[0]).strip()] = kvpair[1].strip() + return pin_versions + else: + stderr.write(f"pip freeze failed with error code {process.returncode}\n") + stderr.write(process.stderr) + exit(1) + + +def process_setup_cfg(input_fname, output_fname, dependency_processor): + with open(input_fname) as input_file: + with open(output_fname, "w") as output_file: + while line := input_file.readline(): + output_file.write(line) + if line.startswith("install_requires"): + break + + while (line := input_file.readline()) and not line.isspace(): + dependency_processor(line, output_file) + + output_file.write(line) + + while line := input_file.readline(): + output_file.write(line) + + +def strip_comment(line: str): + split = line.split("#", 1) + return split[0], (split[1] if len(split) > 1 else None) + + +def write_with_comment(comment, text, output_file): + output_file.write(text) + if comment: + output_file.write("#" + comment) + output_file.write("\n") + + +def update_setup_cfg_line(version_map: dict[str, str], line, output_file): + stripped_line, comment = strip_comment(line) + if match := SETUP_CFG_PATTERN.match(stripped_line): + normalized_name = normalize(match[1].strip()) + if normalized_name not in version_map: + stderr.write( + f"Unable to find {normalized_name} in installed python packages\n" + ) + exit(1) + + write_with_comment( + comment, + f" {normalized_name} @ {version_map[normalized_name]}", + output_file, + ) + else: + output_file.write(line) + + +def write_commit_message(pinned_versions: dict[str, str]): + message = f"Pin dependencies prior to release. Dodal {pinned_versions['dls-dodal']}, nexgen {pinned_versions['nexgen']}" + stdout.write(message) + + +def unpin_versions(line, output_file): + stripped_line, comment = strip_comment(line) + if match := SETUP_CFG_PATTERN.match(stripped_line): + if match[3] and match[3].strip().startswith("git+"): + write_with_comment(comment, match[1], output_file) + return + + output_file.write(line) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Pin dependency versions in setup.cfg") + parser.add_argument( + "--unpin", + help="remove pinned hashes from setup.cfg prior to pip installing latest", + action="store_true", + ) + args = parser.parse_args() + + if args.unpin: + rename_original(".orig") + process_setup_cfg("setup.cfg.orig", "setup.cfg", unpin_versions) + else: + rename_original(".unpinned") + installed_versions = fetch_pin_versions() + process_setup_cfg( + "setup.cfg.unpinned", + "setup.cfg", + partial(update_setup_cfg_line, installed_versions), + ) + write_commit_message(installed_versions) diff --git a/.github/workflows/pre_release_workflow.yml b/.github/workflows/pre_release_workflow.yml new file mode 100644 index 000000000..b47331608 --- /dev/null +++ b/.github/workflows/pre_release_workflow.yml @@ -0,0 +1,38 @@ +name: pre_release_workflow +on: + workflow_dispatch: + inputs: + tagName: + description: 'Tag to create' + required: true + type: string + default: 'vX.Y.Z' +jobs: + pin_dependency_versions: + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.9" + architecture: x64 + - name: checkout + uses: actions/checkout@v4 + - name: Reset pinned hash versions + run: .github/workflows/pin_versions.py --unpin + - name: Install with latest dependencies + run: pip install -e .[dev] + - id: pin_versions + name: Pin Versions + run: | + MSG=$(.github/workflows/pin_versions.py) + echo "COMMIT_MESSAGE=\"$MSG\"" >> "$GITHUB_OUTPUT" + - name: Add setup.cfg + run: git add setup.cfg + - name: Commit changes + run: | + git config --global user.name "GitHub Workflow" + git config --global user.email "robert.tuck@diamond.ac.uk" + git commit -m ${{steps.pin_versions.outputs.COMMIT_MESSAGE}} + git tag ${{inputs.tagName}} + git push origin ${{inputs.tagName}}