Merge pull request #2 from olliesilvester/merge_in_hyperion

Merge in hyperion

.coveragerc

@@ -0,0 +1,18 @@
+[run]
+omit = 
+    */test_*
+    **/conftest.py
+data_file = /tmp/hyperion.coverage
+
+[report]
+exclude_also = 
+    if TYPE_CHECKING:
+    def __repr__
+    raise NotImplementedError
+    @(abc\.)?abstractmethod
+
+[paths]
+# Tests are run from installed location, map back to the src directory
+source =
+    src
+    **/site-packages/

.devcontainer/Dockerfile

@@ -3,7 +3,7 @@
 # The devcontainer should use the build target and run as root with podman
 # or docker with user namespaces.
 #
-FROM python:3.11 as build
+FROM python:3.11 AS build
 
 ARG PIP_OPTIONS
 
@@ -24,7 +24,7 @@ WORKDIR /context
 # install python package into /venv
 RUN pip install ${PIP_OPTIONS}
 
-FROM python:3.11-slim as runtime
+FROM python:3.11-slim AS runtime
 
 # Add apt-get system dependecies for runtime here if needed
 

.github/CONTRIBUTING.md

@@ -1,10 +1,40 @@
-# Contribute to the project
+# How to contribute to mx-bluesky
 
-Contributions and issues are most welcome! All issues and pull requests are
-handled through [GitHub](https://github.com/DiamondLightSource/mx-bluesky/issues). Also, please check for any existing issues before
-filing a new one. If you have a great idea but it involves big changes, please
-file a ticket before making a pull request! We want to make sure you don't spend
-your time coding something that might not fit the scope of the project.
+We should prioritise working, agile code over strict standards, particularly for contributors that are not full time developers. The standards below are thus guidelines that developers should try to follow as much as possible (apart from when something is specified in bold, which is required).
+
+Contributions and issues are most welcome! All issues and pull requests are handled through [GitHub](https://github.com/DiamondLightSource/mx-bluesky/issues). Also, please check for any existing issues before filing a new one. If you have a great idea but it involves big changes, please file a ticket before making a pull request! We want to make sure you don't spend your time coding something that might not fit the scope of the project.
+
+## General Workflow
+
+1. An issue is created for the work. This issue should describe in as much detail as possible the work that needs to be done. Anyone is free to make a ticket and/or comment on one.
+2. If a developer is going to do the work they assign themselves to the issue.
+3. The developer makes a new branch with the format `issue_short_description` e.g. `122_create_a_contributing_file`. (External developers are also welcome to make forks)
+4. The developer does the work on this branch, adding their work in small commits. Commit messages should be informative and prefixed with the issue number e.g. `(#122) Added contributing file`.
+5. The developer submits a PR for the work. In the pull request should start with `Fixes #issue_num` e.g. `Fixes #122`, this will ensure the issue is automatically closed when the PR is merged. The developer should also add some background on how the reviewer might test the change.
+6. If the developer has a particular person in mind to review the work they should assign that person to the PR as a reviewer.
+7. The reviewer and developer go back and forth on the code until the reviewer approves it. (See "Reviewing Work" below)
+8. Once the work is approved the original developer merges it.
+
+**Work should not be done without an associated ticket describing what the work is**
+
+## Reviewing Work
+
+**Work must be reviewed by another developer before it can be merged**. Remember that work is reviewed for a number of reasons:
+
+- In order to maintain quality and avoid errors
+- Help people learn
+
+It is not a judgement on someone's technical abilities so be kind.
+
+It is suggested that the reviewer prefixes comments with something like the following:
+
+- **must**: A change that must be made before merging e.g. it will break something if not made
+- **should/sugg**: A change that should be made e.g. definitely improves code quality but does not neccessarily break anything
+- **nit**: A minor suggestion that the reviewer would like to see but is happy to leave as is e.g. rename a variable to something
+
+Developers are welcome to ignore **nit** comments if they wish and can choose not to do **should** comments but the must give a reason as to why they disagree with the change.
+
+For minor changes to code reviewers are welcome to make the changes themselves but in this case the original developer should then recheck what the reviewer has done.
 
 ## Issue or Discussion?
 

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -1,8 +1,13 @@
 Fixes #ISSUE
 
+Link to dodal PR (if required): #XXX
+(remember to update `setup.cfg` with the dodal commit tag if you need it for tests to pass!)
+
 ### Instructions to reviewer on how to test:
+
 1. Do thing x
 2. Confirm thing y happens
 
 ### Checks for reviewer
+
 - [ ] Would the PR title make sense to a user on a set of release notes

.github/actions/verify-nexus/Dockerfile

@@ -0,0 +1,3 @@
+FROM ghcr.io/githubgphl/imginfo:main
+COPY entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/verify-nexus/action.yml

@@ -0,0 +1,18 @@
+name: "Verify nexus"
+description: "Verify nexus files against imginfo"
+inputs:
+  filename:
+    description: "nexus file to verify"
+    required: true
+outputs:
+  imginfo_stdout:
+    description: "imginfo output"
+  imginfo_stderr:
+    description: "imginfo error output"
+  imginfo_exit_code:
+    description: "imginfo exit code"
+runs:
+  using: "docker"
+  image: "Dockerfile"
+  args:
+    - ${{ inputs.filename }}

.github/actions/verify-nexus/entrypoint.sh

@@ -0,0 +1,25 @@
+#!/bin/sh -l
+echo "Contents of /github/workspace:"
+ls /github/workspace
+echo "current dir: $(pwd)"
+echo "Running imginfo on $1"
+/imginfo/imginfo  $1 >> imginfo_out_file 2>> imginfo_err_file
+{ echo "imginfo_output<<EOF"
+  cat imginfo_out_file
+  echo EOF
+} >> "$GITHUB_OUTPUT"
+{ echo "imginfo_output<<EOF"
+  cat imginfo_err_file
+  echo EOF
+} >> "$GITHUB_OUTPUT"
+echo "imginfo_exit_code=$?" >> "$GITHUB_OUTPUT"
+echo "------------- IMGINFO STDOUT -------------"
+cat imginfo_out_file
+echo "------------- IMGINFO STDERR -------------"
+cat imginfo_err_file
+echo "------------------------------------------"
+if [ -s imginfo_err_file ]; then
+  echo "ERRORS IN IMGINFO PROCESSING!"
+  exit 1
+fi
+echo "VALIDATED SUCCESSFULLY!"

.github/workflows/_test.yml

@@ -51,7 +51,7 @@ jobs:
           pip-install: ".[dev]"
 
       - name: Run tests
-        run: tox -e tests
+        run: tox -e tests -- -m "not dlstbx"
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       matrix:
         runs-on: ["ubuntu-latest"] # can add windows-latest, macos-latest
-        python-version: ["3.10", "3.11"] # , "3.12"] # add 3.12 when p4p #145 is fixed
+        python-version: ["3.11"] # , "3.12"] # add 3.12 when p4p #145 is fixed
         include:
           # Include one that runs in the dev environment
           - runs-on: "ubuntu-latest"
@@ -52,12 +52,12 @@ jobs:
     if: needs.check.outputs.branch-pr == ''
     uses: ./.github/workflows/_dist.yml
 
-  pypi:
-    if: github.ref_type == 'tag'
-    needs: dist
-    uses: ./.github/workflows/_pypi.yml
-    permissions:
-      id-token: write
+  # pypi:
+  #   if: github.ref_type == 'tag'
+  #   needs: dist
+  #   uses: ./.github/workflows/_pypi.yml
+  #   permissions:
+  #     id-token: write
 
   release:
     if: github.ref_type == 'tag'

.github/workflows/pin_versions.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+import argparse
+import locale
+import os
+import re
+import subprocess
+from functools import partial
+from sys import stderr, stdout
+
+SETUP_CFG_PATTERN = re.compile("(.*?\\S)\\s*(@(.*))?$")
+SETUP_UNPINNED_PATTERN = re.compile("(.*?\\S)\\s*([<>=]+(.*))?$")
+PIP = "pip"
+
+
+def rename_original(suffix):
+    os.rename("setup.cfg", "setup.cfg" + suffix)
+
+
+def normalize(package_name: str):
+    return re.sub(r"[-_.]+", "-", package_name).lower()
+
+
+def fetch_pin_versions() -> dict[str, str]:
+    process = run_pip_freeze()
+    if process.returncode == 0:
+        output = process.stdout
+        lines = output.split("\n")
+        pin_versions = {}
+        for line in lines:
+            kvpair = line.split("==")
+            if len(kvpair) != 2:
+                stderr.write(f"Unable to parse {line} - ignored\n")
+            else:
+                pin_versions[normalize(kvpair[0]).strip()] = kvpair[1].strip()
+        return pin_versions
+    else:
+        stderr.write(f"pip freeze failed with error code {process.returncode}\n")
+        stderr.write(process.stderr)
+        exit(1)
+
+
+def run_pip_freeze():
+    process = subprocess.run(
+        [PIP, "freeze"], capture_output=True, encoding=locale.getpreferredencoding()
+    )
+    return process
+
+
+def process_setup_cfg(input_fname, output_fname, dependency_processor):
+    with open(input_fname) as input_file:
+        with open(output_fname, "w") as output_file:
+            process_files(input_file, output_file, dependency_processor)
+
+
+def process_files(input_file, output_file, dependency_processor):
+    while line := input_file.readline():
+        output_file.write(line)
+        if line.startswith("install_requires"):
+            break
+    while (line := input_file.readline()) and not line.startswith("["):
+        if line.isspace():
+            output_file.write(line)
+        else:
+            dependency_processor(line, output_file)
+    output_file.write(line)
+    while line := input_file.readline():
+        output_file.write(line)
+
+
+def strip_comment(line: str):
+    split = line.rstrip("\n").split("#", 1)
+    return split[0], (split[1] if len(split) > 1 else None)
+
+
+def write_with_comment(comment, text, output_file):
+    output_file.write(text)
+    if comment:
+        output_file.write(" #" + comment)
+    output_file.write("\n")
+
+
+def update_setup_cfg_line(version_map: dict[str, str], line, output_file):
+    stripped_line, comment = strip_comment(line)
+    if match := SETUP_UNPINNED_PATTERN.match(stripped_line):
+        normalized_name = normalize(match[1].strip())
+        if normalized_name not in version_map:
+            stderr.write(
+                f"Unable to find {normalized_name} in installed python packages\n"
+            )
+            exit(1)
+
+        write_with_comment(
+            comment,
+            f"    {normalized_name} == {version_map[normalized_name]}",
+            output_file,
+        )
+    else:
+        output_file.write(line)
+
+
+def write_commit_message(pinned_versions: dict[str, str]):
+    message = f"Pin dependencies prior to release. Dodal {pinned_versions['dls-dodal']}, nexgen {pinned_versions['nexgen']}"
+    stdout.write(message)
+
+
+def unpin_versions(line, output_file):
+    stripped_line, comment = strip_comment(line)
+    if match := SETUP_CFG_PATTERN.match(stripped_line):
+        if match[3] and match[3].strip().startswith("git+"):
+            write_with_comment(comment, match[1], output_file)
+            return
+
+    output_file.write(line)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Pin dependency versions in setup.cfg")
+    parser.add_argument(
+        "--unpin",
+        help="remove pinned hashes from setup.cfg prior to pip installing latest",
+        action="store_true",
+    )
+    args = parser.parse_args()
+
+    if args.unpin:
+        rename_original(".orig")
+        process_setup_cfg("setup.cfg.orig", "setup.cfg", unpin_versions)
+    else:
+        rename_original(".unpinned")
+        installed_versions = fetch_pin_versions()
+        process_setup_cfg(
+            "setup.cfg.unpinned",
+            "setup.cfg",
+            partial(update_setup_cfg_line, installed_versions),
+        )
+        write_commit_message(installed_versions)

.github/workflows/pre_release_workflow.yml

@@ -0,0 +1,48 @@
+name: pre_release_workflow
+on:
+  workflow_dispatch:
+    inputs:
+      tagName:
+        description: 'Tag to create'
+        required: true
+        type: string
+        default: 'vX.Y.Z'
+jobs:
+  pin_dependency_versions:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{secrets.GITHUB_TOKEN}}
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          architecture: x64
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Reset pinned hash versions
+        run: .github/workflows/pin_versions.py --unpin
+      - name: Install with latest dependencies
+        run: pip install -e .[dev]
+      - id: pin_versions
+        name: Pin Versions
+        run: |
+          MSG=$(.github/workflows/pin_versions.py)
+          echo "COMMIT_MESSAGE=\"$MSG\"" >> "$GITHUB_OUTPUT"
+      - name: Add setup.cfg
+        run: |
+          git add setup.cfg
+      - name: Commit changes
+        run: |
+          git config --global user.name "${{ github.actor }}"
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git commit -m ${{steps.pin_versions.outputs.COMMIT_MESSAGE}}
+          git tag ${{inputs.tagName}}
+          git push origin ${{inputs.tagName}}
+      - name: Create Release
+        run: gh release create --generate-notes --draft ${{inputs.tagName}}
+      - name: Edit Release Notes
+        run: |
+          echo -e "${{ steps.pin_versions.outputs.COMMIT_MESSAGE }}\n\n" > "$RUNNER_TEMP/relnotes.txt" 
+          gh release view ${{ inputs.tagName }} | sed '0,/^--$/d' >> "$RUNNER_TEMP/relnotes.txt"
+          gh release edit ${{ inputs.tagName }} --notes-file "$RUNNER_TEMP/relnotes.txt"