DiamondLightSource · joeshannon · Mar 6, 2024 · Mar 6, 2024 · Mar 6, 2024 · Mar 6, 2024
diff --git a/.copier-answers.yml b/.copier-answers.yml
@@ -0,0 +1,15 @@
+# Changes here will be overwritten by Copier
+_commit: 1.0.0
+_src_path: gh:DiamondLightSource/python-copier-template
+author_email: [email protected]
+author_name: Callum Forrester
+component_owner: user:vid18871
+description: Lightweight bluesky-as-a-service wrapper application. Also usable as
+    a library.
+distribution_name: blueapi
+docker: true
+docs_type: sphinx
+git_platform: github.com
+github_org: DiamondLightSource
+package_name: blueapi
+repo_name: blueapi
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -2,7 +2,7 @@
 {
     "name": "Python 3 Developer Container",
     "build": {
-        "dockerfile": "Dockerfile",
+        "dockerfile": "../Dockerfile",
         "target": "build",
         // Only upgrade pip, we will install the project below
         "args": {
@@ -50,5 +50,5 @@
     "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind",
     "workspaceFolder": "${localWorkspaceFolder}",
     // After the container is created, install the python project in editable form
-    "postCreateCommand": "pip install -e .[dev]"
-}
+    "postCreateCommand": "pip install -e '.[dev]'"
+}
diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst
@@ -32,4 +32,4 @@ The `Developer Guide`_ contains information on setting up a development
 environment, running the tests and what standards the code and documentation
 should follow.
 
-.. _Developer Guide: https://diamondlightsource.github.io/blueapi/main/developer/how-to/contribute.html
+.. _Developer Guide: https://DiamondLightSource.github.io/blueapi/main/developer/how-to/contribute.html
diff --git a/.github/actions/install_requirements/action.yml b/.github/actions/install_requirements/action.yml
@@ -7,16 +7,19 @@ inputs:
   install_options:
     description: Parameters to pass to pip install
     required: true
+  artifact_name:
+    description: A user friendly name to give the produced artifacts
+    required: true
   python_version:
     description: Python version to install
-    default: "3.9"
+    default: "3.x"
 
 runs:
   using: composite
 
   steps:
     - name: Setup python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python_version }}
 
@@ -30,15 +33,15 @@ runs:
     - name: Create lockfile
       run: |
         mkdir -p lockfiles
-        pip freeze --exclude-editable --exclude ophyd-async --exclude dls-dodal --exclude dls-bluesky-core --exclude bluesky > lockfiles/${{ inputs.requirements_file }}
+        pip freeze --exclude-editable > lockfiles/${{ inputs.requirements_file }}
         # delete the self referencing line and make sure it isn't blank
-        sed -i '/file:/d' lockfiles/${{ inputs.requirements_file }}
+        sed -i'' -e '/file:/d' lockfiles/${{ inputs.requirements_file }}
       shell: bash
 
     - name: Upload lockfiles
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4.0.0
       with:
-        name: lockfiles
+        name: lockfiles-${{ inputs.python_version }}-${{ inputs.artifact_name }}-${{ github.sha }}
         path: lockfiles
 
     # This eliminates the class of problems where the requirements being given no

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -9,6 +9,10 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    groups:
+      github-artifacts:
+        patterns:
+          - actions/*-artifact
 
   - package-ecosystem: "pip"
     directory: "/"

diff --git a/.github/pages/index.html b/.github/pages/index.html
@@ -2,10 +2,10 @@
 <html>
 
 <head>
-  <title>Redirecting to main branch</title>
-  <meta charset="utf-8">
-  <meta http-equiv="refresh" content="0; url=./main/index.html">
-  <link rel="canonical" href="main/index.html">
+    <title>Redirecting to main branch</title>
+    <meta charset="utf-8">
+    <meta http-equiv="refresh" content="0; url=./main/index.html">
+    <link rel="canonical" href="main/index.html">
 </head>
 
-</html>
+</html>
diff --git a/.github/pages/make_switcher.py b/.github/pages/make_switcher.py
@@ -59,12 +59,12 @@ def get_versions(ref: str, add: Optional[str], remove: Optional[str]) -> List[st
 def write_json(path: Path, repository: str, versions: str):
     org, repo_name = repository.split("/")
     struct = [
-        dict(version=version, url=f"https://{org}.github.io/{repo_name}/{version}/")
+        {"version": version, "url": f"https://{org}.github.io/{repo_name}/{version}/"}
         for version in versions
     ]
     text = json.dumps(struct, indent=2)
     print(f"JSON switcher:\n{text}")
-    path.write_text(text)
+    path.write_text(text, encoding="utf-8")
 
 
 def main(args=None):

diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml
@@ -3,12 +3,10 @@ name: Code CI
 on:
   push:
   pull_request:
-  schedule:
-    # Run weekly to check latest versions of dependencies
-    - cron: "0 8 * * WED"
 env:
   # The target python version, which must match the Dockerfile version
   CONTAINER_PYTHON: "3.11"
+  DIST_WHEEL_PATH: dist-${{ github.sha }}
 
 jobs:
   lint:
@@ -18,13 +16,14 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install python packages
         uses: ./.github/actions/install_requirements
         with:
           requirements_file: requirements-dev-3.x.txt
           install_options: -e .[dev]
+          artifact_name: lint
 
       - name: Lint
         run: tox -e pre-commit,mypy
@@ -36,7 +35,12 @@ jobs:
       matrix:
         os: ["ubuntu-latest"] # can add windows-latest, macos-latest
         python: ["3.9", "3.10", "3.11"]
-        install: [".[dev]", "-e .[dev]"]
+        install: ["-e .[dev]"]
+        # Make one version be non-editable to test both paths of version code
+        include:
+          - os: "ubuntu-latest"
+            python: "3.9"
+            install: ".[dev]"
 
     services:
       activemq:
@@ -48,7 +52,6 @@ jobs:
     env:
       # https://github.com/pytest-dev/pytest/issues/2042
       PY_IGNORE_IMPORTMISMATCH: "1"
-      BLUEAPI_TEST_STOMP_PORTS: "[61613,61614]"
 
     steps:
       - name: Start RabbitMQ
@@ -58,7 +61,7 @@ jobs:
           plugins: rabbitmq_stomp
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # Need this to get version number from last tag
           fetch-depth: 0
@@ -69,12 +72,13 @@ jobs:
           python_version: ${{ matrix.python }}
           requirements_file: requirements-test-${{ matrix.os }}-${{ matrix.python }}.txt
           install_options: ${{ matrix.install }}
+          artifact_name: tests
 
       - name: List dependency tree
         run: pipdeptree
 
       - name: Run tests
-        run: pytest
+        run: tox -e pytest
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
@@ -88,7 +92,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # Need this to get version number from last tag
           fetch-depth: 0
@@ -99,9 +103,9 @@ jobs:
           pipx run build
 
       - name: Upload sdist and wheel as artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4.0.0
         with:
-          name: dist
+          name: ${{ env.DIST_WHEEL_PATH }}
           path: dist
 
       - name: Check for packaging errors
@@ -113,10 +117,11 @@ jobs:
           python_version: ${{env.CONTAINER_PYTHON}}
           requirements_file: requirements.txt
           install_options: dist/*.whl
+          artifact_name: dist
 
       - name: Test module --version works using the installed wheel
         # If more than one module in src/ replace with module name to test
-        run: python -m $(ls src | head -1) --version
+        run: python -m $(ls --hide='*.egg-info' src | head -1) --version
 
   container:
     needs: [lint, dist, test]
@@ -126,82 +131,119 @@ jobs:
       contents: read
       packages: write
 
+    env:
+      TEST_TAG: "testing"
+
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # image names must be all lower case
       - name: Generate image repo name
         run: echo IMAGE_REPOSITORY=ghcr.io/$(tr '[:upper:]' '[:lower:]' <<< "${{ github.repository }}") >> $GITHUB_ENV
 
+      - name: Set lockfile location in environment
+        run: |
+          echo "DIST_LOCKFILE_PATH=lockfiles-${{ env.CONTAINER_PYTHON }}-dist-${{ github.sha }}" >> $GITHUB_ENV
+
       - name: Download wheel and lockfiles
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4.1.0
         with:
-          path: .devcontainer
+          path: artifacts/
+          pattern: "*dist*"
 
       - name: Log in to GitHub Docker Registry
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Docker meta
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and export to Docker local cache
+        uses: docker/build-push-action@v5
+        with:
+          # Note build-args, context, file, and target must all match between this
+          # step and the later build-push-action, otherwise the second build-push-action
+          # will attempt to build the image again
+          build-args: |
+            PIP_OPTIONS=-r ${{ env.DIST_LOCKFILE_PATH }}/requirements.txt ${{ env.DIST_WHEEL_PATH }}/*.whl
+          context: artifacts/
+          file: ./Dockerfile
+          target: runtime
+          load: true
+          tags: ${{ env.TEST_TAG }}
+          # If you have a long docker build (2+ minutes), uncomment the
+          # following to turn on caching. For short build times this
+          # makes it a little slower
+          #cache-from: type=gha
+          #cache-to: type=gha,mode=max
+
+      - name: Test cli works in cached runtime image
+        run: docker run docker.io/library/${{ env.TEST_TAG }} --version
+
+      - name: Create tags for publishing image
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.IMAGE_REPOSITORY }}
           tags: |
             type=ref,event=tag
-            type=raw,value=latest
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Build runtime image
-        uses: docker/build-push-action@v3
+            type=raw,value=latest, enable=${{ github.ref_type == 'tag' }}
+          # type=edge,branch=main
+          # Add line above to generate image for every commit to given branch,
+          # and uncomment the end of if clause in next step
+
+      - name: Push cached image to container registry
+        if: github.ref_type == 'tag' # || github.ref_name == 'main'
+        uses: docker/build-push-action@v5
+        # This does not build the image again, it will find the image in the
+        # Docker cache and publish it
         with:
+          # Note build-args, context, file, and target must all match between this
+          # step and the previous build-push-action, otherwise this step will
+          # attempt to build the image again
           build-args: |
-            PIP_OPTIONS=-r lockfiles/requirements.txt git+https://github.com/bluesky/bluesky.git git+https://github.com/DiamondLightSource/dls-bluesky-core.git git+https://github.com/DiamondLightSource/dodal.git@directory_provider dist/*.whl
-          push: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }}
-          load: ${{ ! (github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) }}
+            PIP_OPTIONS=-r ${{ env.DIST_LOCKFILE_PATH }}/requirements.txt ${{ env.DIST_WHEEL_PATH }}/*.whl
+          context: artifacts/
+          file: ./Dockerfile
+          target: runtime
+          push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          context: .devcontainer
-          # If you have a long docker build, uncomment the following to turn on caching
-          # For short build times this makes it a little slower
-          #cache-from: type=gha
-          #cache-to: type=gha,mode=max
-
-      - name: Test cli works in runtime image
-        run: docker run ${{ env.IMAGE_REPOSITORY }} --version
 
   release:
     # upload to PyPI and make a release on every tag
     needs: [lint, dist, test]
-    if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags') }}
+    if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
     runs-on: ubuntu-latest
     env:
       HAS_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN != '' }}
 
     steps:
-      - uses: actions/download-artifact@v3
+      - name: Download wheel and lockfiles
+        uses: actions/[email protected]
+        with:
+          path: artifacts/
+          pattern: "*dist*"
 
       - name: Fixup blank lockfiles
         # Github release artifacts can't be blank
-        run: for f in lockfiles/*; do [ -s $f ] || echo '# No requirements' >> $f; done
+        run: for f in ${{ env.DIST_LOCKFILE_PATH }}/*; do [ -s $f ] || echo '# No requirements' >> $f; done
 
       - name: Github Release
         # We pin to the SHA, not the tag, for security reasons.
         # https://docs.github.com/en/actions/learn-github-actions/security-hardening-for-github-actions#using-third-party-actions
-        uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 # v0.1.14
+        uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15
         with:
           prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }}
           files: |
-            dist/*
-            lockfiles/*
+            ${{ env.DIST_WHEEL_PATH }}/*
+            ${{ env.DIST_LOCKFILE_PATH }}/*
           generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}