ci: slim with two stage docker build, added explicit semantic release…

… steps for clarity, and test pypi for pip install

.dockerignore

@@ -0,0 +1,10 @@
+dist/*
+*.jpg
+tests/*
+docs/*
+.gitignore
+.idea/*
+.vscode/*
+.ipynb_checkpoints/*
+examples/*
+__pycache__/*

.github/workflows/release.yml

@@ -18,29 +18,64 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Python Semantic Release
-        id: semantic
-        uses: python-semantic-release/python-semantic-release@master
+      - name: Set up Python
+        uses: actions/setup-python@v2
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          python-version: '3.11'
 
-      - name: Publish PyPi
-        if: steps.semantic.outputs.released == 'true'
+      - name: Install Poetry
         run: |
-          pip install poetry
-          poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
-          poetry publish --build
-          poetry publish
+          curl -sSL https://install.python-poetry.org | python3 -
+          echo "${HOME}/.local/bin" >> $GITHUB_PATH
 
-      - name: docker push version
-        if: steps.semantic.outputs.released == 'true'
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Set PYTHONPATH to subdirectory sdcat
+        run: echo "PYTHONPATH=." >> $GITHUB_ENV
+      - name: Check release status
+        id: release-status
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          pip install python-semantic-release
+          if semantic-release --noop --strict version
+          then
+            echo "Releasing new version."
+          else
+            echo "Skipping release steps."
+          fi
+
+      - if: steps.release-status.outputs.released == 'true'
+        name: Release to GitHub
+        id: github-release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          semantic-release version
+          git fetch --tags
+          for file in ./dist/**
+            do gh release upload "${{steps.release-status.outputs.tag}}" $file
+          done
+
+      - if: steps.release-status.outputs.released == 'true'
+        name: Release to Test PyPI
+        id: test-pypi-release
+        env:
+          TEST_PYPI_TOKEN: ${{ secrets.TEST_PYPI_TOKEN }}
+        run: |
+          poetry config repositories.test-pypi https://test.pypi.org/legacy/
+          poetry config pypi-token.test-pypi $TEST_PYPI_TOKEN
+          poetry publish -r test-pypi -u __token__
+
+      - if: steps.release-status.outputs.released == 'true'
+        name: docker push version
         run: |
-          export RELEASE_VERSION=$(echo ${{ steps.semantic.outputs.tag }} | cut -c 2-)
+          export RELEASE_VERSION=$(echo ${{ steps.release-status.outputs.tag }} | cut -c 2-)
           docker login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_TOKEN }} 
           docker buildx create --name mybuilder --platform linux/amd64,linux/arm64 --use
-          cd docker && \
-          docker buildx build --push --platform linux/amd64 -t mbari/sdcat:$RELEASE_VERSION-cuda124 --label GIT_VERSION=$RELEASE_VERSION --label IMAGE_URI=mbari/sdcat:$RELEASE_VERSION-cuda124 -f Dockerfile.cuda .
-          docker buildx build --push --platform linux/amd64,linux/arm64 -t mbari/sdcat:$RELEASE_VERSION --label GIT_VERSION=$RELEASE_VERSION --label IMAGE_URI=mbari/sdcat:$RELEASE_VERSION -f Dockerfile .
+          docker buildx build --push --platform linux/amd64 -t mbari/sdcat:$RELEASE_VERSION-cuda124 --label GIT_VERSION=$RELEASE_VERSION --label IMAGE_URI=mbari/sdcat:$RELEASE_VERSION-cuda124 -f docker/Dockerfile.cuda .
+          docker buildx build --push --platform linux/amd64,linux/arm64 -t mbari/sdcat:$RELEASE_VERSION --label GIT_VERSION=$RELEASE_VERSION --label IMAGE_URI=mbari/sdcat:$RELEASE_VERSION -f docker/Dockerfile .
   push_readme_to_dockerhub:
     runs-on: ubuntu-latest
     name: Push README to Docker Hub

docker/Dockerfile

@@ -1,31 +1,32 @@
-FROM ubuntu:22.04
+FROM python:3.11-slim
 
 LABEL vendor="MBARI"
 LABEL maintainer="[email protected]"
 LABEL license="Apache License 2.0"
 
-RUN apt-get update && apt-get install -y \
-    software-properties-common \
-    && apt-get update && apt-get install -y \
-    python3.11 \
-    python3.11-dev \
-    python3.11-distutils \
-    python3-pip \
-    curl \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install --no-install-recommends -y build-essential && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m venv /venv
 
 ARG GIT_VERSION=latest
 ARG IMAGE_URI=mbari/sdcat:${GIT_VERSION}
 
+ENV PATH="/venv/bin:$PATH"
+
 ENV APP_HOME=/app
-ENV PYTHONPATH=${APP_HOME}/sdcat
 ENV HF_HOME=/tmp/transformers_cache
-WORKDIR ${APP_HOME}
-RUN if [ "$GIT_VERSION" != "latest" ]; then \
-    python3.11 -m pip install sdcat:${GIT_VERSION}; \
-    else \
-    python3.11 -m pip install sdcat; \
-    fi
 
-ENTRYPOINT ["sdcat"]
+RUN python3 -m pip install --upgrade pip
+
+ADD . .
+RUN pip install poetry && poetry build && pip install dist/*.whl && rm -rf dist
+
+RUN apt-get remove -y build-essential && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR ${APP_HOME}
+ENV HOME=${APP_HOME}
+RUN chmod a+rwx -R ${APP_HOME}
+CMD ["sdcat"]

docker/Dockerfile.cuda

@@ -1,39 +1,37 @@
-FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
+FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04 AS builder
+
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    python3 python3-dev python3-venv python3-pip python3-wheel build-essential && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN python3 -m pip install --upgrade pip
+
+ADD . .
+RUN pip install poetry && poetry build && python3 -m pip install dist/*.whl
+
+FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
 
 LABEL vendor="MBARI"
 LABEL maintainer="[email protected]"
 LABEL license="Apache License 2.0"
 
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    cmake \
-    git \
-    wget \
-    curl \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN apt-get update && apt-get install -y \
-    python3 \
-    python3-pip \
-    && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /venv /venv
 
-ARG GIT_VERSION=latest
-ARG IMAGE_URI=mbari/sdcat:${GIT_VERSION}
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    python3 && apt-get clean && rm -rf /var/lib/apt/lists/*
 
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+ENV PATH="/venv/bin:$PATH"
 ENV APP_HOME=/app
-ENV PYTHONPATH=${APP_HOME}/sdcat
 ENV HF_HOME=/tmp/transformers_cache
-WORKDIR ${APP_HOME}
-RUN if [ "$GIT_VERSION" != "latest" ]; then \
-    python3 -m pip install sdcat:${GIT_VERSION}; \
-    else \
-    python3 -m pip install sdcat; \
-    fi
-
-RUN chmod a+rwx ${APP_HOME}
 ENV HOME=${APP_HOME}
 ENV NUMBA_CACHE_DIR=/tmp
+
 WORKDIR ${APP_HOME}
-RUN python3 -m pip install --upgrade pip && \
-    python3 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121
-ENTRYPOINT ["sdcat"]
+ENV HOME=${APP_HOME}
+RUN chmod a+rwx -R ${APP_HOME}
+CMD ["sdcat"]
+

pyproject.toml

@@ -1,36 +1,12 @@
-[tool.ruff]
-line-length = 170
-
-[tool.semantic_release]
-version_variable = [
-"sdcat/__init__.py:__version__"
-]
-major_on_zero = false
-branch = "main"
-
-[tool.semantic_release.remote.token]
-env = "GITHUB_TOKEN"
-
-[tool.semantic_release.changelog]
-changelog_file = "CHANGELOG.md"
-exclude_commit_patterns = [
-  '''chore(?:\([^)]*?\))?: .+''',
-  '''ci(?:\([^)]*?\))?: .+''',
-  '''refactor(?:\([^)]*?\))?: .+''',
-  '''style(?:\([^)]*?\))?: .+''',
-  '''test(?:\([^)]*?\))?: .+''',
-  '''build\((?!deps\): .+)''',
-  '''Merged? .*''',
-  '''Initial Commit.*''',
-  # Old semantic-release version commits
-  '''^\d+\.\d+\.\d+''',
-]
-
 [tool.poetry]
 name = "sdcat"
 version = "1.8.0"
 description = "Sliced Detection and Clustering Analysis Toolkit - Developed by MBARI"
-authors = ["danellecline <[email protected]>"]
+authors = [
+    "Danelle Cline <[email protected]>",
+    "Duane Edgington <[email protected]>",
+    "Fernanda Lecaros Saavedra <[email protected]>",
+]
 license = "Apache"
 readme = "README.md"
 packages = [
@@ -41,6 +17,35 @@ exclude = ["sdcat/tests"]
 [tool.poetry.scripts]
 sdcat = "sdcat.__main__:cli"
 
+[tool.semantic_release]
+branch = "main"
+upload_to_PyPI = false
+upload_to_release = true
+version_variable = [
+    "sdcat/__init__.py:__version__"
+]
+version_toml = [
+    "pyproject.toml:tool.poetry.version"
+]
+build_command = "pip install poetry && poetry build"
+
+[tool.semantic_release.commit_parser_options]
+allowed_tags = [
+    "build",
+    "chore",
+    "ci",
+    "docs",
+    "feat",
+    "fix",
+    "perf",
+    "style",
+    "refactor",
+    "test"
+]
+minor_tags = ["feat"]
+patch_tags = ["fix", "perf"]
+
+
 [tool.poetry.dependencies]
 python = ">=3.9,<3.12"
 pillow = "^10.4.0"
@@ -82,4 +87,7 @@ ignore_missing_imports = true
 
 [tool.pytest.ini_options]
 testpaths = ["sdcat/tests",]
-pythonpath = ["."]
+pythonpath = ["."]
+
+[tool.ruff]
+line-length = 125