Switch Python package to src-layout

.docker/app_dockerfile

@@ -1,5 +1,5 @@
-# syntax=docker/dockerfile:experimental
-FROM node:20-bullseye as base
+# syntax=docker/dockerfile:1
+FROM node:20-bullseye AS base
 SHELL ["/bin/bash", "--login", "-c"]
 
 WORKDIR /app
@@ -10,9 +10,9 @@
 
 # Using a custom node_modules location to avoid mounting it outside of docker
 RUN --mount=type=cache,target=/root/.cache/yarn yarn install --frozen-lockfile --modules-folder /node_modules
 ENV PATH $PATH:/node_modules/.bin
 
 FROM base as development
 
 # Install cypress dependencies for local testing
 RUN apt-get update \
@@ -21,8 +21,8 @@
 
 CMD [ "yarn", "serve", "--port", "8081"]
 
 FROM base as production
 ENV NODE_ENV production
 
 # These get replaced by the entrypoint script for production builds.
 # Set the real values in `.env` files or an external docker-compose.

.docker/server_dockerfile

@@ -1,5 +1,5 @@
-# syntax=docker/dockerfile:experimental
-FROM python:3.10 as base
+# syntax=docker/dockerfile:1
+FROM python:3.10 AS base
 SHELL ["/bin/bash", "--login", "-c"]
 
 # Useful for installing deps from git, preventing big downloads and bandwith quotas
@@ -12,31 +12,36 @@
 WORKDIR /opt
 RUN wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-100.9.0.deb && apt install ./mongodb-database-tools-*-100.9.0.deb
 
 FROM base as app
 
 WORKDIR /app
 
-# Set up uv venv in /opt/.venv to avoid polluting the mounted /app dir
-RUN pip install uv
+COPY --from=ghcr.io/astral-sh/uv:0.4 /uv /usr/local/bin/uv
+ENV UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_PROJECT_ENVIRONMENT=/opt/.venv \
+    UV_PYTHON=python3.10
+
 WORKDIR /opt
+COPY pydatalab/pyproject.toml /_lock/
+COPY pydatalab/uv.lock /_lock/
+COPY pydatalab/requirements/requirements-all.txt /_lock/
 RUN uv venv
+RUN uv pip install -r /_lock/requirements-all.txt
 
-COPY pydatalab/requirements/requirements-all.txt .
-RUN uv pip install -r requirements-all.txt
-
-WORKDIR /app
-COPY pydatalab/pyproject.toml .
 # Create development image using flask's dev server with hot-reload
 FROM app as development
 
-ENV FLASK_APP "pydatalab.main:create_app()"
+
+WORKDIR /app
+ENV FLASK_APP "pydatalab.main"
 ENV FLASK_ENV "development"
 ENV PORT=5001
-
-CMD ["/bin/bash", "-c", "/opt/.venv/bin/python -m flask run --reload --port ${PORT} --host 0.0.0.0"]
+CMD [ "/bin/bash", "-c", "source /opt/.venv/bin/activate && exec flask run --reload --port ${PORT} --host 0.0.0.0" ]
 
 # Create production image using gunicorn and minimal dependencies
 FROM app as production
 
 WORKDIR /opt
 RUN [ "uv", "pip", "install", "gunicorn" ]
@@ -54,8 +59,8 @@
 
 # Install the local version of the package and mount the repository data to get version info
 RUN git config --global --add safe.directory /
-RUN --mount=type=bind,target=/.git,source=./.git ["uv", "pip", "install", "--python", "/opt/.venv/bin/python", "."]
+RUN --mount=type=bind,target=/.git,source=./.git ["uv", "pip", "install", "--python", "/opt/.venv/bin/python", "--no-deps", "."]
 
 CMD ["/bin/bash", "-c", "/opt/.venv/bin/python -m gunicorn --preload -w ${WEB_CONCURRENCY} --error-logfile /logs/pydatalab_error.log --access-logfile - -b 0.0.0.0:${PORT} 'pydatalab.main:create_app()'"]
 
-HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD curl --fail http://localhost:${PORT}/healthcheck/is_ready || exit 1
+HEALTHCHECK --interval=30s --timeout=30s --start-interval=15s --start-period=30s --retries=3 CMD curl --fail http://localhost:${PORT}/healthcheck/is_ready || exit 1

.github/workflows/ci.yml

@@ -123,12 +123,6 @@ jobs:
   docker:
     name: Test dev Docker builds
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        target:
-          - "app_dev"
-          - "api_dev"
-          - "database_dev"
 
     steps:
       - uses: actions/checkout@v4
@@ -143,8 +137,20 @@ jobs:
           load: true
           targets: ${{ matrix.target }}
           set: |
-            *.cache-to=type=gha,scope=build-${{ matrix.target }},mode=max
-            *.cache-from=type=gha,scope=build-${{ matrix.target }}
+            app_dev.cache-to=type=gha,scope=build-app_dev,mode=max
+            app_dev.cache-from=type=gha,scope=build-app_dev
+            app_dev.tags=datalab-app_dev:latest
+            api_dev.cache-to=type=gha,scope=build-api_dev,mode=max
+            api_dev.cache-from=type=gha,scope=build-api_dev
+            api_dev.tags=datalab-api_dev:latest
+            database_dev.cache-to=type=gha,scope=build-database-dev,mode=max
+            database_dev.cache-from=type=gha,scope=build-database_dev
+            database_dev.tags=datalab-database_dev:latest
+
+      - name: Start services
+        run: |
+          # Launch dev container profiles and wait for them to come up with healthchecks
+          docker compose up app_dev api_dev database_dev --wait --no-build --force-recreate -d
 
   e2e:
     name: e2e tests

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
       - id: mixed-line-ending
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.5.0"
+    rev: v0.6.3
     hooks:
       - id: ruff
         args: [--fix]
@@ -57,12 +57,12 @@ repos:
         args: [--fix]
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.16.0
+    rev: v3.17.0
     hooks:
       - id: pyupgrade
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.10.1
+    rev: v1.11.2
     hooks:
       - id: mypy
         additional_dependencies:
@@ -73,7 +73,7 @@ repos:
     hooks:
       - id: generate-schemas
         name: Regenerate item model JSONSchemas
-        files: "^pydatalab/pydatalab/models/.*.$"
+        files: "^pydatalab/src/pydatalab/models/.*.$"
         description: Check if the current code changes have enacted changes to the resulting JSONSchemas
         entry: invoke -r pydatalab dev.generate-schemas
         pass_filenames: false

INSTALL.md

@@ -105,14 +105,14 @@ To make use of this file:
     ```shell
     cd pydatalab
     source .venv/bin/activate
-    flask --app 'pydatalab:main.create_app()' --reload run
+    flask --app 'pydatalab:main' run --reload
     ```
 
 === "Launching with `pipenv`"
 
     ```shell
     cd pydatalab
-    pipenv run flask --app 'pydatalab:main.create_app()' --reload run
+    pipenv run flask --app 'pydatalab:main' run --reload
     ```
 
 The server should now be accessible at [http://localhost:5001](http://localhost:5001).

docker-compose.yml

@@ -64,7 +64,7 @@ services:
     volumes:
       - ./logs:/logs
       - ./.git:/.git
-      - ./pydatalab:/app
+      - ./pydatalab/src:/app
     ports:
       - "5001:5001"
     networks: