Merge branch 'main' into ui-12

.github/actions/toolchain-cargo-cached/action.yml

@@ -0,0 +1,108 @@
+name: 'Checkout, rust toolchain, and cache'
+description: 'This action checks out the commit, sets up rust and installs dependencies. Uses caching internally.'
+
+inputs:
+  # Toolchain, targets, target, and components are passed into dtolnay/rust-toolchain
+  toolchain:
+    description: Rust toolchain specification -- see https://rust-lang.github.io/rustup/concepts/toolchains.html#toolchain-specification
+    required: true
+    default: "1.77.1"
+  targets:
+    description: Comma-separated list of target triples to install for this toolchain
+    required: false
+  target:
+    description: Alias for `targets`
+    required: false
+  components:
+    description: Comma-separated list of components to be additionally installed
+    required: false
+
+  # Install additional crates. For example cargo-audit.
+  crates:
+    description: Comma-separated list of crates to be additionally installed
+    required: false
+
+  cached-workspaces:
+    description: |
+      The cargo workspaces and target directory configuration.
+      Has the form `$workspace -> $target`. The `$target` part is treated as a directory
+      default: ". -> target"
+    default: ". -> target"
+    required: false
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - id: toolchain-cache-key
+      run: |
+        : Get current year and week like "2024-w4"
+        date="$(date +'%Y-w%U')"
+        : construct toolchain cache key
+        toolchain_cache_key="$(echo -n "$components" | md5sum | awk '{ print $1 }')"
+        echo "cachekey=$toolchain_cache_key-$date" >> $GITHUB_OUTPUT
+      env:
+        components: ${{ inputs.components }}
+      shell: bash
+
+      # Caching the toolchain, so we don't install rust for every commit/PR.
+      # Including the date from the previous step in the cache key means
+      # that the cache is invalidated on a weekly basis - so we're still
+      # up-to-date.
+    - name: Cache toolchain
+      # Don't cache if it's a scheduled job.
+      if: github.event_name != 'schedule'
+      id: toolchain-cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.rustup/settings.toml
+          ~/.rustup/toolchains/${{ inputs.toolchain }}-*
+          ~/.rustup/update-hashes/${{ inputs.toolchain }}-*
+        key: rust-toolchain-${{ inputs.toolchain }}-${{ steps.toolchain-cache-key.outputs.cachekey }}
+
+    - name: Install rust toolchain
+      # Only install the toolchain if it isn't cached.
+      if: steps.toolchain-cache.outputs.cache-hit != 'true'
+      id: toolchain
+      # Pin on SHA for immutability.
+      uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17
+      with:
+        toolchain: ${{ inputs.toolchain }}
+        targets: ${{ inputs.targets }}
+        target: ${{ inputs.target }}
+        components: ${{ inputs.components }}
+
+    - id: cargo-crates-cachekey
+      run: |
+        : construct crate cache key
+        crates_cache_key="$(echo -n "$crates" | md5sum | awk '{ print $1 }')"
+        echo "cache_key=$crates_cache_key" >> $GITHUB_OUTPUT
+      env:
+        crates: ${{ inputs.crates }}
+      shell: bash
+
+      # Pin on SHA for immutability.
+    - uses: Swatinem/rust-cache@9bdad043e88c75890e36ad3bbc8d27f0090dd609
+      id: rust-cache
+      if: github.event_name != 'schedule'
+      with:
+        shared-key: ${{ steps.cargo-crates-cachekey.outputs.cachekey }}
+        workspaces: ${{ inputs.cached-workspaces }}
+
+    - name: Cargo install dependencies
+      # Only install crates if they aren't cached.
+      if: steps.rust-cache.outputs.cache-hit != 'true'
+      id: cargo-install
+      run: |
+        IFS=',' read -ra crates_array <<< "$crates"
+        for c in "${crates_array[@]}"; do
+          echo "::debug::cargo install $c"
+          cargo install $c;
+        done
+        unset IFS;
+      env:
+        crates: ${{ inputs.crates }}
+      shell: bash

.github/file-filters.yaml

@@ -0,0 +1,72 @@
+# This is used by the action https://github.com/dorny/paths-filter
+
+# Changes to actions/workflows that can affect any of our project components.
+action_changes: &action_changes
+  - '.github/actions/**/action.yaml'
+  - '.github/workflows/ci.yaml'
+  - '.github/file-filters.yaml'
+
+# Frontend
+frontend_dependencies: &frontend_dependencies
+  - 'frontend/{package,now,vercel}.json'
+  - 'frontend/package-lock.json'
+
+eslint_config: &eslint_config
+  - 'frontend/.eslint*'
+
+frontend_workflow: &frontend_workflow
+  - '.github/workflows/frontend.yaml'
+
+frontend: &frontend
+  - *action_changes
+  - *frontend_dependencies
+  - *frontend_workflow
+  - *eslint_config
+  - 'frontend/**/*.[tj]{s,sx}'
+  - 'frontend/**/tsconfig*.json'
+
+proto: &proto
+  - 'proto/*.proto'
+
+# Agency
+agency_dependencies: &agency_dependencies
+  - 'agency/pyproject.toml'
+  - 'agency/poetry.lock'
+  - 'agency/.python-version'
+
+agency_workflow: &agency_workflow
+  - '.github/workflows/agency.yaml'
+
+agency: &agency
+  - *action_changes
+  - *agency_dependencies
+  - *agency_workflow
+  - *proto
+  - 'agency/app/**/*.py'
+
+# Server
+server_dependencies: &server_dependencies
+  - 'Cargo.toml'
+  - 'Cargo.lock'
+  - 'rust-toolchain.toml'
+
+server_workflow: &server_workflow
+  - '.github/workflows/server.yaml'
+
+server_migrations: &server_migrations
+  - 'server/migrations/**.sql'
+
+sqlx_query_cache: &sqlx_query_cache
+  - 'server/.sqlx/**.json'
+
+server_config: &server_config
+  - 'server/config/**.toml'
+
+server: &server
+  - *action_changes
+  - *server_dependencies
+  - *server_workflow
+  - *server_migrations
+  - *sqlx_query_cache
+  - *proto
+  - 'server/src/**/*.rs'

.github/workflows/python.yaml → .github/workflows/agency.yaml

@@ -1,7 +1,6 @@
-name: Python CI
-
+name: Agency CI
 on:
-  push:
+  workflow_call:
 
 defaults:
   run:
@@ -14,16 +13,20 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install poetry
-        run: pipx install poetry
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: "1.8.3"
+          virtualenvs-create: true
+          virtualenvs-in-project: true
 
       - uses: actions/setup-python@v5
         with:
           cache: "poetry"
           python-version-file: "agency/.python-version"
 
       - name: Install dependencies
-        run: poetry install
+        run: source $VENV & poetry install
 
       - name: Linting & formatting
         run: make lint & make fmt-check

.github/workflows/ci.yaml

@@ -0,0 +1,44 @@
+name: CI
+
+on: push
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+  CARGO_NET_RETRY: 10
+  RUST_BACKTRACE: short
+  RUSTFLAGS: "-D warnings"
+  RUSTUP_MAX_RETRIES: 10
+
+jobs:
+  detect-changes:
+    name: Detect file changes
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    # Map a step output to a job output
+    outputs:
+      agency: ${{ steps.changes.outputs.agency }}
+      frontend: ${{ steps.changes.outputs.frontend }}
+      llmlingua_service: ${{ steps.changes.outputs.llmlingua_service }}
+      proto: ${{ steps.changes.outputs.proto }}
+      server: ${{ steps.changes.outputs.server }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Find changes
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
+        id: changes
+        with:
+          base: main
+          token: ${{ github.token }}
+          filters: .github/file-filters.yaml
+
+  agency:
+    if: needs.detect-changes.outputs.agency == 'true'
+    needs: detect-changes
+    uses: ./.github/workflows/agency.yaml
+
+  server:
+    if: needs.detect-changes.outputs.server == 'true'
+    needs: detect-changes
+    uses: ./.github/workflows/server.yaml

.github/workflows/server.yaml

@@ -0,0 +1,55 @@
+name: Rust Server CI
+on:
+  workflow_call:
+
+defaults:
+  run:
+    working-directory: server
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+  CARGO_NET_RETRY: 10
+  RUST_BACKTRACE: short
+  RUSTFLAGS: "-D warnings"
+  RUSTUP_MAX_RETRIES: 10
+
+jobs:
+  coverage-and-linting:
+    name: Test coverage & linting
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      # Copy proto files
+      - run: cp -r ../proto ./proto
+
+      - name: Setup Rust Toolchain
+        uses: ./.github/actions/toolchain-cargo-cached
+        with:
+          components: llvm-tools-preview, rustfmt, clippy
+          crates: cargo-llvm-cov
+          cached-workspaces: server -> target
+
+      - run: |
+          docker compose -f sqlx-test-docker-compose.yaml run -d -p 5432:5432 --name sqlx_postgres sqlx_postgres
+          docker exec sqlx_postgres bash -c "until pg_isready; do sleep 1; done"
+
+      - name: Tests & coverage
+        run: cargo llvm-cov test --no-fail-fast --workspace
+        env:
+          DATABASE_URL: postgres://postgres:password@localhost:5432/curieo_search
+          SQLX_OFFLINE: true
+          SQLX_OFFLINE_DIR: .sqlx
+
+      - name: Linting rustfmt
+        run: cargo fmt --all -- --check
+
+      - name: Linting clippy
+        run: cargo clippy --

renovate.json

@@ -6,27 +6,17 @@
   ],
   "packageRules": [
     {
-      "description": "Group all patch updates together",
-      "matchUpdateTypes": ["patch"],
-      "groupName": "all patch updates"
-    },
-    {
-      "description": "Group all minor updates together",
-      "matchUpdateTypes": ["minor"],
-      "groupName": "all minor updates"
-    },
-    {
-      "description": "Group all major updates together",
-      "matchUpdateTypes": ["major"],
-      "groupName": "all major updates"
-    },
-    {
-      "description": "Group all pin and digest updates together",
-      "matchUpdateTypes": ["pin", "digest"],
-      "groupName": "all pin and digest updates"
+      "groupName": "{{manager}} non-major dependencies",
+      "groupSlug": "{{manager}}-minor-patch",
+      "matchPackagePatterns": [
+        "*"
+      ],
+      "matchUpdateTypes": [
+        "minor",
+        "patch"
+      ]
     }
   ],
-  "schedule": ["before 5am on monday"],
   "baseBranches": ["main"],
   "labels": ["dependencies"],
   "timezone": "Europe/Amsterdam"