Filter out b64-encoded data before sending to LLM (#870) #2759
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI tests | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| env: | |
| FORCE_COLOR: true | |
| concurrency: | |
| group: ${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre-commit: | |
| name: Run linters and other pre-commit hooks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Set up uv (latest) | |
| run: curl -LsSf https://astral.sh/uv/install.sh | sh | |
| - name: Install dependencies | |
| working-directory: ./pydatalab | |
| run: | | |
| uv venv | |
| uv pip install -r requirements/requirements-all-dev.txt | |
| uv pip install -e '.[all, dev]' | |
| - name: Run pre-commit | |
| working-directory: ./pydatalab | |
| run: | | |
| source .venv/bin/activate | |
| pre-commit run --all-files --show-diff-on-failure | |
| pytest: | |
| name: Run Python unit tests | |
| services: | |
| mongo: | |
| image: mongo:3 | |
| ports: | |
| - 27017:27017 | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 2 | |
| matrix: | |
| python-version: ["3.10", "3.11"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # tests need an unshallowed version of the repository to check the version | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install MongoDB tools binaries | |
| run: | | |
| wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-100.9.0.deb && sudo apt install ./mongodb-database-tools-*-100.9.0.deb | |
| - name: Set up uv (latest) | |
| run: curl -LsSf https://astral.sh/uv/install.sh | sh | |
| - name: Install locked versions of dependencies | |
| working-directory: ./pydatalab | |
| run: | | |
| uv venv | |
| uv pip install -r requirements/requirements-all-dev.txt | |
| uv pip install -e '.[all, dev]' | |
| - name: Run all tests | |
| working-directory: ./pydatalab | |
| run: | | |
| source .venv/bin/activate | |
| pytest -rs -vvv --cov-report=term --cov-report=xml --cov ./pydatalab ./tests | |
| - name: Upload coverage to Codecov | |
| if: matrix.python-version == '3.10' && github.repository == 'datalab-org/datalab' | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| slug: datalab-org/datalab | |
| webapp: | |
| name: Test the app build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Node | |
| uses: actions/setup-node@v4 | |
| env: | |
| FORCE_COLOR: 0 | |
| with: | |
| node-version: "20" | |
| cache: "yarn" | |
| cache-dependency-path: ./webapp/yarn.lock | |
| - name: Install web app | |
| working-directory: ./webapp | |
| run: yarn install --frozen-lockfile | |
| - name: Build web app | |
| working-directory: ./webapp | |
| run: yarn build | |
| docker: | |
| name: Test dev Docker builds | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| target: | |
| - "app_dev" | |
| - "api_dev" | |
| - "database_dev" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build Docker containers | |
| uses: docker/bake-action@v5 | |
| with: | |
| files: docker-compose.yml | |
| load: true | |
| targets: ${{ matrix.target }} | |
| set: | | |
| *.cache-to=type=gha,scope=build-${{ matrix.target }},mode=max | |
| *.cache-from=type=gha,scope=build-${{ matrix.target }} | |
| e2e: | |
| name: e2e tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 3 | |
| matrix: | |
| browser-backend: ["electron", "chrome", "firefox"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Set some environment variables for CI containers pre-build | |
| run: | | |
| cp webapp/.env.test_e2e .env | |
| echo "PYDATALAB_TESTING=true" >> pydatalab/.env | |
| - name: Load previously built Docker images | |
| uses: docker/bake-action@v5 | |
| with: | |
| files: docker-compose.yml | |
| load: true | |
| targets: 'app,api,database' | |
| set: | | |
| app.cache-to=type=gha,scope=build-app,mode=max | |
| app.cache-from=type=gha,scope=build-app | |
| app.tags=datalab-app:latest | |
| api.cache-to=type=gha,scope=build-api,mode=max | |
| api.cache-from=type=gha,scope=build-api | |
| api.tags=datalab-api:latest | |
| database.cache-to=type=gha,scope=build-database,mode=max | |
| database.cache-from=type=gha,scope=build-database | |
| database.tags=datalab-database:latest | |
| - name: Start services | |
| run: | | |
| # Launch production container profiles and wait for them to come up | |
| docker compose up database api app --no-build --force-recreate -d --wait | |
| - name: Run end-to-end tests | |
| uses: cypress-io/github-action@v6 | |
| with: | |
| config: baseUrl=http://localhost:8081 | |
| working-directory: ./webapp | |
| record: ${{ github.repository == 'datalab-org/datalab' }} | |
| browser: ${{ matrix.browser-backend }} | |
| group: "End-to-end tests (${{ matrix.browser-backend }})" | |
| env: | |
| CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} | |
| - name: Run component tests | |
| uses: cypress-io/github-action@v6 | |
| with: | |
| working-directory: ./webapp | |
| record: ${{ github.repository == 'datalab-org/datalab' }} | |
| install: false | |
| component: true | |
| publish-summary: false | |
| browser: ${{ matrix.browser-backend }} | |
| group: "Component tests (${{ matrix.browser-backend }})" | |
| env: | |
| CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} |