Add process lock for remote filesystem scraper #1834
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI tests | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| env: | |
| FORCE_COLOR: true | |
| # Need to set this to avoid pipenv syncing an entire LFS repo and failing with bandwidth quota (see | |
| # https://github.com/the-grey-group/datalab/issues/603) | |
| GIT_LFS_SKIP_SMUDGE: 1 | |
| concurrency: | |
| group: ${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre-commit: | |
| name: Run linters and other pre-commit hooks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies | |
| working-directory: ./pydatalab | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -U setuptools | |
| pip install pipenv | |
| pipenv sync --dev | |
| pipenv run pip install invoke | |
| pipenv run pip install -e . | |
| - name: Run pre-commit | |
| working-directory: ./pydatalab | |
| run: | | |
| pipenv run pre-commit run --all-files --show-diff-on-failure | |
| pytest: | |
| name: Run Python unit tests | |
| services: | |
| mongo: | |
| image: mongo:3 | |
| ports: | |
| - 27017:27017 | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 2 | |
| matrix: | |
| python-version: ["3.10", "3.11"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install MongoDB tools binaries | |
| run: | | |
| wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-100.9.0.deb && sudo apt install ./mongodb-database-tools-*-100.9.0.deb | |
| - name: Install latest compatible versions of immediate dependencies | |
| working-directory: ./pydatalab | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -U setuptools | |
| pip install pipenv | |
| pipenv sync --dev | |
| pipenv run pip install -e . | |
| pipenv graph | |
| - name: Run all tests | |
| working-directory: ./pydatalab | |
| run: | | |
| pipenv run pytest -rs -vvv --cov-report=term --cov-report=xml --cov ./pydatalab ./tests | |
| - name: Upload coverage to Codecov | |
| if: matrix.python-version == '3.10' && github.repository == 'the-grey-group/datalabvue' | |
| uses: codecov/codecov-action@v3 | |
| webapp: | |
| name: Test the app build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Node | |
| uses: actions/setup-node@v3 | |
| env: | |
| FORCE_COLOR: 0 | |
| with: | |
| node-version: "16.15.0" | |
| cache: "yarn" | |
| cache-dependency-path: ./webapp/yarn.lock | |
| - name: Install web app | |
| working-directory: ./webapp | |
| run: yarn install --frozen-lockfile | |
| - name: Build web app | |
| working-directory: ./webapp | |
| run: yarn build | |
| e2e: | |
| name: Run e2e tests across app and API | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Debugging | |
| run: | | |
| docker image ls | |
| - name: Build the Docker images | |
| uses: docker/bake-action@v4 | |
| with: | |
| files: docker-compose.yml | |
| load: true | |
| set: | | |
| *.cache-from=type=gha,scope=cached-stage | |
| *.cache-to=type=gha,scope=cached-stage,mode=max | |
| - name: Start Docker images | |
| run: | | |
| # Create a named docker network that all containers attach to | |
| docker network create nginx | |
| # Add default API URL argument to Vue prod build | |
| echo "VUE_APP_API_URL=http://localhost:5001" >> .env | |
| # Launch production container profiles and wait for them to come up | |
| docker compose up database api_dev app -d --wait | |
| - name: Run end-to-end tests | |
| uses: cypress-io/github-action@v6 | |
| with: | |
| config: baseUrl=http://localhost:8081 | |
| working-directory: ./webapp | |
| record: true | |
| spec: | | |
| ./cypress/e2e/sampleTablePage.cy.js | |
| ./cypress/e2e/editPage.cy.js | |
| browser: "electron" | |
| env: | |
| CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} | |
| - name: Run end-to-end tests (Chrome) | |
| uses: cypress-io/github-action@v6 | |
| with: | |
| config: baseUrl=http://localhost:8081 | |
| working-directory: ./webapp | |
| record: true | |
| spec: | | |
| ./cypress/e2e/sampleTablePage.cy.js | |
| ./cypress/e2e/editPage.cy.js | |
| browser: "electron" | |
| env: | |
| CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} | |
| - name: Run end-to-end tests (Firefox) | |
| uses: cypress-io/github-action@v6 | |
| with: | |
| config: baseUrl=http://localhost:8081 | |
| working-directory: ./webapp | |
| record: true | |
| spec: | | |
| ./cypress/e2e/sampleTablePage.cy.js | |
| ./cypress/e2e/editPage.cy.js | |
| browser: "firefox" | |
| env: | |
| CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} |