E2E tests #3517
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # several test modes: | |
| # - ALL: run all modes below (only useful for manual trigger) | |
| # - CUSTOM: choose git branch and endpoint (only useful for manual trigger) | |
| # - MAIN_STAGING: main branch against STAGING | |
| # - RELEASE_PREPROD: latest release branch against PREPROD | |
| # - MAIN_LTS: main branch against LTS | |
| # - PREV_RELEASE_TESTS_STAGING: SDK from main, previous release tests against STAGING | |
| # - LTS_CYCLE_FIRST_SDK_STAGING: test with first SDK of LTS cycle, against STAGING | |
| name: E2E tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| runE2ETests: | |
| description: "Run E2E tests?" | |
| required: true | |
| type: boolean | |
| default: true | |
| runNotebookTests: | |
| description: "Run notebook tests?" | |
| required: true | |
| type: boolean | |
| default: true | |
| testMode: | |
| description: "Test mode: ALL (run all modes), CUSTOM (specify branch and endpoint), MAIN_STAGING (main branch against STAGING), RELEASE_PREPROD (latest release branch against PREPROD), MAIN_LTS (main branch against LTS), PREV_RELEASE_TESTS_STAGING (SDK from main, previous release tests against STAGING), LTS_CYCLE_FIRST_SDK_STAGING (test with first SDK of LTS cycle, against STAGING)" | |
| required: true | |
| type: choice | |
| default: "MAIN_STAGING" | |
| options: | |
| - ALL | |
| - CUSTOM | |
| - MAIN_STAGING | |
| - RELEASE_PREPROD | |
| - MAIN_LTS | |
| - PREV_RELEASE_TESTS_STAGING | |
| - LTS_CYCLE_FIRST_SDK_STAGING | |
| customBranch: | |
| description: "Custom branch to test a PR (only used if test mode is CUSTOM)" | |
| required: false | |
| type: string | |
| default: "" | |
| customEndpoint: | |
| description: "Custom endpoint to test a PR (only used if test mode is CUSTOM)" | |
| required: false | |
| type: choice | |
| default: "STAGING" | |
| options: | |
| - "STAGING" | |
| - "PROD" | |
| - "PREPROD" | |
| - "LTS" | |
| push: | |
| branches: | |
| - "release/**" | |
| - main | |
| schedule: | |
| - cron: "0 7,15 * * 1-5" # MAIN_STAGING | |
| - cron: "0 11 * * 1-5" # PREV_RELEASE_TESTS_STAGING | |
| - cron: "0 13 * * 1-5" # LTS_CYCLE_FIRST_SDK_STAGING | |
| jobs: | |
| test_setup: | |
| name: E2E tests setup | |
| if: github.repository == 'kili-technology/kili-python-sdk' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test_mode_array: ${{ steps.setup_step.outputs.TEST_MODE_ARRAY }} | |
| steps: | |
| - name: Setup test mode | |
| id: setup_step | |
| shell: bash | |
| run: | | |
| if [[ "${{ github.event_name }}" == "push" ]]; then | |
| if [[ "${{ github.ref_name }}" == release/* ]]; then | |
| # if the push was on a release branch, it can be either | |
| # the lts branch -> LTS_CYCLE_FIRST_SDK_STAGING mode | |
| # or the latest release branch -> RELEASE_PREPROD mode | |
| if [[ "${{ github.ref_name }}" == release/2.146.0 ]]; then | |
| json_str="{\"array\": [\"LTS_CYCLE_FIRST_SDK_STAGING\"]}" | |
| else | |
| json_str="{\"array\": [\"RELEASE_PREPROD\"]}" | |
| fi | |
| else | |
| json_str="{\"array\": [\"MAIN_STAGING\"]}" | |
| fi | |
| elif [[ "${{ github.event_name }}" == "schedule" ]]; then | |
| if [[ "${{ github.event.schedule }}" == "0 7,15 * * 1-5" ]]; then | |
| json_str="{\"array\": [\"MAIN_STAGING\"]}" | |
| elif [[ "${{ github.event.schedule }}" == "0 9 * * 1-5" ]]; then | |
| json_str="{\"array\": [\"MAIN_LTS\"]}" | |
| elif [[ "${{ github.event.schedule }}" == "0 11 * * 1-5" ]]; then | |
| json_str="{\"array\": [\"PREV_RELEASE_TESTS_STAGING\"]}" | |
| elif [[ "${{ github.event.schedule }}" == "0 13 * * 1-5" ]]; then | |
| json_str="{\"array\": [\"LTS_CYCLE_FIRST_SDK_STAGING\"]}" | |
| fi | |
| elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| if [[ "${{ github.event.inputs.testMode }}" == "ALL" ]]; then | |
| json_str="{\"array\": [\"MAIN_STAGING\", \"RELEASE_PREPROD\", \"MAIN_LTS\", \"PREV_RELEASE_TESTS_STAGING\", \"LTS_CYCLE_FIRST_SDK_STAGING\"]}" | |
| else | |
| json_str="{\"array\": [\"${{ github.event.inputs.testMode }}\"]}" | |
| fi | |
| fi | |
| echo "TEST_MODE_ARRAY=$json_str" >> $GITHUB_OUTPUT | |
| tests: | |
| name: ${{ matrix.test_mode }}, ${{ matrix.os }}, ${{ matrix.python-version }} | |
| if: github.repository == 'kili-technology/kili-python-sdk' | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| test_mode: ${{ fromJson(needs.test_setup.outputs.test_mode_array).array }} | |
| os: | |
| - ubuntu-latest | |
| - windows-latest | |
| include: | |
| - os: ubuntu-latest | |
| python-version: 3.8 | |
| - os: windows-latest | |
| python-version: 3.12 | |
| runs-on: ${{ matrix.os }} | |
| needs: [test_setup] | |
| steps: | |
| - name: Set test mode | |
| shell: bash | |
| run: echo "TEST_MODE=${{ matrix.test_mode }}" >> $GITHUB_ENV | |
| - name: Check that test mode is valid | |
| shell: bash | |
| run: | | |
| if [[ ${{ env.TEST_MODE }} != "MAIN_STAGING" ]] && \ | |
| [[ ${{ env.TEST_MODE }} != "CUSTOM" ]] && \ | |
| [[ ${{ env.TEST_MODE }} != "RELEASE_PREPROD" ]] && \ | |
| [[ ${{ env.TEST_MODE }} != "MAIN_LTS" ]] && \ | |
| [[ ${{ env.TEST_MODE }} != "PREV_RELEASE_TESTS_STAGING" ]] && \ | |
| [[ ${{ env.TEST_MODE }} != "LTS_CYCLE_FIRST_SDK_STAGING" ]]; then | |
| echo "TEST_MODE is not valid: ${{ env.TEST_MODE }}" | |
| exit 1 | |
| fi | |
| - name: Set first SDK version of LTS cycle | |
| if: env.TEST_MODE == 'LTS_CYCLE_FIRST_SDK_STAGING' | |
| shell: bash | |
| run: echo "LTS_CYCLE_FIRST_SDK_REF=release/2.146.0" >> $GITHUB_ENV # this should be updated when a new LTS cycle starts | |
| - name: Add latest release branch to github env | |
| shell: bash | |
| timeout-minutes: 3 | |
| run: | | |
| curl https://raw.githubusercontent.com/kili-technology/kili-python-sdk/main/.github/scripts/add_latest_release_branch_to_github_env.py --output add_latest_release_branch_to_github_env.py | |
| python add_latest_release_branch_to_github_env.py | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set checkout branch/tag and endpoint | |
| shell: bash | |
| run: | | |
| if [[ "${{ env.TEST_MODE }}" == "MAIN_STAGING" ]]; then | |
| echo "REF=main" >> $GITHUB_ENV | |
| echo "TEST_AGAINST=STAGING" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_MODE }}" == "RELEASE_PREPROD" ]]; then | |
| echo "REF=${{ env.LASTEST_RELEASE_BRANCH }}" >> $GITHUB_ENV | |
| echo "TEST_AGAINST=PREPROD" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_MODE }}" == "MAIN_LTS" ]]; then | |
| echo "REF=main" >> $GITHUB_ENV | |
| echo "TEST_AGAINST=LTS" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_MODE }}" == "PREV_RELEASE_TESTS_STAGING" ]]; then | |
| echo "REF=${{ env.LASTEST_RELEASE_BRANCH }}" >> $GITHUB_ENV | |
| echo "TEST_AGAINST=STAGING" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_MODE }}" == "LTS_CYCLE_FIRST_SDK_STAGING" ]]; then | |
| echo "REF=${{ env.LTS_CYCLE_FIRST_SDK_REF }}" >> $GITHUB_ENV | |
| echo "TEST_AGAINST=STAGING" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_MODE }}" == "CUSTOM" ]]; then | |
| echo "REF=${{ github.event.inputs.customBranch }}" >> $GITHUB_ENV | |
| echo "TEST_AGAINST=${{ github.event.inputs.customEndpoint }}" >> $GITHUB_ENV | |
| fi | |
| - name: Set KILI_API_ENDPOINT | |
| shell: bash | |
| run: | | |
| if [[ "${{ env.TEST_AGAINST }}" == "STAGING" ]]; then | |
| echo "KILI_API_ENDPOINT=https://staging.cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_AGAINST }}" == "PREPROD" ]]; then | |
| echo "KILI_API_ENDPOINT=https://preproduction.cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_AGAINST }}" == "LTS" ]]; then | |
| echo "KILI_API_ENDPOINT=https://lts.cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV | |
| elif [[ "${{ env.TEST_AGAINST }}" == "PROD" ]]; then | |
| echo "KILI_API_ENDPOINT=https://cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV | |
| else | |
| echo "TEST_AGAINST is not valid: ${{ env.TEST_AGAINST }}" | |
| exit 1 | |
| fi | |
| - name: Check REF is set | |
| shell: bash | |
| run: | | |
| if [[ -z "${{ env.REF }}" ]]; then | |
| echo "REF is not set" | |
| exit 1 | |
| fi | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.REF }} | |
| - name: Delete src for PREV_RELEASE_TESTS_STAGING mode | |
| if: env.TEST_MODE == 'PREV_RELEASE_TESTS_STAGING' | |
| shell: bash | |
| run: rm -rf src # we want the previous release tests but the kili from main | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "pip" | |
| - name: Upgrade pip | |
| run: python -m pip install --upgrade pip | |
| - name: Install kili | |
| shell: bash | |
| run: | | |
| if [[ "${{ env.TEST_MODE }}" == "PREV_RELEASE_TESTS_STAGING" ]]; then | |
| pip install "kili[dev] @ git+https://github.com/kili-technology/kili-python-sdk.git@main" | |
| else | |
| pip install -e ".[dev]" | |
| fi | |
| - name: Get kili SDK version | |
| shell: bash | |
| run: echo "KILI_SDK_VERSION=$(python -c "from kili import __version__; print(__version__)")" >> $GITHUB_ENV | |
| - name: E2E tests | |
| id: e2e | |
| if: github.event.inputs.runE2ETests != 'false' | |
| timeout-minutes: 60 | |
| run: pytest --timeout=600 -ra -sv --color yes --code-highlight yes --durations=0 -vv --ignore tests/e2e/test_notebooks.py tests/e2e | |
| env: | |
| KILI_API_CLOUD_VISION: ${{ secrets.KILI_API_CLOUD_VISION }} | |
| KILI_API_ENDPOINT: ${{ env.KILI_API_ENDPOINT }} | |
| KILI_API_KEY: ${{ secrets[format('KILI_USER_API_KEY_{0}', env.TEST_AGAINST)] }} | |
| KILI_USER_EMAIL: ${{ secrets[format('KILI_USER_EMAIL_{0}', env.TEST_AGAINST)] }} | |
| KILI_USER_ID: ${{ secrets[format('KILI_USER_ID_{0}', env.TEST_AGAINST)] }} | |
| KILI_TEST_DATA_INTEGRATION_ID: ${{ secrets.KILI_TEST_DATA_INTEGRATION_ID }} | |
| KILI_TEST_ORGANIZATION_ID: ${{ secrets[format('KILI_TEST_ORGANIZATION_ID_{0}', env.TEST_AGAINST)] }} | |
| KILI_TEST_AWS_ACCESS_POINT_ARN: ${{ secrets.KILI_TEST_AWS_ACCESS_POINT_ARN }} | |
| KILI_TEST_AWS_ROLE_ARN: ${{ secrets.KILI_TEST_AWS_ROLE_ARN }} | |
| KILI_TEST_AWS_ROLE_EXTERNAL_ID: ${{ secrets.KILI_TEST_AWS_ROLE_EXTERNAL_ID }} | |
| - name: Notebook tests | |
| # we don't run notebook tests on push to main | |
| # we run regardless of the outcome of the e2e tests | |
| if: ${{ !cancelled() && github.event.inputs.runNotebookTests != 'false' && (github.event_name != 'push' || github.ref_name != 'main') && (steps.e2e.outcome != 'cancelled') }} | |
| timeout-minutes: 60 | |
| run: pytest --timeout=600 -ra -sv --color yes --code-highlight yes --durations=0 -vv tests/e2e/test_notebooks.py | |
| env: | |
| KILI_API_CLOUD_VISION: ${{ secrets.KILI_API_CLOUD_VISION }} | |
| KILI_API_ENDPOINT: ${{ env.KILI_API_ENDPOINT }} | |
| KILI_API_KEY: ${{ secrets[format('KILI_USER_API_KEY_{0}', env.TEST_AGAINST)] }} | |
| KILI_USER_EMAIL: ${{ secrets[format('KILI_USER_EMAIL_{0}', env.TEST_AGAINST)] }} | |
| KILI_USER_ID: ${{ secrets[format('KILI_USER_ID_{0}', env.TEST_AGAINST)] }} | |
| KILI_TEST_DATA_INTEGRATION_ID: ${{ secrets.KILI_TEST_DATA_INTEGRATION_ID }} | |
| PYDEVD_DISABLE_FILE_VALIDATION: true | |
| - name: Check that kili SDK version has not changed | |
| shell: bash | |
| run: | | |
| kili_sdk_version=$(python -c "from kili import __version__; print(__version__)") | |
| if [[ "${{ env.KILI_SDK_VERSION }}" != "$kili_sdk_version" ]]; then | |
| echo "Kili has been updated (maybe by notebooks)." | |
| echo "The version should not be updated during tests." | |
| echo "version before: ${{ env.KILI_SDK_VERSION }}" | |
| echo "version after: $kili_sdk_version" | |
| exit 1 | |
| fi | |
| - name: Slack notification if failure | |
| if: failure() && (github.event_name != 'workflow_dispatch') # doesn't notify for manual runs | |
| id: slack | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: | | |
| { | |
| "text": "E2E tests '${{ env.TEST_MODE }}' (branch '${{ env.REF }}' against endpoint '${{ env.TEST_AGAINST }}') failed with status '${{ job.status }}'\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "E2E tests '${{ env.TEST_MODE }}' (branch '${{ env.REF }}' against endpoint '${{ env.TEST_AGAINST }}') failed with status '${{ job.status }}'\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_PYTHON_SDK_ALARMING }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |