feat: enable remote pilot logging system #138
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Extensions full test | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - gubbins | |
| pull_request: | |
| branches: | |
| - main | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| jobs: | |
| unittest: | |
| name: Unit test - ${{ matrix.package }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # In principle, the dependencies could be limited to the extension packages. | |
| # However, we want to use the local packages, and not those published on pypi | |
| include: | |
| - package: "./extensions/gubbins/gubbins-core" | |
| dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./diracx-core" | |
| - package: "./extensions/gubbins/gubbins-db" | |
| dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core " | |
| - package: "./extensions/gubbins/gubbins-routers" | |
| dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-db ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core ./diracx-routers" | |
| - package: "./extensions/gubbins/gubbins-client" | |
| dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-client ./diracx-core " | |
| - package: "./extensions/gubbins/gubbins-cli" | |
| dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-cli ./diracx-client ./diracx-core ./diracx-api" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| # TODO: Use a conda environment file used for the diracx/base container image | |
| environment-name: test-env | |
| create-args: >- | |
| python=3.11 | |
| m2crypto | |
| python-gfal2 | |
| mypy | |
| pip | |
| init-shell: bash | |
| post-cleanup: 'all' | |
| - name: Set up environment | |
| run: | | |
| pip install pytest-github-actions-annotate-failures | |
| pip install git+https://github.com/DIRACGrid/DIRAC.git@integration | |
| pip install ${{ matrix.dependencies }} ${{ matrix.package }}[types] | |
| - name: Run mypy | |
| run: | | |
| mypy ${{ matrix.package }}/src | |
| - name: Run pytest | |
| run: | | |
| cd ${{ matrix.package }} | |
| pip install .[testing] | |
| export DIRACX_EXTENSIONS=gubbins,diracx | |
| pytest --cov-report=xml:coverage.xml --junitxml=report.xml | |
| - name: Upload coverage report | |
| uses: codecov/[email protected] | |
| build-wheels: | |
| name: Build wheels | |
| runs-on: "ubuntu-latest" | |
| if: github.event_name != 'push' || github.repository == 'DIRACGrid/diracx' | |
| defaults: | |
| run: | |
| # We need extglob for REFERENCE_BRANCH substitution | |
| shell: bash -l -O extglob {0} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Installing dependencies | |
| run: | | |
| python -m pip install \ | |
| build \ | |
| python-dateutil \ | |
| pytz \ | |
| readme_renderer[md] \ | |
| requests \ | |
| setuptools_scm | |
| - name: Build distributions | |
| run: | | |
| for pkg_dir in $PWD/diracx-*; do | |
| echo "Building $pkg_dir" | |
| python -m build --outdir $PWD/dist $pkg_dir | |
| done | |
| # Also build the diracx metapackage | |
| python -m build --outdir $PWD/dist . | |
| # And build the gubbins package | |
| for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do | |
| # Skip the testing package | |
| if [[ "${pkg_dir}" =~ .*testing.* ]]; | |
| then | |
| echo "Do not build ${pkg_dir}"; | |
| continue; | |
| fi | |
| echo "Building $pkg_dir" | |
| python -m build --outdir $PWD/dist $pkg_dir | |
| done | |
| - name: 'Upload Artifact' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gubbins-whl | |
| path: dist/*.whl | |
| retention-days: 5 | |
| # Build to docker image with the code in it | |
| build-image: | |
| needs: build-wheels | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Download gubbins wheels | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gubbins-whl | |
| - name: "Find wheels" | |
| id: find_wheel | |
| run: | | |
| # We need to copy them there to be able to access them in the RUN --mount | |
| cp diracx*.whl gubbins*.whl extensions/containers/services/ | |
| for wheel_fn in *.whl; do | |
| pkg_name=$(basename "${wheel_fn}" | cut -d '-' -f 1) | |
| echo "${pkg_name}-wheel-name=$(ls "${pkg_name}"-*.whl)" >> $GITHUB_OUTPUT | |
| done | |
| - name: Build and export service | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: extensions/containers/services | |
| tags: gubbins/services:dev | |
| outputs: type=docker,dest=/tmp/gubbins_services_image.tar | |
| build-args: | | |
| EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration | |
| EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl | |
| - name: Build and export client | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: extensions/containers/client | |
| tags: gubbins/client:dev | |
| outputs: type=docker,dest=/tmp/gubbins_client_image.tar | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gubbins-services-img | |
| path: /tmp/gubbins_services_image.tar | |
| pytest-integration: | |
| needs: build-image | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download gubbins-image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gubbins-services-img | |
| path: /tmp | |
| - name: Load image | |
| run: | | |
| docker load --input /tmp/gubbins_services_image.tar | |
| docker image ls -a | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| environment-file: environment.yml | |
| init-shell: bash | |
| post-cleanup: 'all' | |
| - name: Set up environment | |
| run: | | |
| pip install pytest-github-actions-annotate-failures | |
| pip install git+https://github.com/DIRACGrid/DIRAC.git@integration | |
| pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/[testing] ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-cli/[testing] ./extensions/gubbins/gubbins-core/[testing] | |
| - name: Start demo | |
| run: | | |
| git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts | |
| # We have to copy the code to another directory | |
| # and make it a git repository by itself because otherwise the | |
| # root in the pyproject to do not make sense once mounted | |
| # in the containers. | |
| cp -r ./extensions/gubbins /tmp/ | |
| sed -i 's@../..@.@g' /tmp/gubbins/pyproject.toml | |
| sed -i 's@../../@@g' /tmp/gubbins/gubbins-*/pyproject.toml | |
| git init /tmp/gubbins/ | |
| ../diracx-charts/run_demo.sh --enable-open-telemetry --enable-coverage --exit-when-done --set-value developer.autoReload=false --ci-values ../diracx-charts/demo/ci_values.yaml --ci-values ./extensions/gubbins_values.yaml --load-docker-image "gubbins/services:dev" $PWD /tmp/gubbins/ | |
| - name: Debugging information | |
| run: | | |
| DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo | |
| export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf | |
| export PATH=${DIRACX_DEMO_DIR}:$PATH | |
| kubectl get pods | |
| for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do | |
| echo "${pod_name}" | |
| kubectl describe pod/"${pod_name}" || true | |
| for container_name in $(kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do | |
| echo $pod_name $container_name | |
| kubectl logs "${pod_name}" -c "${container_name}" || true | |
| done | |
| done | |
| if [ ! -f "${DIRACX_DEMO_DIR}/.success" ]; then | |
| cat "${DIRACX_DEMO_DIR}/.failed" | |
| exit 1 | |
| fi | |
| - name: Run pytest | |
| run: | | |
| cd extensions/gubbins | |
| export DIRACX_EXTENSIONS=gubbins,diracx | |
| pytest --demo-dir=../../../diracx-charts/ --cov-report=xml:coverage-pytest.xml --junitxml=report.xml | |
| - name: Collect demo coverage | |
| run: | | |
| DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo | |
| export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf | |
| export PATH=${DIRACX_DEMO_DIR}:$PATH | |
| # Shutdown the pods so we collect coverage data | |
| for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do | |
| kubectl delete pod/"${pod_name}" | |
| done | |
| set -x | |
| # Combine the coverage data from the demo and make an XML report | |
| coverage_data=$(mktemp) | |
| sudo chown -R $(id -u) "${DIRACX_DEMO_DIR}"/coverage-reports/ | |
| coverage combine --keep --data-file "${coverage_data}" "${DIRACX_DEMO_DIR}"/coverage-reports/* | |
| # coverage can't handle having multiple src directories, so we need to make a fake one with symlinks | |
| fake_module=$(mktemp -d) | |
| mkdir -p "${fake_module}/src/diracx" | |
| for fn in "${PWD}"/*/src/diracx/*; do | |
| ln -sf "${fn}" "${fake_module}/src/diracx/$(basename "${fn}")" | |
| done | |
| mkdir -p "${fake_module}/src/gubbins" | |
| for fn in "${PWD}"/extensions/gubbins/*/src/gubbins/*; do | |
| ln -sf "${fn}" "${fake_module}/src/gubbins/$(basename "${fn}")" | |
| done | |
| sed -i "s@source =@source =\n ${fake_module}/src@g" .coveragerc | |
| cat .coveragerc | |
| coverage xml -o coverage-demo.xml --data-file "${coverage_data}" | |
| - name: Upload coverage report | |
| uses: codecov/[email protected] | |
| with: | |
| files: ./coverage-pytest.xml,./coverage-demo.xml | |
| client-generation: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| environment-file: environment.yml | |
| init-shell: bash | |
| post-cleanup: 'all' | |
| - name: Set up environment | |
| run: | | |
| micromamba install -c conda-forge nodejs pre-commit | |
| pip install git+https://github.com/DIRACGrid/DIRAC.git@integration | |
| pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/[testing] ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-testing/[testing] -e ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-core/[testing] | |
| npm install -g autorest | |
| - name: Run autorest | |
| run: | | |
| autorest --python --help | |
| $HOME/.autorest/\@autorest_python\@*/node_modules/\@autorest/python/venv/bin/python -m pip install --upgrade setuptools | |
| export DIRACX_EXTENSIONS=gubbins,diracx | |
| pytest --no-cov --regenerate-client extensions/gubbins/gubbins-client/tests/test_regenerate.py |