Introduce extensions: Gubbins

.coveragerc

@@ -2,8 +2,10 @@
 omit =
     tests/*
     */tests/*
-    **/diracx/client/*
+    **/diracx/client/generated/*
     **/diracx/testing/*
+    **/gubbins/testing/*
+    **/gubbins/client/generated/*
 
 [paths]
 source =

.github/workflows/extensions.yml

@@ -0,0 +1,284 @@
+name: Extensions full test
+
+on:
+  push:
+    branches:
+      - main
+      - gubbins
+  pull_request:
+    branches:
+      - main
+
+
+defaults:
+  run:
+    shell: bash -el {0}
+
+jobs:
+  unittest:
+    name: Unit test - ${{ matrix.package }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # In principle, the dependencies could be limited to the extension packages.
+        # However, we want to use the local packages, and not those published on pypi
+        include:
+          - package: "./extensions/gubbins/gubbins-core"
+            dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./diracx-core"
+          - package: "./extensions/gubbins/gubbins-db"
+            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core "
+          - package: "./extensions/gubbins/gubbins-routers"
+            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-db ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core ./diracx-routers"
+          - package: "./extensions/gubbins/gubbins-client"
+            dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-client ./diracx-core "
+          - package: "./extensions/gubbins/gubbins-cli"
+            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-cli ./diracx-client ./diracx-core ./diracx-api"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - uses: mamba-org/setup-micromamba@v2
+        with:
+          # TODO: Use a conda environment file used for the diracx/base container image
+          environment-name: test-env
+          create-args: >-
+            python=3.11
+            m2crypto
+            python-gfal2
+            mypy
+            pip
+          init-shell: bash
+          post-cleanup: 'all'
+      - name: Set up environment
+        run: |
+          pip install pytest-github-actions-annotate-failures
+          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
+          pip install ${{ matrix.dependencies }} ${{ matrix.package }}[types]
+      - name: Run mypy
+        run: |
+          mypy ${{ matrix.package }}/src
+      - name: Run pytest
+        run: |
+          cd ${{ matrix.package }}
+          pip install .[testing]
+          export DIRACX_EXTENSIONS=gubbins,diracx
+          pytest --cov-report=xml:coverage.xml --junitxml=report.xml
+      - name: Upload coverage report
+        uses: codecov/[email protected]
+
+
+  build-wheels:
+    name: Build wheels
+    runs-on: "ubuntu-latest"
+    if: github.event_name != 'push' || github.repository == 'DIRACGrid/diracx'
+    defaults:
+      run:
+        # We need extglob for REFERENCE_BRANCH substitution
+        shell: bash -l -O extglob {0}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Installing dependencies
+        run: |
+          python -m pip install \
+              build \
+              python-dateutil \
+              pytz \
+              readme_renderer[md] \
+              requests \
+              setuptools_scm
+      - name: Build distributions
+        run: |
+          for pkg_dir in $PWD/diracx-*; do
+            echo "Building $pkg_dir"
+            python -m build --outdir $PWD/dist $pkg_dir
+          done
+          # Also build the diracx metapackage
+          python -m build --outdir $PWD/dist .
+          # And build the gubbins package
+          for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do
+            # Skip the testing package
+            if [[ "${pkg_dir}" =~ .*testing.* ]];
+            then
+              echo "Do not build ${pkg_dir}";
+              continue;
+            fi
+            echo "Building $pkg_dir"
+            python -m build --outdir $PWD/dist $pkg_dir
+          done
+      - name: 'Upload Artifact'
+        uses: actions/upload-artifact@v4
+        with:
+          name: gubbins-whl
+          path: dist/*.whl
+          retention-days: 5
+
+  # Build to docker image with the code in it
+  build-image:
+    needs: build-wheels
+    timeout-minutes:  30
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Download gubbins wheels
+        uses: actions/download-artifact@v4
+        with:
+            name: gubbins-whl
+      - name: "Find wheels"
+        id: find_wheel
+        run: |
+          # We need to copy them there to be able to access them in the RUN --mount
+          cp diracx*.whl gubbins*.whl extensions/containers/services/
+          for wheel_fn in *.whl; do
+            pkg_name=$(basename "${wheel_fn}" | cut -d '-' -f 1)
+            echo "${pkg_name}-wheel-name=$(ls "${pkg_name}"-*.whl)" >> $GITHUB_OUTPUT
+          done
+      - name: Build and export service
+        uses: docker/build-push-action@v6
+        with:
+          context: extensions/containers/services
+          tags: gubbins/services:dev
+          outputs: type=docker,dest=/tmp/gubbins_services_image.tar
+          build-args: |
+            EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration
+            EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl
+      - name: Build and export client
+        uses: docker/build-push-action@v6
+        with:
+          context: extensions/containers/client
+          tags: gubbins/client:dev
+          outputs: type=docker,dest=/tmp/gubbins_client_image.tar
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: gubbins-services-img
+          path: /tmp/gubbins_services_image.tar
+
+
+  pytest-integration:
+    needs: build-image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download gubbins-image
+        uses: actions/download-artifact@v4
+        with:
+          name: gubbins-services-img
+          path: /tmp
+      - name: Load image
+        run: |
+          docker load --input /tmp/gubbins_services_image.tar
+          docker image ls -a
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: environment.yml
+          init-shell: bash
+          post-cleanup: 'all'
+      - name: Set up environment
+        run: |
+          pip install pytest-github-actions-annotate-failures
+          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
+          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/[testing]  ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-cli/[testing] ./extensions/gubbins/gubbins-core/[testing]
+      - name: Start demo
+        run: |
+          git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts
+          # We have to copy the code to another directory
+          # and make it a git repository by itself because otherwise the
+          # root in the pyproject to do not make sense once mounted
+          # in the containers.
+          cp -r ./extensions/gubbins /tmp/
+          sed -i 's@../..@.@g' /tmp/gubbins/pyproject.toml
+          sed -i 's@../../@@g' /tmp/gubbins/gubbins-*/pyproject.toml
+          git init /tmp/gubbins/
+          ../diracx-charts/run_demo.sh --enable-open-telemetry --enable-coverage --exit-when-done --set-value developer.autoReload=false --ci-values ../diracx-charts/demo/ci_values.yaml --ci-values ./extensions/gubbins_values.yaml --load-docker-image "gubbins/services:dev" $PWD /tmp/gubbins/
+      - name: Debugging information
+        run: |
+          DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
+          export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf
+          export PATH=${DIRACX_DEMO_DIR}:$PATH
+          kubectl get pods
+          for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
+            echo "${pod_name}"
+            kubectl describe pod/"${pod_name}" || true
+            for container_name in $(kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do
+              echo $pod_name $container_name
+              kubectl logs "${pod_name}" -c "${container_name}" || true
+            done
+          done
+          if [ ! -f "${DIRACX_DEMO_DIR}/.success" ]; then
+            cat "${DIRACX_DEMO_DIR}/.failed"
+            exit 1
+          fi
+      - name: Run pytest
+        run: |
+          cd extensions/gubbins
+          export DIRACX_EXTENSIONS=gubbins,diracx
+          pytest --demo-dir=../../../diracx-charts/ --cov-report=xml:coverage-pytest.xml --junitxml=report.xml
+      - name: Collect demo coverage
+        run: |
+          DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
+          export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf
+          export PATH=${DIRACX_DEMO_DIR}:$PATH
+          # Shutdown the pods so we collect coverage data
+          for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
+            kubectl delete pod/"${pod_name}"
+          done
+          set -x
+          # Combine the coverage data from the demo and make an XML report
+          coverage_data=$(mktemp)
+          sudo chown -R $(id -u) "${DIRACX_DEMO_DIR}"/coverage-reports/
+          coverage combine --keep --data-file "${coverage_data}" "${DIRACX_DEMO_DIR}"/coverage-reports/*
+
+          # coverage can't handle having multiple src directories, so we need to make a fake one with symlinks
+          fake_module=$(mktemp -d)
+
+          mkdir -p "${fake_module}/src/diracx"
+          for fn in "${PWD}"/*/src/diracx/*; do
+            ln -sf "${fn}" "${fake_module}/src/diracx/$(basename "${fn}")"
+          done
+
+
+          mkdir -p "${fake_module}/src/gubbins"
+          for fn in "${PWD}"/extensions/gubbins/*/src/gubbins/*; do
+            ln -sf "${fn}" "${fake_module}/src/gubbins/$(basename "${fn}")"
+          done
+
+          sed -i "s@source =@source =\n    ${fake_module}/src@g" .coveragerc
+
+          cat .coveragerc
+
+          coverage xml -o coverage-demo.xml --data-file "${coverage_data}"
+      - name: Upload coverage report
+        uses: codecov/[email protected]
+        with:
+          files: ./coverage-pytest.xml,./coverage-demo.xml
+
+  client-generation:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: environment.yml
+          init-shell: bash
+          post-cleanup: 'all'
+      - name: Set up environment
+        run: |
+          micromamba install -c conda-forge nodejs pre-commit
+          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
+          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/[testing]  ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-testing/[testing] -e ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-core/[testing]
+          npm install -g autorest
+      - name: Run autorest
+        run: |
+          autorest --python --help
+          $HOME/.autorest/\@autorest_python\@*/node_modules/\@autorest/python/venv/bin/python -m pip install --upgrade setuptools
+          export DIRACX_EXTENSIONS=gubbins,diracx
+          pytest --no-cov --regenerate-client extensions/gubbins/gubbins-client/tests/test_regenerate.py

.github/workflows/main.yml

@@ -29,7 +29,8 @@ jobs:
       run: |
         find -name '*.sh' -print0 | xargs -0 -n1 shellcheck --exclude=SC1090,SC1091 --external-source
 
-  pytest:
+
+  unittest:
     name: Unit test - ${{ matrix.package }}
     runs-on: ubuntu-latest
     strategy:
@@ -61,6 +62,9 @@ jobs:
       - name: Set up environment
         run: |
           pip install pytest-github-actions-annotate-failures
+          # Note: DIRAC will install pretty much everything
+          # from diracx so installing just the dependency may
+          # be a bit useless
           pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
           pip install ${{ matrix.dependencies }}
       - name: Run pytest

.gitignore

@@ -21,6 +21,7 @@ parts
 bin
 develop-eggs
 .installed.cfg
+*.whl
 
 # Translations
 *.mo

.pre-commit-config.yaml

@@ -35,4 +35,4 @@ repos:
           - types-requests
           - types-aiobotocore[essential]
           - boto3-stubs[essential]
-        exclude: ^(diracx-client/src/diracx/client/|diracx-[a-z]+/tests/|diracx-testing/|build)
+        exclude: ^(diracx-client/src/diracx/client/generated|diracx-[a-z]+/tests/|diracx-testing/|build|extensions/gubbins/gubbins-client/src/gubbins/client/generated)

codecov.yml

@@ -12,6 +12,10 @@ coverage:
 
 codecov:
   notify:
-    after_n_builds: 7
+    # 6 diracx unit tests
+    # 1 diracx integration test
+    # 5 gubbins unit test
+    # 1 gubbins integration test
+    after_n_builds: 13
 
 comment: false

diracx-cli/pyproject.toml

@@ -36,6 +36,15 @@ types = [
 [project.scripts]
 dirac = "diracx.cli:app"
 
+[project.entry-points."diracx.cli"]
+jobs = "diracx.cli.jobs:app"
+config = "diracx.cli.config:app"
+
+[project.entry-points."diracx.cli.hidden"]
+internal = "diracx.cli.internal:app"
+
+
+
 [tool.setuptools.packages.find]
 where = ["src"]
 
@@ -58,3 +67,4 @@ asyncio_mode = "auto"
 markers = [
     "enabled_dependencies: List of dependencies which should be available to the FastAPI test client",
 ]
+asyncio_default_fixture_loop_scope = "session"