mydiracx extension extensions/src/mydiracx

.github/workflows/extensions.yml

@@ -0,0 +1,72 @@
+name: Extensions Basic Tests
+
+on:
+  push:
+    branches:
+      - main
+      - mydiracx-extension
+  pull_request:
+    branches:
+      - main
+
+
+defaults:
+  run:
+    shell: bash -el {0}
+
+jobs:
+  pytest:
+    name: Unit test - ${{ matrix.package }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - package: "./diracx-db"
+            dependencies: "./diracx-testing ./diracx-core"
+          - package: "./diracx-routers"
+            dependencies: "./diracx-testing ./diracx-core ./diracx-db"
+          - package: "./diracx-client"
+            dependencies: "./diracx-testing ./diracx-core"
+          - package: "./diracx-api"
+            dependencies: "./diracx-testing ./diracx-core ./diracx-client"
+          - package: "./diracx-cli"
+            dependencies: "./diracx-testing ./diracx-core ./diracx-client ./diracx-api"
+          - package: "./extensions/mydiracx/mydiracx-db"
+            dependencies: "./diracx-testing ./diracx-core ./diracx-db"
+          - package: "./extensions/mydiracx/mydiracx-routers"
+            dependencies: "./diracx-testing ./diracx-core ./diracx-db ./diracx-routers ./extensions/mydiracx/mydiracx-db"
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+          # TODO: Use a conda environment file used for the diracx/base container image
+          environment-name: test-env
+          create-args: >-
+            python=3.11
+            m2crypto
+            python-gfal2
+            mypy
+            pip
+          init-shell: bash
+          post-cleanup: 'all'
+      - name: Set up environment
+        run: |
+          pip install pytest-github-actions-annotate-failures
+          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
+          pip install 'git+https://github.com/ruslan33/diracx.git@mydiracx-extension#egg=mydiracx-db&subdirectory=extensions/mydiracx/mydiracx-db'
+          pip install 'git+https://github.com/ruslan33/diracx.git@mydiracx-extension#egg=mydiracx-routers&subdirectory=extensions/mydiracx/mydiracx-routers'
+          pip install 'git+https://github.com/ruslan33/diracx.git@mydiracx-extension#egg=mydiracx&subdirectory=extensions/mydiracx'
+          pip install ${{ matrix.dependencies }} ${{ matrix.package }}[types]
+      - name: Run mypy
+        run: |
+          mypy ${{ matrix.package }}/src
+      - name: Run pytest
+        run: |
+          cd ${{ matrix.package }}
+          pip install .[testing]
+          pytest --cov-report=xml:coverage.xml --junitxml=report.xml
+      - name: Upload coverage report
+        uses: codecov/[email protected]

extensions/docker/Dockerfile

@@ -0,0 +1,10 @@
+FROM ghcr.io/diracgrid/diracx/services:dev
+
+#Extension
+ARG MAMBA_DOCKERFILE_ACTIVATE=1
+ENV DIRACX_EXTENSIONS "mydiracx,diracx"
+ENV DIRACX_DB_URL_DUMMYDB "sqlite+aiosqlite:///:memory:"
+
+RUN pip install 'git+https://github.com/ruslan33/diracx.git@mydiracx-extension#egg=mydiracx-db&subdirectory=extensions/mydiracx/mydiracx-db'
+RUN pip install 'git+https://github.com/ruslan33/diracx.git@mydiracx-extension#egg=mydiracx-routers&subdirectory=extensions/mydiracx/mydiracx-routers'
+RUN pip install 'git+https://github.com/ruslan33/diracx.git@mydiracx-extension#egg=mydiracx&subdirectory=extensions/mydiracx'

extensions/mydiracx/mydiracx-db/pyproject.toml

@@ -0,0 +1,54 @@
+[project]
+name = "mydiracx-db"
+description = "TODO"
+readme = "README.md"
+requires-python = ">=3.10"
+keywords = []
+license = {text = "GPL-3.0-only"}
+classifiers = [
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering",
+    "Topic :: System :: Distributed Computing",
+]
+dependencies = [
+    "dirac",
+    "diracx-core",
+    "fastapi",
+    "opensearch-py[async]",
+    "pydantic",
+    "sqlalchemy[aiomysql,aiosqlite]",
+]
+dynamic = ["version"]
+
+[project.optional-dependencies]
+testing = [
+    "diracx-testing",
+]
+
+[project.entry-points."diracx.db.sql"]
+DummyDB = "mydiracx.db.sql:DummyDB"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[build-system]
+requires = ["setuptools>=61", "wheel", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+root = "../../.."
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = [
+    "-v",
+    "--cov=diracx.db", "--cov-report=term-missing",
+    "-pdiracx.testing", "-pdiracx.testing.osdb",
+    "--import-mode=importlib",
+]
+asyncio_mode = "auto"
+markers = [
+    "enabled_dependencies: List of dependencies which should be available to the FastAPI test client",
+]

extensions/mydiracx/mydiracx-db/src/mydiracx/db/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+__all__ = ("sql",)
+
+from . import sql

extensions/mydiracx/mydiracx-db/src/mydiracx/db/sql/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+__all__ = ("DummyDB",)
+
+from .dummy.db import DummyDB

extensions/mydiracx/mydiracx-db/src/mydiracx/db/sql/dummy/db.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from uuid import UUID
+
+from diracx.db.sql.utils import BaseSQLDB, apply_search_filters
+from sqlalchemy import func, insert, select
+
+from .schema import Base as DummyDBBase
+from .schema import Cars, Owners
+
+
+class DummyDB(BaseSQLDB):
+    """
+    This DummyDB is just to illustrate some important aspect of writing
+    DB classes in DiracX.
+
+    It is mostly pure SQLAlchemy, with a few convention
+
+    Document the secrets
+    """
+
+    # This needs to be here for the BaseSQLDB to create the engine
+    metadata = DummyDBBase.metadata
+
+    async def summary(self, group_by, search) -> list[dict[str, str | int]]:
+        columns = [Cars.__table__.columns[x] for x in group_by]
+
+        stmt = select(*columns, func.count(Cars.licensePlate).label("count"))
+        stmt = apply_search_filters(Cars.__table__, stmt, search)
+        stmt = stmt.group_by(*columns)
+
+        # Execute the query
+        return [
+            dict(row._mapping)
+            async for row in (await self.conn.stream(stmt))
+            if row.count > 0  # type: ignore
+        ]
+
+    async def insert_owner(self, name: str) -> int:
+        stmt = insert(Owners).values(name=name)
+        result = await self.conn.execute(stmt)
+        # await self.engine.commit()
+        return result.lastrowid
+
+    async def get_owner(self) -> list[str]:
+        stmt = select(Owners.name)
+        result = await self.conn.execute(stmt)
+        # await self.engine.commit()
+        return [row[0] for row in result]
+
+    async def insert_car(self, license_plate: UUID, model: str, owner_id: int) -> int:
+        stmt = insert(Cars).values(
+            licensePlate=license_plate, model=model, ownerID=owner_id
+        )
+
+        result = await self.conn.execute(stmt)
+        # await self.engine.commit()
+        return result.lastrowid
+
+    async def get_car(self) -> list[str]:
+        stmt = select(Cars.model)
+        result = await self.conn.execute(stmt)
+        # await self.engine.commit()
+        return [row[0] for row in result]

extensions/mydiracx/mydiracx-db/src/mydiracx/db/sql/dummy/schema.py

@@ -0,0 +1,21 @@
+# The utils class define some boilerplate types that should be used
+# in place of the SQLAlchemy one. Have a look at them
+from diracx.db.sql.utils import Column, DateNowColumn
+from sqlalchemy import ForeignKey, Integer, String, Uuid
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()
+
+
+class Owners(Base):
+    __tablename__ = "Owners"
+    ownerID = Column(Integer, primary_key=True, autoincrement=True)
+    creation_time = DateNowColumn()
+    name = Column(String(255))
+
+
+class Cars(Base):
+    __tablename__ = "Cars"
+    licensePlate = Column(Uuid(), primary_key=True)
+    model = Column(String(255))
+    ownerID = Column(Integer, ForeignKey(Owners.ownerID))

extensions/mydiracx/mydiracx-db/tests/test_dummyDB.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import asyncio
+from uuid import uuid4
+
+import pytest
+from diracx.core.exceptions import InvalidQueryError
+from diracx.db.sql.utils import SQLDBUnavailable
+
+from mydiracx.db.sql.dummy.db import DummyDB
+
+# Each DB test class must defined a fixture looking like this one
+# It allows to get an instance of an in memory DB,
+
+
+@pytest.fixture
+async def dummy_db(tmp_path) -> DummyDB:
+    dummy_db = DummyDB("sqlite+aiosqlite:///:memory:")
+    async with dummy_db.engine_context():
+        async with dummy_db.engine.begin() as conn:
+            await conn.run_sync(dummy_db.metadata.create_all)
+        yield dummy_db
+
+
+async def test_insert_and_summary(dummy_db: DummyDB):
+    # Each context manager creates a transaction
+    # So it is important to write test this way
+    async with dummy_db as dummy_db:
+        # First we check that the DB is empty
+        result = await dummy_db.summary(["model"], [])
+        assert not result
+
+    # Now we add some data in the DB
+    async with dummy_db as dummy_db:
+        # Add a car owner
+        owner_id = await dummy_db.insert_owner(name="Magnum")
+        assert owner_id
+
+        # Add cars, belonging to the same guy
+        result = await asyncio.gather(
+            *(dummy_db.insert_car(uuid4(), f"model_{i}", owner_id) for i in range(10))
+        )
+        assert result
+
+    # Check that there are now 10 cars assigned to a single driver
+    async with dummy_db as dummy_db:
+        result = await dummy_db.summary(["ownerID"], [])
+
+        assert result[0]["count"] == 10
+
+    # Test the selection
+    async with dummy_db as dummy_db:
+        result = await dummy_db.summary(
+            ["ownerID"], [{"parameter": "model", "operator": "eq", "value": "model_1"}]
+        )
+
+        assert result[0]["count"] == 1
+
+    async with dummy_db as dummy_db:
+        with pytest.raises(InvalidQueryError):
+            result = await dummy_db.summary(
+                ["ownerID"],
+                [
+                    {
+                        "parameter": "model",
+                        "operator": "BADSELECTION",
+                        "value": "model_1",
+                    }
+                ],
+            )
+
+
+async def test_bad_connection():
+    dummy_db = DummyDB("mysql+aiomysql://tata:[email protected]:3306/name")
+    async with dummy_db.engine_context():
+        with pytest.raises(SQLDBUnavailable):
+            async with dummy_db:
+                dummy_db.ping()

extensions/mydiracx/mydiracx-routers/pyproject.toml

@@ -0,0 +1,72 @@
+[project]
+name = "mydiracx-routers"
+description = "TODO"
+readme = "README.md"
+requires-python = ">=3.10"
+keywords = []
+license = {text = "GPL-3.0-only"}
+classifiers = [
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering",
+    "Topic :: System :: Distributed Computing",
+]
+dependencies = [
+    "aiobotocore",
+    "authlib",
+    "botocore",
+    "cachetools",
+    "dirac",
+    "diracx-core",
+    "diracx-db",
+    "python-dotenv",  # TODO: We might not need this
+    "python-multipart",
+    "fastapi",
+    "httpx",
+    "pydantic",
+    "sqlalchemy",
+]
+dynamic = ["version"]
+
+[project.optional-dependencies]
+testing = [
+    "diracx-testing",
+    "moto[server]",
+    "pytest-httpx",
+]
+types = [
+    "boto3-stubs",
+    "types-aiobotocore[essential]",
+    "types-aiobotocore-s3",
+    "types-cachetools",
+    "types-python-dateutil",
+    "types-PyYAML",
+    "types-requests",
+]
+
+[project.entry-points."diracx.services"]
+mymanager = "mydiracx.routers.mymanager:router"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[build-system]
+requires = ["setuptools>=61", "wheel", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+root = "../../.."
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = [
+    "-v",
+    "--cov=diracx.routers", "--cov-report=term-missing",
+    "-pdiracx.testing", "-pdiracx.testing.osdb",
+    "--import-mode=importlib",
+]
+asyncio_mode = "auto"
+markers = [
+    "enabled_dependencies: List of dependencies which should be available to the FastAPI test client",
+]