Split diracx.db into diracx.db.sql and diracx.db.os

setup.cfg

@@ -74,12 +74,14 @@ console_scripts =
     dirac = diracx.cli:app
 diracx =
 	properties_module = diracx.core:properties
-diracx.dbs =
+diracx.sql.db =
 	AuthDB = diracx.db:AuthDB
 	JobDB = diracx.db:JobDB
 	JobLoggingDB = diracx.db:JobLoggingDB
 	SandboxMetadataDB = diracx.db:SandboxMetadataDB
 	#DummyDB = diracx.db:DummyDB
+diracx.os.db =
+	JobParametersDB = diracx.db:JobParametersDB
 diracx.services =
 	jobs = diracx.routers.job_manager:router
 	config = diracx.routers.configuration:router

src/diracx/db/__init__.py

@@ -1,7 +1,7 @@
 __all__ = ("AuthDB", "JobDB", "JobLoggingDB", "SandboxMetadataDB")
 
-from .auth.db import AuthDB
-from .jobs.db import JobDB, JobLoggingDB
-from .sandbox_metadata.db import SandboxMetadataDB
+from .sql.auth.db import AuthDB
+from .sql.jobs.db import JobDB, JobLoggingDB
+from .sql.sandbox_metadata.db import SandboxMetadataDB
 
 # from .dummy.db import DummyDB

src/diracx/db/__main__.py

@@ -28,7 +28,7 @@ def parse_args():
 
 async def init_sql():
     logger.info("Initialising SQL databases")
-    from diracx.db.sql_utils import BaseSQLDB
+    from diracx.db.sql.utils import BaseSQLDB
 
     for db_name, db_url in BaseSQLDB.available_urls().items():
         logger.info("Initialising %s", db_name)
@@ -40,7 +40,7 @@ async def init_sql():
 
 async def init_os():
     logger.info("Initialising OpenSearch databases")
-    from diracx.db.os_utils import BaseOSDB
+    from diracx.db.os.utils import BaseOSDB
 
     for db_name, db_url in BaseOSDB.available_urls().items():
         logger.info("Initialising %s", db_name)

src/diracx/db/os/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+__all__ = ("JobParametersDB",)
+
+from .job_parameters import JobParametersDB

src/diracx/db/jobs/job_parameters.py → src/diracx/db/os/job_parameters.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from diracx.db.os_utils import BaseOSDB
+from diracx.db.os.utils import BaseOSDB
 
 
 class JobParametersDB(BaseOSDB):

src/diracx/db/auth/db.py → src/diracx/db/sql/auth/db.py

@@ -12,8 +12,8 @@
     ExpiredFlowError,
     PendingAuthorizationError,
 )
+from diracx.db.sql.utils import BaseSQLDB, substract_date
 
-from ..sql_utils import BaseSQLDB, substract_date
 from .schema import (
     AuthorizationFlows,
     DeviceFlows,

src/diracx/db/auth/schema.py → src/diracx/db/sql/auth/schema.py

@@ -7,7 +7,7 @@
 )
 from sqlalchemy.orm import declarative_base
 
-from ..sql_utils import Column, DateNowColumn, EnumColumn, NullColumn
+from diracx.db.sql.utils import Column, DateNowColumn, EnumColumn, NullColumn
 
 USER_CODE_LENGTH = 8
 

src/diracx/db/dummy/db.py → src/diracx/db/sql/dummy/db.py

@@ -4,7 +4,8 @@
 
 from sqlalchemy import func, insert, select
 
-from ..sql_utils import BaseSQLDB, apply_search_filters
+from diracx.db.sql.utils import BaseSQLDB, apply_search_filters
+
 from .schema import Base as DummyDBBase
 from .schema import Cars, Owners
 

src/diracx/db/dummy/schema.py → src/diracx/db/sql/dummy/schema.py

@@ -3,7 +3,7 @@
 
 # The utils class define some boilerplate types that should be used
 # in place of the SQLAlchemy one. Have a look at them
-from ..sql_utils import Column, DateNowColumn
+from diracx.db.sql.utils import Column, DateNowColumn
 
 Base = declarative_base()
 

src/diracx/db/jobs/db.py → src/diracx/db/sql/jobs/db.py

@@ -11,7 +11,7 @@
 from diracx.core.models import JobStatusReturn, LimitedJobStatusReturn
 from diracx.core.utils import JobStatus
 
-from ..sql_utils import BaseSQLDB, apply_search_filters
+from ..utils import BaseSQLDB, apply_search_filters
 from .schema import (
     InputData,
     JobDBBase,

src/diracx/db/jobs/schema.py → src/diracx/db/sql/jobs/schema.py

@@ -13,7 +13,7 @@
 )
 from sqlalchemy.orm import declarative_base
 
-from ..sql_utils import Column, DateNowColumn, NullColumn
+from ..utils import Column, DateNowColumn, NullColumn
 
 JobDBBase = declarative_base()
 JobLoggingDBBase = declarative_base()

src/diracx/db/jobs/status_utility.py → src/diracx/db/sql/jobs/status_utility.py

@@ -9,7 +9,7 @@
     SetJobStatusReturn,
 )
 from diracx.core.utils import JobStatus
-from diracx.db.jobs.db import JobDB, JobLoggingDB
+from diracx.db.sql.jobs.db import JobDB, JobLoggingDB
 
 
 async def set_job_status(

src/diracx/db/sandbox_metadata/db.py → src/diracx/db/sql/sandbox_metadata/db.py

@@ -7,7 +7,8 @@
 
 import sqlalchemy
 
-from ..sql_utils import BaseSQLDB
+from diracx.db.sql.utils import BaseSQLDB
+
 from .schema import Base as SandboxMetadataDBBase
 from .schema import sb_Owners, sb_SandBoxes
 

src/diracx/db/sandbox_metadata/schema.py → src/diracx/db/sql/sandbox_metadata/schema.py

@@ -9,7 +9,7 @@
 )
 from sqlalchemy.orm import declarative_base
 
-from ..sql_utils import Column, DateNowColumn
+from diracx.db.sql.utils import Column, DateNowColumn
 
 Base = declarative_base()
 

src/diracx/routers/__init__.py

@@ -17,8 +17,8 @@
 from diracx.core.exceptions import DiracError, DiracHttpResponse
 from diracx.core.extensions import select_from_extension
 from diracx.core.utils import dotenv_files_from_environment
-from diracx.db.os_utils import BaseOSDB
-from diracx.db.sql_utils import BaseSQLDB
+from diracx.db.os.utils import BaseOSDB
+from diracx.db.sql.utils import BaseSQLDB
 
 from ..core.settings import ServiceSettingsBase
 from .auth import verify_dirac_access_token

src/diracx/routers/auth.py

@@ -39,7 +39,7 @@
     UnevaluatedProperty,
 )
 from diracx.core.settings import ServiceSettingsBase, TokenSigningKey
-from diracx.db.auth.schema import FlowStatus, RefreshTokenStatus
+from diracx.db.sql.auth.schema import FlowStatus, RefreshTokenStatus
 
 from .dependencies import (
     AuthDB,

src/diracx/routers/job_manager/__init__.py

@@ -22,7 +22,7 @@
 )
 from diracx.core.properties import JOB_ADMINISTRATOR, NORMAL_USER
 from diracx.core.utils import JobStatus
-from diracx.db.jobs.status_utility import (
+from diracx.db.sql.jobs.status_utility import (
     set_job_status,
 )
 

tests/conftest.py

@@ -91,7 +91,7 @@ def with_app(test_auth_settings, with_config_repo):
     @contextlib.asynccontextmanager
     async def create_db_schemas(app=app):
         """Create DB schema's based on the DBs available in app.dependency_overrides"""
-        from diracx.db.sql_utils import BaseSQLDB
+        from diracx.db.sql.utils import BaseSQLDB
 
         for k, v in app.dependency_overrides.items():
             # Ignore dependency overrides which aren't BaseSQLDB.transaction

tests/db/auth/test_authorization_flow.py

@@ -4,7 +4,7 @@
 from sqlalchemy.exc import NoResultFound
 
 from diracx.core.exceptions import AuthorizationError
-from diracx.db.auth.db import AuthDB
+from diracx.db.sql.auth.db import AuthDB
 
 MAX_VALIDITY = 2
 EXPIRED = 0

tests/db/auth/test_device_flow.py

@@ -6,8 +6,8 @@
 from sqlalchemy.exc import NoResultFound
 
 from diracx.core.exceptions import AuthorizationError, ExpiredFlowError
-from diracx.db.auth.db import AuthDB
-from diracx.db.auth.schema import USER_CODE_LENGTH
+from diracx.db.sql.auth.db import AuthDB
+from diracx.db.sql.auth.schema import USER_CODE_LENGTH
 
 MAX_VALIDITY = 2
 EXPIRED = 0

tests/db/auth/test_refresh_token.py

@@ -2,8 +2,8 @@
 
 import pytest
 
-from diracx.db.auth.db import AuthDB
-from diracx.db.auth.schema import RefreshTokenStatus
+from diracx.db.sql.auth.db import AuthDB
+from diracx.db.sql.auth.schema import RefreshTokenStatus
 
 
 @pytest.fixture

tests/db/jobs/test_jobDB.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from diracx.db.jobs.db import JobDB
+from diracx.db.sql.jobs.db import JobDB
 
 
 @pytest.fixture

tests/db/test_dummyDB.py

@@ -6,7 +6,7 @@
 import pytest
 
 from diracx.core.exceptions import InvalidQueryError
-from diracx.db.dummy.db import DummyDB
+from diracx.db.sql.dummy.db import DummyDB
 
 # Each DB test class must defined a fixture looking like this one
 # It allows to get an instance of an in memory DB,

tests/db/test_sandboxMetadataDB.py

@@ -3,7 +3,7 @@
 import pytest
 import sqlalchemy
 
-from diracx.db.sandbox_metadata.db import SandboxMetadataDB
+from diracx.db.sql.sandbox_metadata.db import SandboxMetadataDB
 
 
 @pytest.fixture