diff --git a/setup.cfg b/setup.cfg index 45eb2a1ec..c5c3488dc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -74,12 +74,14 @@ console_scripts = dirac = diracx.cli:app diracx = properties_module = diracx.core:properties -diracx.dbs = +diracx.sql.db = AuthDB = diracx.db:AuthDB JobDB = diracx.db:JobDB JobLoggingDB = diracx.db:JobLoggingDB SandboxMetadataDB = diracx.db:SandboxMetadataDB #DummyDB = diracx.db:DummyDB +diracx.os.db = + JobParametersDB = diracx.db:JobParametersDB diracx.services = jobs = diracx.routers.job_manager:router config = diracx.routers.configuration:router diff --git a/src/diracx/db/__init__.py b/src/diracx/db/__init__.py index 3dd13c3c9..74c97c4b7 100644 --- a/src/diracx/db/__init__.py +++ b/src/diracx/db/__init__.py @@ -1,7 +1,7 @@ __all__ = ("AuthDB", "JobDB", "JobLoggingDB", "SandboxMetadataDB") -from .auth.db import AuthDB -from .jobs.db import JobDB, JobLoggingDB -from .sandbox_metadata.db import SandboxMetadataDB +from .sql.auth.db import AuthDB +from .sql.jobs.db import JobDB, JobLoggingDB +from .sql.sandbox_metadata.db import SandboxMetadataDB # from .dummy.db import DummyDB diff --git a/src/diracx/db/__main__.py b/src/diracx/db/__main__.py index 19506e6e9..abd7f3e0f 100644 --- a/src/diracx/db/__main__.py +++ b/src/diracx/db/__main__.py @@ -28,7 +28,7 @@ def parse_args(): async def init_sql(): logger.info("Initialising SQL databases") - from diracx.db.sql_utils import BaseSQLDB + from diracx.db.sql.utils import BaseSQLDB for db_name, db_url in BaseSQLDB.available_urls().items(): logger.info("Initialising %s", db_name) @@ -40,7 +40,7 @@ async def init_sql(): async def init_os(): logger.info("Initialising OpenSearch databases") - from diracx.db.os_utils import BaseOSDB + from diracx.db.os.utils import BaseOSDB for db_name, db_url in BaseOSDB.available_urls().items(): logger.info("Initialising %s", db_name) diff --git a/src/diracx/db/os/__init__.py b/src/diracx/db/os/__init__.py new file mode 100644 index 000000000..535e2a954 --- /dev/null +++ b/src/diracx/db/os/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +__all__ = ("JobParametersDB",) + +from .job_parameters import JobParametersDB diff --git a/src/diracx/db/jobs/job_parameters.py b/src/diracx/db/os/job_parameters.py similarity index 95% rename from src/diracx/db/jobs/job_parameters.py rename to src/diracx/db/os/job_parameters.py index d29bc9550..0e1e888d2 100644 --- a/src/diracx/db/jobs/job_parameters.py +++ b/src/diracx/db/os/job_parameters.py @@ -1,6 +1,6 @@ from __future__ import annotations -from diracx.db.os_utils import BaseOSDB +from diracx.db.os.utils import BaseOSDB class JobParametersDB(BaseOSDB): diff --git a/src/diracx/db/os_utils.py b/src/diracx/db/os/utils.py similarity index 100% rename from src/diracx/db/os_utils.py rename to src/diracx/db/os/utils.py diff --git a/src/diracx/db/auth/__init__.py b/src/diracx/db/sql/__init__.py similarity index 100% rename from src/diracx/db/auth/__init__.py rename to src/diracx/db/sql/__init__.py diff --git a/src/diracx/db/dummy/__init__.py b/src/diracx/db/sql/auth/__init__.py similarity index 100% rename from src/diracx/db/dummy/__init__.py rename to src/diracx/db/sql/auth/__init__.py diff --git a/src/diracx/db/auth/db.py b/src/diracx/db/sql/auth/db.py similarity index 99% rename from src/diracx/db/auth/db.py rename to src/diracx/db/sql/auth/db.py index c4993a5e8..fcdb82124 100644 --- a/src/diracx/db/auth/db.py +++ b/src/diracx/db/sql/auth/db.py @@ -12,8 +12,8 @@ ExpiredFlowError, PendingAuthorizationError, ) +from diracx.db.sql.utils import BaseSQLDB, substract_date -from ..sql_utils import BaseSQLDB, substract_date from .schema import ( AuthorizationFlows, DeviceFlows, diff --git a/src/diracx/db/auth/schema.py b/src/diracx/db/sql/auth/schema.py similarity index 97% rename from src/diracx/db/auth/schema.py rename to src/diracx/db/sql/auth/schema.py index ccd53aa0e..c85822ed0 100644 --- a/src/diracx/db/auth/schema.py +++ b/src/diracx/db/sql/auth/schema.py @@ -7,7 +7,7 @@ ) from sqlalchemy.orm import declarative_base -from ..sql_utils import Column, DateNowColumn, EnumColumn, NullColumn +from diracx.db.sql.utils import Column, DateNowColumn, EnumColumn, NullColumn USER_CODE_LENGTH = 8 diff --git a/src/diracx/db/jobs/__init__.py b/src/diracx/db/sql/dummy/__init__.py similarity index 100% rename from src/diracx/db/jobs/__init__.py rename to src/diracx/db/sql/dummy/__init__.py diff --git a/src/diracx/db/dummy/db.py b/src/diracx/db/sql/dummy/db.py similarity index 96% rename from src/diracx/db/dummy/db.py rename to src/diracx/db/sql/dummy/db.py index a791bf62e..aaed634f6 100644 --- a/src/diracx/db/dummy/db.py +++ b/src/diracx/db/sql/dummy/db.py @@ -4,7 +4,8 @@ from sqlalchemy import func, insert, select -from ..sql_utils import BaseSQLDB, apply_search_filters +from diracx.db.sql.utils import BaseSQLDB, apply_search_filters + from .schema import Base as DummyDBBase from .schema import Cars, Owners diff --git a/src/diracx/db/dummy/schema.py b/src/diracx/db/sql/dummy/schema.py similarity index 92% rename from src/diracx/db/dummy/schema.py rename to src/diracx/db/sql/dummy/schema.py index bbd953a1c..d81e0a8f0 100644 --- a/src/diracx/db/dummy/schema.py +++ b/src/diracx/db/sql/dummy/schema.py @@ -3,7 +3,7 @@ # The utils class define some boilerplate types that should be used # in place of the SQLAlchemy one. Have a look at them -from ..sql_utils import Column, DateNowColumn +from diracx.db.sql.utils import Column, DateNowColumn Base = declarative_base() diff --git a/src/diracx/db/sandbox_metadata/__init__.py b/src/diracx/db/sql/jobs/__init__.py similarity index 100% rename from src/diracx/db/sandbox_metadata/__init__.py rename to src/diracx/db/sql/jobs/__init__.py diff --git a/src/diracx/db/jobs/db.py b/src/diracx/db/sql/jobs/db.py similarity index 99% rename from src/diracx/db/jobs/db.py rename to src/diracx/db/sql/jobs/db.py index ee520b6d5..7c060448b 100644 --- a/src/diracx/db/jobs/db.py +++ b/src/diracx/db/sql/jobs/db.py @@ -11,7 +11,7 @@ from diracx.core.models import JobStatusReturn, LimitedJobStatusReturn from diracx.core.utils import JobStatus -from ..sql_utils import BaseSQLDB, apply_search_filters +from ..utils import BaseSQLDB, apply_search_filters from .schema import ( InputData, JobDBBase, diff --git a/src/diracx/db/jobs/schema.py b/src/diracx/db/sql/jobs/schema.py similarity index 99% rename from src/diracx/db/jobs/schema.py rename to src/diracx/db/sql/jobs/schema.py index d187421bb..fe76eedd2 100644 --- a/src/diracx/db/jobs/schema.py +++ b/src/diracx/db/sql/jobs/schema.py @@ -13,7 +13,7 @@ ) from sqlalchemy.orm import declarative_base -from ..sql_utils import Column, DateNowColumn, NullColumn +from ..utils import Column, DateNowColumn, NullColumn JobDBBase = declarative_base() JobLoggingDBBase = declarative_base() diff --git a/src/diracx/db/jobs/status_utility.py b/src/diracx/db/sql/jobs/status_utility.py similarity index 98% rename from src/diracx/db/jobs/status_utility.py rename to src/diracx/db/sql/jobs/status_utility.py index c83c05e04..b60dcaa3e 100644 --- a/src/diracx/db/jobs/status_utility.py +++ b/src/diracx/db/sql/jobs/status_utility.py @@ -9,7 +9,7 @@ SetJobStatusReturn, ) from diracx.core.utils import JobStatus -from diracx.db.jobs.db import JobDB, JobLoggingDB +from diracx.db.sql.jobs.db import JobDB, JobLoggingDB async def set_job_status( diff --git a/src/diracx/db/sql/sandbox_metadata/__init__.py b/src/diracx/db/sql/sandbox_metadata/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/diracx/db/sandbox_metadata/db.py b/src/diracx/db/sql/sandbox_metadata/db.py similarity index 98% rename from src/diracx/db/sandbox_metadata/db.py rename to src/diracx/db/sql/sandbox_metadata/db.py index bfe6cc649..6900f58a7 100644 --- a/src/diracx/db/sandbox_metadata/db.py +++ b/src/diracx/db/sql/sandbox_metadata/db.py @@ -7,7 +7,8 @@ import sqlalchemy -from ..sql_utils import BaseSQLDB +from diracx.db.sql.utils import BaseSQLDB + from .schema import Base as SandboxMetadataDBBase from .schema import sb_Owners, sb_SandBoxes diff --git a/src/diracx/db/sandbox_metadata/schema.py b/src/diracx/db/sql/sandbox_metadata/schema.py similarity index 96% rename from src/diracx/db/sandbox_metadata/schema.py rename to src/diracx/db/sql/sandbox_metadata/schema.py index 74bd675ea..9e3dc4afc 100644 --- a/src/diracx/db/sandbox_metadata/schema.py +++ b/src/diracx/db/sql/sandbox_metadata/schema.py @@ -9,7 +9,7 @@ ) from sqlalchemy.orm import declarative_base -from ..sql_utils import Column, DateNowColumn +from diracx.db.sql.utils import Column, DateNowColumn Base = declarative_base() diff --git a/src/diracx/db/sql_utils.py b/src/diracx/db/sql/utils.py similarity index 100% rename from src/diracx/db/sql_utils.py rename to src/diracx/db/sql/utils.py diff --git a/src/diracx/routers/__init__.py b/src/diracx/routers/__init__.py index deab42840..c3b492391 100644 --- a/src/diracx/routers/__init__.py +++ b/src/diracx/routers/__init__.py @@ -17,8 +17,8 @@ from diracx.core.exceptions import DiracError, DiracHttpResponse from diracx.core.extensions import select_from_extension from diracx.core.utils import dotenv_files_from_environment -from diracx.db.os_utils import BaseOSDB -from diracx.db.sql_utils import BaseSQLDB +from diracx.db.os.utils import BaseOSDB +from diracx.db.sql.utils import BaseSQLDB from ..core.settings import ServiceSettingsBase from .auth import verify_dirac_access_token diff --git a/src/diracx/routers/auth.py b/src/diracx/routers/auth.py index 3fe15f3bc..faf217919 100644 --- a/src/diracx/routers/auth.py +++ b/src/diracx/routers/auth.py @@ -39,7 +39,7 @@ UnevaluatedProperty, ) from diracx.core.settings import ServiceSettingsBase, TokenSigningKey -from diracx.db.auth.schema import FlowStatus, RefreshTokenStatus +from diracx.db.sql.auth.schema import FlowStatus, RefreshTokenStatus from .dependencies import ( AuthDB, diff --git a/src/diracx/routers/job_manager/__init__.py b/src/diracx/routers/job_manager/__init__.py index 2f06a4756..f848d7051 100644 --- a/src/diracx/routers/job_manager/__init__.py +++ b/src/diracx/routers/job_manager/__init__.py @@ -22,7 +22,7 @@ ) from diracx.core.properties import JOB_ADMINISTRATOR, NORMAL_USER from diracx.core.utils import JobStatus -from diracx.db.jobs.status_utility import ( +from diracx.db.sql.jobs.status_utility import ( set_job_status, ) diff --git a/tests/conftest.py b/tests/conftest.py index 99bc7cde7..85ac197df 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -91,7 +91,7 @@ def with_app(test_auth_settings, with_config_repo): @contextlib.asynccontextmanager async def create_db_schemas(app=app): """Create DB schema's based on the DBs available in app.dependency_overrides""" - from diracx.db.sql_utils import BaseSQLDB + from diracx.db.sql.utils import BaseSQLDB for k, v in app.dependency_overrides.items(): # Ignore dependency overrides which aren't BaseSQLDB.transaction diff --git a/tests/db/auth/test_authorization_flow.py b/tests/db/auth/test_authorization_flow.py index bf913f293..43987a95d 100644 --- a/tests/db/auth/test_authorization_flow.py +++ b/tests/db/auth/test_authorization_flow.py @@ -4,7 +4,7 @@ from sqlalchemy.exc import NoResultFound from diracx.core.exceptions import AuthorizationError -from diracx.db.auth.db import AuthDB +from diracx.db.sql.auth.db import AuthDB MAX_VALIDITY = 2 EXPIRED = 0 diff --git a/tests/db/auth/test_device_flow.py b/tests/db/auth/test_device_flow.py index 0430cc439..d42a151fb 100644 --- a/tests/db/auth/test_device_flow.py +++ b/tests/db/auth/test_device_flow.py @@ -6,8 +6,8 @@ from sqlalchemy.exc import NoResultFound from diracx.core.exceptions import AuthorizationError, ExpiredFlowError -from diracx.db.auth.db import AuthDB -from diracx.db.auth.schema import USER_CODE_LENGTH +from diracx.db.sql.auth.db import AuthDB +from diracx.db.sql.auth.schema import USER_CODE_LENGTH MAX_VALIDITY = 2 EXPIRED = 0 diff --git a/tests/db/auth/test_refresh_token.py b/tests/db/auth/test_refresh_token.py index e20a7c1c1..f5e2f18c0 100644 --- a/tests/db/auth/test_refresh_token.py +++ b/tests/db/auth/test_refresh_token.py @@ -2,8 +2,8 @@ import pytest -from diracx.db.auth.db import AuthDB -from diracx.db.auth.schema import RefreshTokenStatus +from diracx.db.sql.auth.db import AuthDB +from diracx.db.sql.auth.schema import RefreshTokenStatus @pytest.fixture diff --git a/tests/db/jobs/test_jobDB.py b/tests/db/jobs/test_jobDB.py index 702b91c4d..24eec16e0 100644 --- a/tests/db/jobs/test_jobDB.py +++ b/tests/db/jobs/test_jobDB.py @@ -4,7 +4,7 @@ import pytest -from diracx.db.jobs.db import JobDB +from diracx.db.sql.jobs.db import JobDB @pytest.fixture diff --git a/tests/db/test_dummyDB.py b/tests/db/test_dummyDB.py index 04a01f1f1..029971ae5 100644 --- a/tests/db/test_dummyDB.py +++ b/tests/db/test_dummyDB.py @@ -6,7 +6,7 @@ import pytest from diracx.core.exceptions import InvalidQueryError -from diracx.db.dummy.db import DummyDB +from diracx.db.sql.dummy.db import DummyDB # Each DB test class must defined a fixture looking like this one # It allows to get an instance of an in memory DB, diff --git a/tests/db/test_sandboxMetadataDB.py b/tests/db/test_sandboxMetadataDB.py index ed5286013..ffcf91eca 100644 --- a/tests/db/test_sandboxMetadataDB.py +++ b/tests/db/test_sandboxMetadataDB.py @@ -3,7 +3,7 @@ import pytest import sqlalchemy -from diracx.db.sandbox_metadata.db import SandboxMetadataDB +from diracx.db.sql.sandbox_metadata.db import SandboxMetadataDB @pytest.fixture