diff --git a/boefjes/boefjes/__main__.py b/boefjes/boefjes/__main__.py index e9621de3e0e..7111cb54eaa 100644 --- a/boefjes/boefjes/__main__.py +++ b/boefjes/boefjes/__main__.py @@ -15,12 +15,7 @@ @click.command() @click.argument("worker_type", type=click.Choice([q.value for q in WorkerManager.Queue])) -@click.option( - "--log-level", - type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"]), - help="Log level", - default="INFO", -) +@click.option("--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"]), help="Log level", default="INFO") def cli(worker_type: str, log_level: str): logger.setLevel(log_level) logger.info("Starting runtime for %s", worker_type) diff --git a/boefjes/boefjes/api.py b/boefjes/boefjes/api.py index c26f64e7979..d7d0d43b606 100644 --- a/boefjes/boefjes/api.py +++ b/boefjes/boefjes/api.py @@ -72,11 +72,7 @@ def get_scheduler_client(): def get_bytes_client(): - return BytesAPIClient( - str(settings.bytes_api), - username=settings.bytes_username, - password=settings.bytes_password, - ) + return BytesAPIClient(str(settings.bytes_api), username=settings.bytes_username, password=settings.bytes_password) @app.get("/healthz") diff --git a/boefjes/boefjes/app.py b/boefjes/boefjes/app.py index a97ef67f426..d7e7b68098b 100644 --- a/boefjes/boefjes/app.py +++ b/boefjes/boefjes/app.py @@ -231,10 +231,7 @@ def _format_exit_code(exitcode: int | None) -> str: def _start_working( - task_queue: mp.Queue, - handler: Handler, - scheduler_client: SchedulerClientInterface, - handling_tasks: dict[int, str], + task_queue: mp.Queue, handler: Handler, scheduler_client: SchedulerClientInterface, handling_tasks: dict[int, str] ): logger.info("Started listening for tasks from worker[pid=%s]", os.getpid()) diff --git a/boefjes/boefjes/clients/bytes_client.py b/boefjes/boefjes/clients/bytes_client.py index f2c5bee5d63..17851e8ccf5 100644 --- a/boefjes/boefjes/clients/bytes_client.py +++ b/boefjes/boefjes/clients/bytes_client.py @@ -38,10 +38,7 @@ def __init__(self, base_url: str, username: str, password: str): transport=(HTTPTransport(retries=6)), ) - self.credentials = { - "username": username, - "password": password, - } + self.credentials = {"username": username, "password": password} self.headers: dict[str, str] = {} def login(self) -> None: @@ -63,9 +60,7 @@ def _get_authentication_headers(self) -> dict[str, str]: def _get_token(self) -> str: response = self._session.post( - "/token", - data=self.credentials, - headers={"content-type": "application/x-www-form-urlencoded"}, + "/token", data=self.credentials, headers={"content-type": "application/x-www-form-urlencoded"} ) return str(response.json()["access_token"]) diff --git a/boefjes/boefjes/config.py b/boefjes/boefjes/config.py index f870a66e777..8e226fdc259 100644 --- a/boefjes/boefjes/config.py +++ b/boefjes/boefjes/config.py @@ -82,23 +82,12 @@ class Settings(BaseSettings): ..., examples=["http://localhost:8001"], description="Octopoes API URL", validation_alias="OCTOPOES_API" ) api: AnyHttpUrl = Field( - ..., - examples=["http://boefje:8000"], - description="The URL on which the boefjes API is available", + ..., examples=["http://boefje:8000"], description="The URL on which the boefjes API is available" ) # Boefje server settings - api_host: str = Field( - "0.0.0.0", - description="Host address of the Boefje API server", - ) - api_port: int = Field( - 8000, - description="Host port of the Boefje API server", - ) - docker_network: str = Field( - "bridge", - description="Docker network to run Boefjes in", - ) + api_host: str = Field("0.0.0.0", description="Host address of the Boefje API server") + api_port: int = Field(8000, description="Host port of the Boefje API server") + docker_network: str = Field("bridge", description="Docker network to run Boefjes in") bytes_api: AnyHttpUrl = Field( ..., examples=["http://localhost:8002"], description="Bytes API URL", validation_alias="BYTES_API" ) diff --git a/boefjes/boefjes/docker_boefjes_runner.py b/boefjes/boefjes/docker_boefjes_runner.py index 8375c57a1be..8bfcbcde3b9 100644 --- a/boefjes/boefjes/docker_boefjes_runner.py +++ b/boefjes/boefjes/docker_boefjes_runner.py @@ -21,9 +21,7 @@ def __init__(self, boefje_resource: Boefje, boefje_meta: BoefjeMeta): self.docker_client = docker.from_env() self.scheduler_client = SchedulerAPIClient(str(settings.scheduler_api)) self.bytes_api_client = BytesAPIClient( - str(settings.bytes_api), - username=settings.bytes_username, - password=settings.bytes_password, + str(settings.bytes_api), username=settings.bytes_username, password=settings.bytes_password ) def run(self) -> None: diff --git a/boefjes/boefjes/job_handler.py b/boefjes/boefjes/job_handler.py index 738166d548e..555e5f55eb8 100644 --- a/boefjes/boefjes/job_handler.py +++ b/boefjes/boefjes/job_handler.py @@ -26,9 +26,7 @@ logger = logging.getLogger(__name__) bytes_api_client = BytesAPIClient( - str(settings.bytes_api), - username=settings.bytes_username, - password=settings.bytes_password, + str(settings.bytes_api), username=settings.bytes_username, password=settings.bytes_password ) @@ -68,8 +66,7 @@ def get_environment_settings(boefje_meta: BoefjeMeta, environment_keys: list[str try: katalogus_api = str(settings.katalogus_api).rstrip("/") response = httpx.get( - f"{katalogus_api}/v1/organisations/{boefje_meta.organization}/{boefje_meta.boefje.id}/settings", - timeout=30, + f"{katalogus_api}/v1/organisations/{boefje_meta.organization}/{boefje_meta.boefje.id}/settings", timeout=30 ) response.raise_for_status() environment = response.json() @@ -91,10 +88,7 @@ def get_environment_settings(boefje_meta: BoefjeMeta, environment_keys: list[str class BoefjeHandler(Handler): def __init__( - self, - job_runner: BoefjeJobRunner, - local_repository: LocalPluginRepository, - bytes_client: BytesAPIClient, + self, job_runner: BoefjeJobRunner, local_repository: LocalPluginRepository, bytes_client: BytesAPIClient ): self.job_runner = job_runner self.local_repository = local_repository diff --git a/boefjes/boefjes/katalogus/api.py b/boefjes/boefjes/katalogus/api.py index 3da6cb85668..e9234b199dc 100644 --- a/boefjes/boefjes/katalogus/api.py +++ b/boefjes/boefjes/katalogus/api.py @@ -48,10 +48,7 @@ def entity_not_found_handler(request: Request, exc: StorageError): logger.exception("some error", exc_info=exc) - return JSONResponse( - status_code=status.HTTP_404_NOT_FOUND, - content={"message": exc.message}, - ) + return JSONResponse(status_code=status.HTTP_404_NOT_FOUND, content={"message": exc.message}) class ServiceHealth(BaseModel): diff --git a/boefjes/boefjes/katalogus/dependencies/plugins.py b/boefjes/boefjes/katalogus/dependencies/plugins.py index 974b23cfde2..3cd3764bffd 100644 --- a/boefjes/boefjes/katalogus/dependencies/plugins.py +++ b/boefjes/boefjes/katalogus/dependencies/plugins.py @@ -160,12 +160,7 @@ def update_by_id(self, repository_id: str, plugin_id: str, organisation_id: str, all_settings = self.settings_storage.get_all(organisation_id, plugin_id) self._assert_settings_match_schema(all_settings, organisation_id, plugin_id) - self.plugin_enabled_store.update_or_create_by_id( - plugin_id, - repository_id, - enabled, - organisation_id, - ) + self.plugin_enabled_store.update_or_create_by_id(plugin_id, repository_id, enabled, organisation_id) def _plugins_for_repos( self, repositories: Iterable[Repository], organisation_id: str diff --git a/boefjes/boefjes/katalogus/dependencies/repositories.py b/boefjes/boefjes/katalogus/dependencies/repositories.py index ee7b21fdf19..05cc1b24c72 100644 --- a/boefjes/boefjes/katalogus/dependencies/repositories.py +++ b/boefjes/boefjes/katalogus/dependencies/repositories.py @@ -8,7 +8,5 @@ logger = logging.getLogger(__name__) -def get_repository_store( - organisation_id: str, -) -> Iterator[RepositoryStorage]: +def get_repository_store(organisation_id: str) -> Iterator[RepositoryStorage]: yield from session_managed_iterator(create_repository_storage) diff --git a/boefjes/boefjes/katalogus/routers/organisations.py b/boefjes/boefjes/katalogus/routers/organisations.py index fcb68e0f99f..04068dd1e62 100644 --- a/boefjes/boefjes/katalogus/routers/organisations.py +++ b/boefjes/boefjes/katalogus/routers/organisations.py @@ -9,8 +9,7 @@ def check_organisation_exists( - organisation_id: str, - storage: OrganisationStorage = Depends(get_organisations_store), + organisation_id: str, storage: OrganisationStorage = Depends(get_organisations_store) ) -> None: """ Checks if an organisation exists, if not, creates it. @@ -24,17 +23,12 @@ def check_organisation_exists( @router.get("", response_model=dict[str, Organisation]) -def list_organisations( - storage: OrganisationStorage = Depends(get_organisations_store), -): +def list_organisations(storage: OrganisationStorage = Depends(get_organisations_store)): return storage.get_all() @router.get("/{organisation_id}", response_model=Organisation) -def get_organisation( - organisation_id: str, - storage: OrganisationStorage = Depends(get_organisations_store), -): +def get_organisation(organisation_id: str, storage: OrganisationStorage = Depends(get_organisations_store)): try: return storage.get_by_id(organisation_id) except (KeyError, ObjectNotFoundException): @@ -42,18 +36,12 @@ def get_organisation( @router.post("/", status_code=status.HTTP_201_CREATED) -def add_organisation( - organisation: Organisation, - storage: OrganisationStorage = Depends(get_organisations_store), -): +def add_organisation(organisation: Organisation, storage: OrganisationStorage = Depends(get_organisations_store)): with storage as store: store.create(organisation) @router.delete("/{organisation_id}") -def remove_organisation( - organisation_id: str, - storage: OrganisationStorage = Depends(get_organisations_store), -): +def remove_organisation(organisation_id: str, storage: OrganisationStorage = Depends(get_organisations_store)): with storage as store: store.delete_by_id(organisation_id) diff --git a/boefjes/boefjes/katalogus/routers/plugins.py b/boefjes/boefjes/katalogus/routers/plugins.py index be3b9cabd83..32d0cc658e0 100644 --- a/boefjes/boefjes/katalogus/routers/plugins.py +++ b/boefjes/boefjes/katalogus/routers/plugins.py @@ -15,9 +15,7 @@ from boefjes.katalogus.types import FilterParameters, PaginationParameters router = APIRouter( - prefix="/organisations/{organisation_id}", - tags=["plugins"], - dependencies=[Depends(check_organisation_exists)], + prefix="/organisations/{organisation_id}", tags=["plugins"], dependencies=[Depends(check_organisation_exists)] ) @@ -49,10 +47,7 @@ def list_plugins( # filter plugins by id, name or description if filter_params.q is not None: - plugins = filter( - partial(_plugin_matches_query, query=filter_params.q), - plugins, - ) + plugins = filter(partial(_plugin_matches_query, query=filter_params.q), plugins) # filter plugins by type if filter_params.type is not None: @@ -74,9 +69,7 @@ def list_plugins( @router.get("/plugins/{plugin_id}", response_model=PluginType) def get_plugin( - plugin_id: str, - organisation_id: str, - plugin_service: PluginService = Depends(get_plugin_service), + plugin_id: str, organisation_id: str, plugin_service: PluginService = Depends(get_plugin_service) ) -> PluginType: try: with plugin_service as p: @@ -87,14 +80,9 @@ def get_plugin( raise HTTPException(ex.response.status_code) -@router.get( - "/repositories/{repository_id}/plugins", - response_model=dict[str, PluginType], -) +@router.get("/repositories/{repository_id}/plugins", response_model=dict[str, PluginType]) def list_repository_plugins( - repository_id: str, - organisation_id: str, - plugin_service: PluginService = Depends(get_plugin_service), + repository_id: str, organisation_id: str, plugin_service: PluginService = Depends(get_plugin_service) ): with plugin_service as p: return p.repository_plugins(repository_id, organisation_id) @@ -135,8 +123,7 @@ def update_plugin_state( @router.get("/plugins/{plugin_id}/schema.json", include_in_schema=False) def get_plugin_schema( - plugin_id: str, - plugin_service: PluginService = Depends(get_plugin_service), + plugin_id: str, plugin_service: PluginService = Depends(get_plugin_service) ) -> JSONResponse: # TODO: support for plugin covers in plugin repositories (?) try: with plugin_service as p: @@ -149,8 +136,7 @@ def get_plugin_schema( @router.get("/plugins/{plugin_id}/cover.jpg", include_in_schema=False) def get_plugin_cover( - plugin_id: str, - plugin_service: PluginService = Depends(get_plugin_service), + plugin_id: str, plugin_service: PluginService = Depends(get_plugin_service) ) -> FileResponse: # TODO: support for plugin covers in plugin repositories (?) try: with plugin_service as p: @@ -163,9 +149,7 @@ def get_plugin_cover( @router.get("/plugins/{plugin_id}/description.md", include_in_schema=False) def get_plugin_description( - plugin_id: str, - organisation_id: str, - plugin_service: PluginService = Depends(get_plugin_service), + plugin_id: str, organisation_id: str, plugin_service: PluginService = Depends(get_plugin_service) ) -> Response: # TODO: support for markdown descriptions in plugin repositories (?) try: with plugin_service as p: @@ -178,9 +162,7 @@ def get_plugin_description( @router.post("/settings/clone/{to_organisation_id}") def clone_organisation_settings( - organisation_id: str, - to_organisation_id: str, - storage: PluginService = Depends(get_plugin_service), + organisation_id: str, to_organisation_id: str, storage: PluginService = Depends(get_plugin_service) ): with storage as store: store.clone_settings_to_organisation(organisation_id, to_organisation_id) diff --git a/boefjes/boefjes/katalogus/routers/repositories.py b/boefjes/boefjes/katalogus/routers/repositories.py index b00d08d16ea..6631b0e9451 100644 --- a/boefjes/boefjes/katalogus/routers/repositories.py +++ b/boefjes/boefjes/katalogus/routers/repositories.py @@ -26,19 +26,13 @@ def get_repository(repository_id: str, storage: RepositoryStorage = Depends(get_ @router.post("/", status_code=status.HTTP_201_CREATED) -def add_repository( - repository: Repository, - storage: RepositoryStorage = Depends(get_repository_store), -): +def add_repository(repository: Repository, storage: RepositoryStorage = Depends(get_repository_store)): with storage as store: store.create(repository) @router.delete("/{repository_id}") -def remove_repository( - repository_id: str, - storage: RepositoryStorage = Depends(get_repository_store), -): +def remove_repository(repository_id: str, storage: RepositoryStorage = Depends(get_repository_store)): if repository_id == RESERVED_LOCAL_ID: raise HTTPException(status.HTTP_403_FORBIDDEN, "LOCAL repository cannot be deleted") with storage as store: diff --git a/boefjes/boefjes/katalogus/routers/settings.py b/boefjes/boefjes/katalogus/routers/settings.py index 9c1bebe8d04..92094051603 100644 --- a/boefjes/boefjes/katalogus/routers/settings.py +++ b/boefjes/boefjes/katalogus/routers/settings.py @@ -11,21 +11,14 @@ @router.get("", response_model=dict) -def list_settings( - organisation_id: str, - plugin_id: str, - plugin_service: PluginService = Depends(get_plugin_service), -): +def list_settings(organisation_id: str, plugin_id: str, plugin_service: PluginService = Depends(get_plugin_service)): with plugin_service as p: return p.get_all_settings(organisation_id, plugin_id) @router.put("") def upsert_settings( - organisation_id: str, - plugin_id: str, - values: dict, - plugin_service: PluginService = Depends(get_plugin_service), + organisation_id: str, plugin_id: str, values: dict, plugin_service: PluginService = Depends(get_plugin_service) ): with plugin_service as p: p.upsert_settings(values, organisation_id, plugin_id) diff --git a/boefjes/boefjes/katalogus/storage/memory.py b/boefjes/boefjes/katalogus/storage/memory.py index a7d2b886b3a..1c2963cd420 100644 --- a/boefjes/boefjes/katalogus/storage/memory.py +++ b/boefjes/boefjes/katalogus/storage/memory.py @@ -34,11 +34,7 @@ def delete_by_id(self, organisation_id: str) -> None: class RepositoryStorageMemory(RepositoryStorage): - def __init__( - self, - organisation_id: str, - defaults: dict[str, Repository] | None = None, - ): + def __init__(self, organisation_id: str, defaults: dict[str, Repository] | None = None): self._data = repositories.setdefault(organisation_id, {}) if defaults is None else defaults self._organisation_id = organisation_id @@ -79,11 +75,7 @@ def delete(self, organisation_id: str, plugin_id: str) -> None: class PluginStatesStorageMemory(PluginEnabledStorage): - def __init__( - self, - organisation: str, - defaults: dict[str, bool] | None = None, - ): + def __init__(self, organisation: str, defaults: dict[str, bool] | None = None): self._data = plugins_state.setdefault(organisation, {}) if defaults is None else defaults self._organisation = organisation diff --git a/boefjes/boefjes/katalogus/tests/integration/test_api.py b/boefjes/boefjes/katalogus/tests/integration/test_api.py index c060014ca3c..18f7cec0a5b 100644 --- a/boefjes/boefjes/katalogus/tests/integration/test_api.py +++ b/boefjes/boefjes/katalogus/tests/integration/test_api.py @@ -40,13 +40,7 @@ def setUp(self) -> None: self.plugin_state_storage = SQLPluginEnabledStorage(session, settings) with self.repository_storage as store: - store.create( - Repository( - id="LOCAL", - name="Test", - base_url="http://test.url", - ) - ) + store.create(Repository(id="LOCAL", name="Test", base_url="http://test.url")) self.org = Organisation(id="test", name="Test Organisation") diff --git a/boefjes/boefjes/katalogus/tests/integration/test_sql_repositories.py b/boefjes/boefjes/katalogus/tests/integration/test_sql_repositories.py index e086df89731..8d4f43a7e3a 100644 --- a/boefjes/boefjes/katalogus/tests/integration/test_sql_repositories.py +++ b/boefjes/boefjes/katalogus/tests/integration/test_sql_repositories.py @@ -75,11 +75,7 @@ def test_repository_storage(self): repository_id = "test" with self.repository_storage as repository_storage: - repo = Repository( - id=repository_id, - name="Test", - base_url="http://test.url", - ) + repo = Repository(id=repository_id, name="Test", base_url="http://test.url") repository_storage.create(repo) returned_repo = repository_storage.get_by_id(repository_id) @@ -93,11 +89,7 @@ def test_repository_storage(self): def test_organisations_repositories(self): org = Organisation(id="org1", name="Test") - repo = Repository( - id="repo-123", - name="Test", - base_url="http://test.url", - ) + repo = Repository(id="repo-123", name="Test", base_url="http://test.url") with self.repository_storage as storage: storage.create(repo) @@ -113,11 +105,7 @@ def test_organisations_repositories(self): self.assertEqual(1, len(repositories)) self.assertEqual(repo, repositories[0]) - repo2 = Repository( - id="repo-321", - name="Test", - base_url="http://test.url", - ) + repo2 = Repository(id="repo-321", name="Test", base_url="http://test.url") with self.repository_storage as storage: storage.create(repo2) @@ -202,11 +190,7 @@ def test_plugin_enabled_storage(self): storage.create(org) with self.repository_storage as storage: - repo = Repository( - id="repo-test", - name="Test", - base_url="http://test.url", - ) + repo = Repository(id="repo-test", name="Test", base_url="http://test.url") storage.create(repo) with self.plugin_state_storage as plugin_state_storage: diff --git a/boefjes/boefjes/katalogus/tests/test_plugin_service.py b/boefjes/boefjes/katalogus/tests/test_plugin_service.py index c91175a2711..ddc03a3e368 100644 --- a/boefjes/boefjes/katalogus/tests/test_plugin_service.py +++ b/boefjes/boefjes/katalogus/tests/test_plugin_service.py @@ -265,9 +265,4 @@ def test_clone_many_settings(self): all_settings_for_new_org = self.service.get_all_settings("org2", "test-boefje-1") assert len(all_settings_for_new_org) == 4 - assert all_settings_for_new_org == { - "0": "1", - "2": "3", - "4": "5", - "6": "7", - } + assert all_settings_for_new_org == {"0": "1", "2": "3", "4": "5", "6": "7"} diff --git a/boefjes/boefjes/katalogus/tests/test_plugins.py b/boefjes/boefjes/katalogus/tests/test_plugins.py index 61c802d9e24..effbc8bd1cc 100644 --- a/boefjes/boefjes/katalogus/tests/test_plugins.py +++ b/boefjes/boefjes/katalogus/tests/test_plugins.py @@ -40,14 +40,7 @@ def test_list_filter_by_type(self): res = self.client.get("/v1/organisations/test-org/plugins?plugin_type=boefje") self.assertEqual(200, res.status_code) self.assertSetEqual( - { - "test-boefje-1", - "test-boefje-2", - "kat_test", - "kat_test_2", - "kat_test_4", - }, - {x["id"] for x in res.json()}, + {"test-boefje-1", "test-boefje-2", "kat_test", "kat_test_2", "kat_test_4"}, {x["id"] for x in res.json()} ) def test_list_filter_by_state(self): @@ -55,12 +48,7 @@ def test_list_filter_by_state(self): self.assertEqual(200, res.status_code) plugins = res.json() self.assertSetEqual( - { - "test-bit-1", - "test-normalizer-1", - "kat_test_normalize", - "kat_test_normalize_2", - }, + {"test-bit-1", "test-normalizer-1", "kat_test_normalize", "kat_test_normalize_2"}, {x["id"] for x in plugins}, ) self.assertTrue(all([x["enabled"] for x in plugins])) @@ -69,42 +57,24 @@ def test_list_filter_by_id(self): res = self.client.get("/v1/organisations/test-org/plugins?q=kat") self.assertEqual(200, res.status_code) self.assertSetEqual( - { - "kat_test", - "kat_test_2", - "kat_test_4", - "kat_test_normalize", - "kat_test_normalize_2", - }, + {"kat_test", "kat_test_2", "kat_test_4", "kat_test_normalize", "kat_test_normalize_2"}, {x["id"] for x in (res.json())}, ) def test_list_pagination(self): res = self.client.get("/v1/organisations/test-org/plugins?offset=2&limit=2") self.assertEqual(200, res.status_code) - self.assertSetEqual( - { - "test-bit-1", - "test-normalizer-1", - }, - {x["id"] for x in (res.json())}, - ) + self.assertSetEqual({"test-bit-1", "test-normalizer-1"}, {x["id"] for x in (res.json())}) def test_list_repository(self): res = self.client.get("/v1/organisations/test-org/repositories/test-repo/plugins") self.assertEqual(200, res.status_code) - self.assertListEqual( - ["test-boefje-1", "test-boefje-2"], - list(res.json().keys()), - ) + self.assertListEqual(["test-boefje-1", "test-boefje-2"], list(res.json().keys())) def test_list_repository2(self): res = self.client.get("/v1/organisations/test-org/repositories/test-repo-2/plugins") self.assertEqual(200, res.status_code) - self.assertListEqual( - ["test-bit-1", "test-normalizer-1"], - list(res.json().keys()), - ) + self.assertListEqual(["test-bit-1", "test-normalizer-1"], list(res.json().keys())) def test_get_plugin(self): res = self.client.get("/v1/organisations/test-org/repositories/test-repo/plugins/test-boefje-1") @@ -132,8 +102,7 @@ def test_default_enabled_property_list(self): def test_patching_enabled_state(self): res = self.client.patch( - "/v1/organisations/test-org/repositories/test-repo/plugins/test-boefje-1", - json={"enabled": False}, + "/v1/organisations/test-org/repositories/test-repo/plugins/test-boefje-1", json={"enabled": False} ) self.assertEqual(200, res.status_code) @@ -156,8 +125,7 @@ def test_patching_enabled_state(self): def test_patching_enabled_state_non_existing_org(self): res = self.client.patch( - "/v1/organisations/non-existing-org/repositories/test-repo/plugins/test-boefje-1", - json={"enabled": False}, + "/v1/organisations/non-existing-org/repositories/test-repo/plugins/test-boefje-1", json={"enabled": False} ) self.assertEqual(200, res.status_code) diff --git a/boefjes/boefjes/katalogus/tests/test_repositories.py b/boefjes/boefjes/katalogus/tests/test_repositories.py index 410d5765fa2..11e57980a35 100644 --- a/boefjes/boefjes/katalogus/tests/test_repositories.py +++ b/boefjes/boefjes/katalogus/tests/test_repositories.py @@ -16,9 +16,7 @@ def setUp(self) -> None: _mocked_repositories = {"test": Repository(id="test", name="Test", base_url="http://localhost:8080")} - def _mocked_get_repository_store( - organisation_id: str, - ): + def _mocked_get_repository_store(organisation_id: str): return RepositoryStorageMemory(organisation_id, _mocked_repositories) app.dependency_overrides[get_repository_store] = _mocked_get_repository_store diff --git a/boefjes/boefjes/local.py b/boefjes/boefjes/local.py index 5d7f799ced5..2b7aa042008 100644 --- a/boefjes/boefjes/local.py +++ b/boefjes/boefjes/local.py @@ -108,17 +108,12 @@ def _parse_results(self, normalizer_meta: NormalizerMeta, results: Iterable[Norm observations = [ NormalizerObservation( - type="observation", - input_ooi=normalizer_meta.raw_data.boefje_meta.input_ooi, - results=oois, + type="observation", input_ooi=normalizer_meta.raw_data.boefje_meta.input_ooi, results=oois ) ] else: observations = [] return NormalizerResults( - observations=observations, - declarations=declarations, - affirmations=affirmations, - scan_profiles=scan_profiles, + observations=observations, declarations=declarations, affirmations=affirmations, scan_profiles=scan_profiles ) diff --git a/boefjes/boefjes/migrations/env.py b/boefjes/boefjes/migrations/env.py index cf7a0a4c2dd..883101aee81 100644 --- a/boefjes/boefjes/migrations/env.py +++ b/boefjes/boefjes/migrations/env.py @@ -58,9 +58,7 @@ def run_migrations_online() -> None: """ connectable = engine_from_config( - config.get_section(config.config_ini_section), - prefix="sqlalchemy.", - poolclass=pool.NullPool, + config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool ) with connectable.connect() as connection: diff --git a/boefjes/boefjes/migrations/versions/0001_add_katalogus_models.py b/boefjes/boefjes/migrations/versions/0001_add_katalogus_models.py index 2a9cfd11232..bf557cca5d7 100644 --- a/boefjes/boefjes/migrations/versions/0001_add_katalogus_models.py +++ b/boefjes/boefjes/migrations/versions/0001_add_katalogus_models.py @@ -37,14 +37,8 @@ def upgrade() -> None: "organisation_repository", sa.Column("organisation_pk", sa.Integer(), nullable=False), sa.Column("repository_pk", sa.Integer(), nullable=False), - sa.ForeignKeyConstraint( - ["organisation_pk"], - ["organisation.pk"], - ), - sa.ForeignKeyConstraint( - ["repository_pk"], - ["repository.pk"], - ), + sa.ForeignKeyConstraint(["organisation_pk"], ["organisation.pk"]), + sa.ForeignKeyConstraint(["repository_pk"], ["repository.pk"]), ) op.create_table( "setting", diff --git a/boefjes/boefjes/migrations/versions/0002_change_lengths_of_several_char_fields.py b/boefjes/boefjes/migrations/versions/0002_change_lengths_of_several_char_fields.py index 34996310e2a..b932fc39714 100644 --- a/boefjes/boefjes/migrations/versions/0002_change_lengths_of_several_char_fields.py +++ b/boefjes/boefjes/migrations/versions/0002_change_lengths_of_several_char_fields.py @@ -22,30 +22,18 @@ def upgrade() -> None: op.create_unique_constraint("organisation_id", "organisation", ["id"]) op.create_unique_constraint( - "unique_plugin_per_repo_per_org", - "plugin_state", - ["plugin_id", "organisation_pk", "repository_pk"], + "unique_plugin_per_repo_per_org", "plugin_state", ["plugin_id", "organisation_pk", "repository_pk"] ) op.create_unique_constraint("repository_id", "repository", ["id"]) op.create_unique_constraint( - "unique_keys_per_organisation_per_plugin", - "setting", - ["key", "organisation_pk", "plugin_id"], + "unique_keys_per_organisation_per_plugin", "setting", ["key", "organisation_pk", "plugin_id"] ) op.alter_column( - "setting", - "key", - existing_type=sa.String(length=32), - type_=sa.String(length=128), - existing_nullable=False, + "setting", "key", existing_type=sa.String(length=32), type_=sa.String(length=128), existing_nullable=False ) op.alter_column( - "setting", - "value", - existing_type=sa.String(length=64), - type_=sa.String(length=128), - existing_nullable=False, + "setting", "value", existing_type=sa.String(length=64), type_=sa.String(length=128), existing_nullable=False ) op.drop_constraint("plugin_state_plugin_id_key", "plugin_state", type_="unique") # ### end Alembic commands ### @@ -56,18 +44,10 @@ def downgrade() -> None: op.drop_constraint("unique_keys_per_organisation_per_plugin", "setting", type_="unique") op.alter_column( - "setting", - "value", - existing_type=sa.String(length=128), - type_=sa.String(length=64), - existing_nullable=False, + "setting", "value", existing_type=sa.String(length=128), type_=sa.String(length=64), existing_nullable=False ) op.alter_column( - "setting", - "key", - existing_type=sa.String(length=128), - type_=sa.String(length=32), - existing_nullable=False, + "setting", "key", existing_type=sa.String(length=128), type_=sa.String(length=32), existing_nullable=False ) op.drop_constraint("repository_id", "repository", type_="unique") diff --git a/boefjes/boefjes/migrations/versions/0003_longer_plugin_ids.py b/boefjes/boefjes/migrations/versions/0003_longer_plugin_ids.py index a219b0c39df..e66ee40c2dd 100644 --- a/boefjes/boefjes/migrations/versions/0003_longer_plugin_ids.py +++ b/boefjes/boefjes/migrations/versions/0003_longer_plugin_ids.py @@ -19,11 +19,7 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.alter_column( - "setting", - "plugin_id", - existing_type=sa.String(length=32), - type_=sa.String(length=64), - existing_nullable=False, + "setting", "plugin_id", existing_type=sa.String(length=32), type_=sa.String(length=64), existing_nullable=False ) op.alter_column( "plugin_state", @@ -45,10 +41,6 @@ def downgrade() -> None: existing_nullable=False, ) op.alter_column( - "setting", - "plugin_id", - existing_type=sa.String(length=64), - type_=sa.String(length=32), - existing_nullable=False, + "setting", "plugin_id", existing_type=sa.String(length=64), type_=sa.String(length=32), existing_nullable=False ) # ### end Alembic commands ### diff --git a/boefjes/boefjes/plugins/helpers.py b/boefjes/boefjes/plugins/helpers.py index 7b3e45d39f3..bce85f23d00 100644 --- a/boefjes/boefjes/plugins/helpers.py +++ b/boefjes/boefjes/plugins/helpers.py @@ -49,12 +49,7 @@ def get_file_from_container(container: docker.models.containers.Container, path: try: stream, _ = container.get_archive(path) except docker.errors.NotFound: - logging.warning( - "%s not found in container %s %s", - path, - container.short_id, - container.image.tags, - ) + logging.warning("%s not found in container %s %s", path, container.short_id, container.image.tags) return None f = tarfile.open(mode="r|", fileobj=TarStream(stream).reader()) diff --git a/boefjes/boefjes/plugins/kat_adr_validator/normalize.py b/boefjes/boefjes/plugins/kat_adr_validator/normalize.py index 5be64a3858e..e93591f4362 100644 --- a/boefjes/boefjes/plugins/kat_adr_validator/normalize.py +++ b/boefjes/boefjes/plugins/kat_adr_validator/normalize.py @@ -18,12 +18,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: message = result["message"] rule = APIDesignRule(name=rule_name) - rule_result = APIDesignRuleResult( - rest_api=ooi_ref, - rule=rule.reference, - passed=passed, - message=message, - ) + rule_result = APIDesignRuleResult(rest_api=ooi_ref, rule=rule.reference, passed=passed, message=message) yield rule yield rule_result @@ -32,11 +27,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: continue ft = ADRFindingType(id=rule_name) - finding = Finding( - finding_type=ft.reference, - ooi=ooi_ref, - description=message, - ) + finding = Finding(finding_type=ft.reference, ooi=ooi_ref, description=message) yield ft yield finding diff --git a/boefjes/boefjes/plugins/kat_answer_parser/normalize.py b/boefjes/boefjes/plugins/kat_answer_parser/normalize.py index 2158841c2c6..acef37ad53f 100644 --- a/boefjes/boefjes/plugins/kat_answer_parser/normalize.py +++ b/boefjes/boefjes/plugins/kat_answer_parser/normalize.py @@ -10,8 +10,4 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: bit_id = data["schema"].removeprefix("/bit/") - yield Config( - ooi=input_ooi["primary_key"], - bit_id=bit_id, - config=data["answer"], - ) + yield Config(ooi=input_ooi["primary_key"], bit_id=bit_id, config=data["answer"]) diff --git a/boefjes/boefjes/plugins/kat_binaryedge/containers/normalize.py b/boefjes/boefjes/plugins/kat_binaryedge/containers/normalize.py index 635d4d2fc7c..04a2ef3726b 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/containers/normalize.py +++ b/boefjes/boefjes/plugins/kat_binaryedge/containers/normalize.py @@ -30,24 +30,13 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: else: ipvx = ipaddress.ip_address(ip) if ipvx.version == 4: - ip_ooi = IPAddressV4( - address=ip, - network=network, - ) + ip_ooi = IPAddressV4(address=ip, network=network) else: - ip_ooi = IPAddressV6( - address=ip, - network=network, - ) + ip_ooi = IPAddressV6(address=ip, network=network) yield ip_ooi ip_ref = ip_ooi.reference - ip_port_ooi = IPPort( - address=ip_ref, - protocol=Protocol(protocol), - port=port_nr, - state=PortState("open"), - ) + ip_port_ooi = IPPort(address=ip_ref, protocol=Protocol(protocol), port=port_nr, state=PortState("open")) yield ip_port_ooi software_ooi = Software(name=module.capitalize()) diff --git a/boefjes/boefjes/plugins/kat_binaryedge/databases/normalize.py b/boefjes/boefjes/plugins/kat_binaryedge/databases/normalize.py index 6dab2a5f798..3f0e3569409 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/databases/normalize.py +++ b/boefjes/boefjes/plugins/kat_binaryedge/databases/normalize.py @@ -15,13 +15,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: network = Network(name="internet").reference # Structure based on https://docs.binaryedge.io/modules// - accepted_modules = ( - "cassandra", - " elasticsearch", - "memcached", - "mongodb", - "redis", - ) + accepted_modules = ("cassandra", " elasticsearch", "memcached", "mongodb", "redis") for scan in results["results"]: module = scan["origin"]["type"] if module not in accepted_modules: @@ -36,24 +30,13 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: else: ipvx = ipaddress.ip_address(ip) if ipvx.version == 4: - ip_ooi = IPAddressV4( - address=ip, - network=network, - ) + ip_ooi = IPAddressV4(address=ip, network=network) else: - ip_ooi = IPAddressV6( - address=ip, - network=network, - ) + ip_ooi = IPAddressV6(address=ip, network=network) yield ip_ooi ip_ref = ip_ooi.reference - ip_port_ooi = IPPort( - address=ip_ref, - protocol=Protocol(protocol), - port=port_nr, - state=PortState("open"), - ) + ip_port_ooi = IPPort(address=ip_ref, protocol=Protocol(protocol), port=port_nr, state=PortState("open")) yield ip_port_ooi software_version = None diff --git a/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalize.py b/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalize.py index 8b9ae1b0511..4e9181fbf8a 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalize.py +++ b/boefjes/boefjes/plugins/kat_binaryedge/http_web/normalize.py @@ -15,10 +15,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: network = Network(name="internet").reference # Structure based on https://docs.binaryedge.io/modules// - accepted_modules = ( - "webv2", - " web-enrich", - ) # http/https: deprecated, so not implemented. + accepted_modules = ("webv2", " web-enrich") # http/https: deprecated, so not implemented. for scan in results["results"]: module = scan["origin"]["type"] if module not in accepted_modules: @@ -33,24 +30,13 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: else: ipvx = ipaddress.ip_address(ip) if ipvx.version == 4: - ip_ooi = IPAddressV4( - address=ip, - network=network, - ) + ip_ooi = IPAddressV4(address=ip, network=network) else: - ip_ooi = IPAddressV6( - address=ip, - network=network, - ) + ip_ooi = IPAddressV6(address=ip, network=network) yield ip_ooi ip_ref = ip_ooi.reference - ip_port_ooi = IPPort( - address=ip_ref, - protocol=Protocol(protocol), - port=port_nr, - state=PortState("open"), - ) + ip_port_ooi = IPPort(address=ip_ref, protocol=Protocol(protocol), port=port_nr, state=PortState("open")) yield ip_port_ooi if module == "webv2": @@ -72,17 +58,11 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: if "version" in app: software_ooi = Software(name=software_name, version=app["version"]) yield software_ooi - yield SoftwareInstance( - ooi=ip_port_ooi.reference, - software=software_ooi.reference, - ) + yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference) else: software_ooi = Software(name=software_name) yield software_ooi - yield SoftwareInstance( - ooi=ip_port_ooi.reference, - software=software_ooi.reference, - ) + yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference) elif module == "web-enrich": # (potential) TODO: # * http_version [string] diff --git a/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalize.py b/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalize.py index e37b0e5bce5..05f1b9765e1 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalize.py +++ b/boefjes/boefjes/plugins/kat_binaryedge/message_queues/normalize.py @@ -30,24 +30,13 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: else: ipvx = ipaddress.ip_address(ip) if ipvx.version == 4: - ip_ooi = IPAddressV4( - address=ip, - network=network, - ) + ip_ooi = IPAddressV4(address=ip, network=network) else: - ip_ooi = IPAddressV6( - address=ip, - network=network, - ) + ip_ooi = IPAddressV6(address=ip, network=network) yield ip_ooi ip_ref = ip_ooi.reference - ip_port_ooi = IPPort( - address=ip_ref, - protocol=Protocol(protocol), - port=port_nr, - state=PortState("open"), - ) + ip_port_ooi = IPPort(address=ip_ref, protocol=Protocol(protocol), port=port_nr, state=PortState("open")) yield ip_port_ooi software_ooi = Software(name=module) diff --git a/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalize.py b/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalize.py index 1ac27cfd201..0a431ba60a8 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalize.py +++ b/boefjes/boefjes/plugins/kat_binaryedge/protocols/normalize.py @@ -29,24 +29,13 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: else: ipvx = ipaddress.ip_address(ip) if ipvx.version == 4: - ip_ooi = IPAddressV4( - address=ip, - network=network, - ) + ip_ooi = IPAddressV4(address=ip, network=network) else: - ip_ooi = IPAddressV6( - address=ip, - network=network, - ) + ip_ooi = IPAddressV6(address=ip, network=network) yield ip_ooi ip_ref = ip_ooi.reference - ip_port_ooi = IPPort( - address=ip_ref, - protocol=Protocol(protocol), - port=port_nr, - state=PortState("open"), - ) + ip_port_ooi = IPPort(address=ip_ref, protocol=Protocol(protocol), port=port_nr, state=PortState("open")) yield ip_port_ooi # TODO: result.data.server_info {openssl_cipher_string_supported,highest_ssl_version_supported,ja3,ja3_digest} diff --git a/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalize.py b/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalize.py index 3c73064ae04..3c276e9e4ac 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalize.py +++ b/boefjes/boefjes/plugins/kat_binaryedge/remote_desktop/normalize.py @@ -30,24 +30,13 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: else: ipvx = ipaddress.ip_address(ip) if ipvx.version == 4: - ip_ooi = IPAddressV4( - address=ip, - network=network, - ) + ip_ooi = IPAddressV4(address=ip, network=network) else: - ip_ooi = IPAddressV6( - address=ip, - network=network, - ) + ip_ooi = IPAddressV6(address=ip, network=network) yield ip_ooi ip_ref = ip_ooi.reference - ip_port_ooi = IPPort( - address=ip_ref, - protocol=Protocol(protocol), - port=port_nr, - state=PortState("open"), - ) + ip_port_ooi = IPPort(address=ip_ref, protocol=Protocol(protocol), port=port_nr, state=PortState("open")) yield ip_port_ooi service_name = "" diff --git a/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalize.py b/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalize.py index cf247ffde5b..08edfaed22d 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalize.py +++ b/boefjes/boefjes/plugins/kat_binaryedge/service_identification/normalize.py @@ -32,24 +32,13 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: else: ipvx = ipaddress.ip_address(ip) if ipvx.version == 4: - ip_ooi = IPAddressV4( - address=ip, - network=network, - ) + ip_ooi = IPAddressV4(address=ip, network=network) else: - ip_ooi = IPAddressV6( - address=ip, - network=network, - ) + ip_ooi = IPAddressV6(address=ip, network=network) yield ip_ooi ip_ref = ip_ooi.reference - ip_port_ooi = IPPort( - address=ip_ref, - protocol=Protocol(protocol), - port=port_nr, - state=PortState("open"), - ) + ip_port_ooi = IPPort(address=ip_ref, protocol=Protocol(protocol), port=port_nr, state=PortState("open")) yield ip_port_ooi if "service" in scan["result"]["data"]: @@ -65,8 +54,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: software_ooi = Software(name=get_name_from_cpe(cpe), cpe=cpe) yield software_ooi software_instance_ooi = SoftwareInstance( - ooi=ip_service_ooi.reference, - software=software_ooi.reference, + ooi=ip_service_ooi.reference, software=software_ooi.reference ) yield software_instance_ooi @@ -90,8 +78,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: yield software_ooi software_instance_ooi = SoftwareInstance( - ooi=ip_service_ooi.reference, - software=software_ooi.reference, + ooi=ip_service_ooi.reference, software=software_ooi.reference ) yield software_instance_ooi diff --git a/boefjes/boefjes/plugins/kat_binaryedge/services/normalize.py b/boefjes/boefjes/plugins/kat_binaryedge/services/normalize.py index c6795a46e2e..b4570abe843 100644 --- a/boefjes/boefjes/plugins/kat_binaryedge/services/normalize.py +++ b/boefjes/boefjes/plugins/kat_binaryedge/services/normalize.py @@ -42,24 +42,13 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: else: ipvx = ipaddress.ip_address(ip) if ipvx.version == 4: - ip_ooi = IPAddressV4( - address=ip, - network=network, - ) + ip_ooi = IPAddressV4(address=ip, network=network) else: - ip_ooi = IPAddressV6( - address=ip, - network=network, - ) + ip_ooi = IPAddressV6(address=ip, network=network) yield ip_ooi ip_ref = ip_ooi.reference - ip_port_ooi = IPPort( - address=ip_ref, - protocol=Protocol(protocol), - port=port_nr, - state=PortState("open"), - ) + ip_port_ooi = IPPort(address=ip_ref, protocol=Protocol(protocol), port=port_nr, state=PortState("open")) yield ip_port_ooi if module == "ssh": @@ -71,10 +60,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: yield software_ooi yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference) elif module == "rsync": - software_ooi = Software( - name=module.upper(), - version=scan.get("result", {}).get("data", {}).get("version"), - ) + software_ooi = Software(name=module.upper(), version=scan.get("result", {}).get("data", {}).get("version")) yield software_ooi yield SoftwareInstance(ooi=ip_port_ooi.reference, software=software_ooi.reference) elif module == "telnet": diff --git a/boefjes/boefjes/plugins/kat_calvin/normalize.py b/boefjes/boefjes/plugins/kat_calvin/normalize.py index c79b4f1525e..1766ad5430e 100644 --- a/boefjes/boefjes/plugins/kat_calvin/normalize.py +++ b/boefjes/boefjes/plugins/kat_calvin/normalize.py @@ -24,10 +24,4 @@ def parse_log(log: dict) -> Iterable[NormalizerOutput]: "event_type": log.pop("eventType"), } - yield NormalizerDeclaration( - ooi=Incident( - application=app.reference, - **mandatory_fields, - meta_data=log, - ), - ) + yield NormalizerDeclaration(ooi=Incident(application=app.reference, **mandatory_fields, meta_data=log)) diff --git a/boefjes/boefjes/plugins/kat_censys/normalize.py b/boefjes/boefjes/plugins/kat_censys/normalize.py index 8f6f30c11a2..0222014e09b 100644 --- a/boefjes/boefjes/plugins/kat_censys/normalize.py +++ b/boefjes/boefjes/plugins/kat_censys/normalize.py @@ -46,11 +46,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: elif "subject" in certificate["leaf_data"]: so = certificate["leaf_data"]["subject_dn"] cert_subject = "C={}, ST={}, O={}, OU={}, CN={}".format( - so["country"], - so["province"], - so["organization"], - so["organizational_unit"], - so["common_name"], + so["country"], so["province"], so["organization"], so["organizational_unit"], so["common_name"] ) else: cert_subject = "n/a" @@ -89,9 +85,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: url = urllib.parse.urlparse(scan["http"]["request"]["uri"]) port = 443 if url.scheme == "https" else 80 ip_port = IPPort( - address=ip_ooi_reference, - protocol=Protocol[scan["transport_protocol"]], - port=port, + address=ip_ooi_reference, protocol=Protocol[scan["transport_protocol"]], port=port ) yield ip_port @@ -111,19 +105,12 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: # todo: implement `HTTPResource.redirects_to` if available http_resource = HTTPResource( website=Website( - ip_service=IPService( - ip_port=ip_port.reference, - service=service.reference, - ).reference, + ip_service=IPService(ip_port=ip_port.reference, service=service.reference).reference, hostname=hostname.reference, ).reference, web_url=web_url.reference, ) yield http_resource - http_header = HTTPHeader( - resource=http_resource.reference, - key=header_field, - value=value, - ) + http_header = HTTPHeader(resource=http_resource.reference, key=header_field, value=value) yield http_header diff --git a/boefjes/boefjes/plugins/kat_dicom/normalize.py b/boefjes/boefjes/plugins/kat_dicom/normalize.py index 7b04d3dd16a..1c68844ee95 100644 --- a/boefjes/boefjes/plugins/kat_dicom/normalize.py +++ b/boefjes/boefjes/plugins/kat_dicom/normalize.py @@ -13,12 +13,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: ooi = Reference.from_str(input_ooi["primary_key"]) for port in results["open_ports"]: - ip_port_ooi = IPPort( - address=ooi, - protocol=Protocol("tcp"), - port=int(port), - state=PortState("open"), - ) + ip_port_ooi = IPPort(address=ooi, protocol=Protocol("tcp"), port=int(port), state=PortState("open")) yield ip_port_ooi software_ooi = Software(name="DICOM") diff --git a/boefjes/boefjes/plugins/kat_dns/main.py b/boefjes/boefjes/plugins/kat_dns/main.py index 635611d29b7..0e2d22065b2 100644 --- a/boefjes/boefjes/plugins/kat_dns/main.py +++ b/boefjes/boefjes/plugins/kat_dns/main.py @@ -12,20 +12,7 @@ from boefjes.job_models import BoefjeMeta logger = logging.getLogger(__name__) -DEFAULT_RECORD_TYPES = { - "A", - "AAAA", - "CAA", - "CERT", - "RP", - "SRV", - "TXT", - "MX", - "NS", - "CNAME", - "DNAME", - "SOA", -} +DEFAULT_RECORD_TYPES = {"A", "AAAA", "CAA", "CERT", "RP", "SRV", "TXT", "MX", "NS", "CNAME", "DNAME", "SOA"} class ZoneNotFoundException(Exception): @@ -36,12 +23,7 @@ def get_record_types() -> list[str]: requested_record_types = getenv("RECORD_TYPES", "") if not requested_record_types: return list(DEFAULT_RECORD_TYPES) - requested_record_types = list( - map( - lambda x: re.sub(r"[^A-Za-z]", "", x), - requested_record_types.upper().split(","), - ) - ) + requested_record_types = list(map(lambda x: re.sub(r"[^A-Za-z]", "", x), requested_record_types.upper().split(","))) return list(set(requested_record_types).intersection(DEFAULT_RECORD_TYPES)) @@ -54,13 +36,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: resolver.nameservers = [nameserver] record_types = get_record_types() - answers = ( - [ - get_parent_zone_soa(resolver, requested_dns_name), - ] - if "SOA" in record_types - else [] - ) + answers = [get_parent_zone_soa(resolver, requested_dns_name)] if "SOA" in record_types else [] for type_ in record_types: try: diff --git a/boefjes/boefjes/plugins/kat_dns/normalize.py b/boefjes/boefjes/plugins/kat_dns/normalize.py index f769e09108f..d70ecc99d72 100644 --- a/boefjes/boefjes/plugins/kat_dns/normalize.py +++ b/boefjes/boefjes/plugins/kat_dns/normalize.py @@ -54,10 +54,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: record_store: dict[str, DNSRecord] = {} def register_hostname(name: str) -> Hostname: - hostname = Hostname( - network=internet.reference, - name=name.rstrip("."), - ) + hostname = Hostname(network=internet.reference, name=name.rstrip(".")) hostname_store[hostname.name] = hostname return hostname @@ -77,17 +74,11 @@ def register_record(record: DNSRecord) -> DNSRecord: rr: Rdata record_hostname = register_hostname(str(rrset.name)) - default_args = { - "hostname": record_hostname.reference, - "value": str(rr), - "ttl": rrset.ttl, - } + default_args = {"hostname": record_hostname.reference, "value": str(rr), "ttl": rrset.ttl} # the soa is the zone of itself, and the argument hostname if isinstance(rr, SOA): - zone = DNSZone( - hostname=record_hostname.reference, - ) + zone = DNSZone(hostname=record_hostname.reference) zone_links[record_hostname.name] = zone zone_links[input_hostname.name] = zone @@ -110,12 +101,7 @@ def register_record(record: DNSRecord) -> DNSRecord: if isinstance(rr, AAAA): ipv6 = IPAddressV6(network=internet.reference, address=IPv6Address(str(rr))) yield ipv6 - register_record( - DNSAAAARecord( - address=ipv6.reference, - **default_args, - ) - ) + register_record(DNSAAAARecord(address=ipv6.reference, **default_args)) if isinstance(rr, TXT): # TODO: concatenated txt records should be handled better @@ -130,30 +116,16 @@ def register_record(record: DNSRecord) -> DNSRecord: mail_hostname_reference = mail_fqdn.reference register_record( - DNSMXRecord( - mail_hostname=mail_hostname_reference, - preference=rr.preference, - **default_args, - ) + DNSMXRecord(mail_hostname=mail_hostname_reference, preference=rr.preference, **default_args) ) if isinstance(rr, NS): ns_fqdn = register_hostname(str(rr.target)) - register_record( - DNSNSRecord( - name_server_hostname=ns_fqdn.reference, - **default_args, - ) - ) + register_record(DNSNSRecord(name_server_hostname=ns_fqdn.reference, **default_args)) if isinstance(rr, CNAME): target_fqdn = register_hostname(str(rr.target)) - register_record( - DNSCNAMERecord( - target_hostname=target_fqdn.reference, - **default_args, - ) - ) + register_record(DNSCNAMERecord(target_hostname=target_fqdn.reference, **default_args)) if isinstance(rr, CAA): record_value = str(rr).split(" ", 2) @@ -174,9 +146,7 @@ def register_record(record: DNSRecord) -> DNSRecord: # DKIM dkim_results = results["dkim_response"] if dkim_results not in ["NXDOMAIN", "Timeout"] and dkim_results.split("\n")[2] == "rcode NOERROR": - yield DKIMExists( - hostname=input_hostname.reference, - ) + yield DKIMExists(hostname=input_hostname.reference) # DMARC dmarc_results = results["dmarc_response"] @@ -185,8 +155,4 @@ def register_record(record: DNSRecord) -> DNSRecord: for rr in rrset: rr: Rdata if isinstance(rr, TXT): - yield DMARCTXTRecord( - hostname=input_hostname.reference, - value=str(rr).strip('"'), - ttl=rrset.ttl, - ) + yield DMARCTXTRecord(hostname=input_hostname.reference, value=str(rr).strip('"'), ttl=rrset.ttl) diff --git a/boefjes/boefjes/plugins/kat_dns_zone/normalize.py b/boefjes/boefjes/plugins/kat_dns_zone/normalize.py index 86073ea7686..cefe6ee8961 100644 --- a/boefjes/boefjes/plugins/kat_dns_zone/normalize.py +++ b/boefjes/boefjes/plugins/kat_dns_zone/normalize.py @@ -18,10 +18,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: lines = section.split("\n") message: Message = from_text("\n".join(lines[1:])) - input_zone_hostname = Hostname( - network=internet.reference, - name=input_ooi["hostname"]["name"], - ) + input_zone_hostname = Hostname(network=internet.reference, name=input_ooi["hostname"]["name"]) input_zone = DNSZone(hostname=input_zone_hostname.reference) diff --git a/boefjes/boefjes/plugins/kat_external_db/normalize.py b/boefjes/boefjes/plugins/kat_external_db/normalize.py index 0d3f490d979..54373060940 100644 --- a/boefjes/boefjes/plugins/kat_external_db/normalize.py +++ b/boefjes/boefjes/plugins/kat_external_db/normalize.py @@ -54,11 +54,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: addresses_count += 1 if mask < interface.ip.max_prefixlen: - block = block_type( - start_ip=ip_address.reference, - mask=mask, - network=network.reference, - ) + block = block_type(start_ip=ip_address.reference, mask=mask, network=network.reference) yield block yield DeclaredScanProfile(reference=block.reference, level=3) blocks_count += 1 diff --git a/boefjes/boefjes/plugins/kat_fierce/fierce.py b/boefjes/boefjes/plugins/kat_fierce/fierce.py index 9bee57c175f..7f2560c73d1 100644 --- a/boefjes/boefjes/plugins/kat_fierce/fierce.py +++ b/boefjes/boefjes/plugins/kat_fierce/fierce.py @@ -115,12 +115,7 @@ def query(resolver, domain, record_type="A", tcp=False): return query(resolver, domain, record_type, tcp=tcp) return None - except ( - dns.resolver.NXDOMAIN, - dns.resolver.NoNameservers, - dns.exception.Timeout, - ValueError, - ): + except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.exception.Timeout, ValueError): return None @@ -210,8 +205,7 @@ def find_nearby(resolver, ips, filter_func=None): reversed_ips = { ip: query_result for ip, query_result in zip( - str_ips, - executor.map(reverse_query, itertools.repeat(resolver, len(str_ips)), str_ips), + str_ips, executor.map(reverse_query, itertools.repeat(resolver, len(str_ips)), str_ips) ) } @@ -274,10 +268,7 @@ def fierce(**kwargs): if kwargs.get("range"): range_ips = range_expander(kwargs.get("range")) - nearby = find_nearby( - resolver, - range_ips, - ) + nearby = find_nearby(resolver, range_ips) if nearby: pass @@ -365,11 +356,7 @@ def parse_args(args): ) p.add_argument("--domain", action="store", help="domain name to test") - p.add_argument( - "--connect", - action="store_true", - help="attempt HTTP connection to non-RFC 1918 hosts", - ) + p.add_argument("--connect", action="store_true", help="attempt HTTP connection to non-RFC 1918 hosts") p.add_argument("--wide", action="store_true", help="scan entire class c of discovered records") p.add_argument( "--traverse", @@ -378,20 +365,9 @@ def parse_args(args): default=5, help="scan IPs near discovered records, this won't enter adjacent class c's", ) - p.add_argument( - "--search", - action="store", - nargs="+", - help="filter on these domains when expanding lookup", - ) + p.add_argument("--search", action="store", nargs="+", help="filter on these domains when expanding lookup") p.add_argument("--range", action="store", help="scan an internal IP range, use cidr notation") - p.add_argument( - "--delay", - action="store", - type=float, - default=None, - help="time to wait between lookups", - ) + p.add_argument("--delay", action="store", type=float, default=None, help="time to wait between lookups") subdomain_group = p.add_mutually_exclusive_group() subdomain_group.add_argument("--subdomains", action="store", nargs="+", help="use these subdomains") @@ -403,16 +379,9 @@ def parse_args(args): ) dns_group = p.add_mutually_exclusive_group() + dns_group.add_argument("--dns-servers", action="store", nargs="+", help="use these dns servers for reverse lookups") dns_group.add_argument( - "--dns-servers", - action="store", - nargs="+", - help="use these dns servers for reverse lookups", - ) - dns_group.add_argument( - "--dns-file", - action="store", - help="use dns servers specified in this file for reverse lookups (one per line)", + "--dns-file", action="store", help="use dns servers specified in this file for reverse lookups (one per line)" ) p.add_argument("--tcp", action="store_true", help="use TCP instead of UDP") diff --git a/boefjes/boefjes/plugins/kat_leakix/normalize.py b/boefjes/boefjes/plugins/kat_leakix/normalize.py index 5e6374cd4d7..828d7a8aa79 100644 --- a/boefjes/boefjes/plugins/kat_leakix/normalize.py +++ b/boefjes/boefjes/plugins/kat_leakix/normalize.py @@ -105,10 +105,7 @@ def handle_ip(event, network_reference, as_ooi_reference): if as_ooi_reference and len(netblock_range) == 2: yield block_type( - network=network_reference, - start_ip=ip_ooi.reference, - mask=netblock_range[1], - announced_by=as_ooi_reference, + network=network_reference, start_ip=ip_ooi.reference, mask=netblock_range[1], announced_by=as_ooi_reference ) # Store port diff --git a/boefjes/boefjes/plugins/kat_masscan/main.py b/boefjes/boefjes/plugins/kat_masscan/main.py index 340cabdff7d..c16b1746fb7 100644 --- a/boefjes/boefjes/plugins/kat_masscan/main.py +++ b/boefjes/boefjes/plugins/kat_masscan/main.py @@ -57,12 +57,7 @@ def get_file_from_container(container: docker.models.containers.Container, path: try: stream, _ = container.get_archive(path) except docker.errors.NotFound: - logging.warning( - "[Masscan] %s not found in container %s %s", - path, - container.short_id, - container.image.tags, - ) + logging.warning("[Masscan] %s not found in container %s %s", path, container.short_id, container.image.tags) return None f = tarfile.open(mode="r|", fileobj=TarStream(stream).reader()) @@ -83,9 +78,7 @@ def run_masscan(target_ip) -> bytes: max_rate = os.getenv("MAX_RATE", 100) logging.info("Starting container %s to run masscan...", IMAGE) res = client.containers.run( - image=IMAGE, - command=f"-p {port_range} --max-rate {max_rate} -oJ {FILE_PATH} {target_ip}", - detach=True, + image=IMAGE, command=f"-p {port_range} --max-rate {max_rate} -oJ {FILE_PATH} {target_ip}", detach=True ) res.wait() logging.debug(res.logs()) diff --git a/boefjes/boefjes/plugins/kat_nmap_ip_range/main.py b/boefjes/boefjes/plugins/kat_nmap_ip_range/main.py index a8827384b03..6ccf43fe33d 100644 --- a/boefjes/boefjes/plugins/kat_nmap_ip_range/main.py +++ b/boefjes/boefjes/plugins/kat_nmap_ip_range/main.py @@ -24,17 +24,7 @@ def build_nmap_arguments(ip_range: IPv6Network | IPv4Network, top_ports: int, pr if not TOP_PORTS_MIN <= top_ports <= TOP_PORTS_MAX: raise ValueError(f"{TOP_PORTS_MIN} <= TOP_PORTS: {top_ports} <= {TOP_PORTS_MAX} is invalid.") - args = [ - "nmap", - "--open", - "-T4", - "-Pn", - "-r", - "-v10", - f"-s{protocol_str}", - "--top-ports", - str(top_ports), - ] + args = ["nmap", "--open", "-T4", "-Pn", "-r", "-v10", f"-s{protocol_str}", "--top-ports", str(top_ports)] if ip_range.version == 6: args.append("-6") args.extend(["-oX", "-", str(ip_range)]) @@ -53,11 +43,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: min_mask = int(getenv("MIN_VLSM_IPV6", 118)) if ip_range.prefixlen < min_mask: - logging.info( - "Minimum expected VLSM %d > %d, skipping this range.", - min_mask, - ip_range.prefixlen, - ) + logging.info("Minimum expected VLSM %d > %d, skipping this range.", min_mask, ip_range.prefixlen) return [({"info/boefje"}, "Skipping range due to unaccepted VSLM.")] top_ports_tcp = int(getenv("TOP_PORTS_TCP", 250)) diff --git a/boefjes/boefjes/plugins/kat_nmap_ports/main.py b/boefjes/boefjes/plugins/kat_nmap_ports/main.py index 45cc2850660..6ae46402a96 100644 --- a/boefjes/boefjes/plugins/kat_nmap_ports/main.py +++ b/boefjes/boefjes/plugins/kat_nmap_ports/main.py @@ -29,10 +29,7 @@ def build_nmap_arguments(host: str, ports: str) -> list[str]: return args -def validate_ports( - ports, - valid=re.compile(NMAP_VALID_PORTS), -) -> str: +def validate_ports(ports, valid=re.compile(NMAP_VALID_PORTS)) -> str: """Returns ports argument if valid. Double slashes are for flake8 W605. A valid port is: diff --git a/boefjes/boefjes/plugins/kat_nmap_tcp/normalize.py b/boefjes/boefjes/plugins/kat_nmap_tcp/normalize.py index 3ee20712f53..cd3d3ccc49a 100644 --- a/boefjes/boefjes/plugins/kat_nmap_tcp/normalize.py +++ b/boefjes/boefjes/plugins/kat_nmap_tcp/normalize.py @@ -28,10 +28,7 @@ def get_ip_ports_and_service(host: NmapHost, network: Network, netblock: Referen continue ip_port = IPPort( - address=ip.reference, - protocol=Protocol(protocol), - port=port, - state=PortState(service.state), + address=ip.reference, protocol=Protocol(protocol), port=port, state=PortState(service.state) ) yield ip_port diff --git a/boefjes/boefjes/plugins/kat_nuclei_cve/main.py b/boefjes/boefjes/plugins/kat_nuclei_cve/main.py index 981866284fd..4a9f6c27d11 100644 --- a/boefjes/boefjes/plugins/kat_nuclei_cve/main.py +++ b/boefjes/boefjes/plugins/kat_nuclei_cve/main.py @@ -22,9 +22,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: # Checks if the url is of object HostnameHTTPURL or Hostname url = verify_hostname_meta(boefje_meta.arguments["input"]) output = client.containers.run( - NUCLEI_IMAGE, - ["-t", "/root/nuclei-templates/http/cves/", "-u", url, "-jsonl"], - remove=True, + NUCLEI_IMAGE, ["-t", "/root/nuclei-templates/http/cves/", "-u", url, "-jsonl"], remove=True ) return [(set(), output)] diff --git a/boefjes/boefjes/plugins/kat_nuclei_exposed_panels/main.py b/boefjes/boefjes/plugins/kat_nuclei_exposed_panels/main.py index a85e6a29406..9c8b782a804 100644 --- a/boefjes/boefjes/plugins/kat_nuclei_exposed_panels/main.py +++ b/boefjes/boefjes/plugins/kat_nuclei_exposed_panels/main.py @@ -22,9 +22,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: # Checks if the url is of object HostnameHTTPURL or Hostname url = verify_hostname_meta(boefje_meta.arguments["input"]) output = client.containers.run( - NUCLEI_IMAGE, - ["-t", "/root/nuclei-templates/http/exposed-panels/", "-u", url, "-jsonl"], - remove=True, + NUCLEI_IMAGE, ["-t", "/root/nuclei-templates/http/exposed-panels/", "-u", url, "-jsonl"], remove=True ) return [(set(), output)] diff --git a/boefjes/boefjes/plugins/kat_nuclei_take_over/main.py b/boefjes/boefjes/plugins/kat_nuclei_take_over/main.py index d6630eaaba7..c23c48ac207 100644 --- a/boefjes/boefjes/plugins/kat_nuclei_take_over/main.py +++ b/boefjes/boefjes/plugins/kat_nuclei_take_over/main.py @@ -22,9 +22,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: # Checks if the url is of object HostnameHTTPURL or Hostname url = verify_hostname_meta(boefje_meta.arguments["input"]) output = client.containers.run( - NUCLEI_IMAGE, - ["-t", "/root/nuclei-templates/http/takeovers/", "-u", url, "-jsonl"], - remove=True, + NUCLEI_IMAGE, ["-t", "/root/nuclei-templates/http/takeovers/", "-u", url, "-jsonl"], remove=True ) return [(set(), output)] diff --git a/boefjes/boefjes/plugins/kat_rpki/main.py b/boefjes/boefjes/plugins/kat_rpki/main.py index bbf247a0101..8d9db21d722 100644 --- a/boefjes/boefjes/plugins/kat_rpki/main.py +++ b/boefjes/boefjes/plugins/kat_rpki/main.py @@ -47,13 +47,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: results = {"vrps_records": roas, "notexpired": notexpired, "exists": exists} - return [ - (set(), json.dumps(results)), - ( - {"rpki/cache-meta"}, - json.dumps(rpki_meta), - ), - ] + return [(set(), json.dumps(results)), ({"rpki/cache-meta"}, json.dumps(rpki_meta))] def create_hash(data: bytes, algo: str) -> str: diff --git a/boefjes/boefjes/plugins/kat_security_txt_downloader/main.py b/boefjes/boefjes/plugins/kat_security_txt_downloader/main.py index 0ca93f35932..48538de257c 100644 --- a/boefjes/boefjes/plugins/kat_security_txt_downloader/main.py +++ b/boefjes/boefjes/plugins/kat_security_txt_downloader/main.py @@ -55,10 +55,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: def do_request(hostname: str, session: Session, uri: str, useragent: str): response = session.get( - uri, - headers={"Host": hostname, "User-Agent": useragent}, - verify=False, - allow_redirects=False, + uri, headers={"Host": hostname, "User-Agent": useragent}, verify=False, allow_redirects=False ) return response diff --git a/boefjes/boefjes/plugins/kat_shodan/normalize.py b/boefjes/boefjes/plugins/kat_shodan/normalize.py index 97c44f659bf..418ff280620 100644 --- a/boefjes/boefjes/plugins/kat_shodan/normalize.py +++ b/boefjes/boefjes/plugins/kat_shodan/normalize.py @@ -21,12 +21,7 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: port_nr = scan["port"] transport = scan["transport"] - ip_port = IPPort( - address=ooi, - protocol=Protocol(transport), - port=int(port_nr), - state=PortState("open"), - ) + ip_port = IPPort(address=ooi, protocol=Protocol(transport), port=int(port_nr), state=PortState("open")) yield ip_port if "vulns" in scan: diff --git a/boefjes/boefjes/plugins/kat_snyk/main.py b/boefjes/boefjes/plugins/kat_snyk/main.py index f587d00d2b6..ec80e7d6fac 100755 --- a/boefjes/boefjes/plugins/kat_snyk/main.py +++ b/boefjes/boefjes/plugins/kat_snyk/main.py @@ -12,11 +12,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: software_name = input_["name"] software_version = input_["version"] - result = { - "table_versions": [], - "table_vulnerabilities": [], - "cve_vulnerabilities": [], - } + result = {"table_versions": [], "table_vulnerabilities": [], "cve_vulnerabilities": []} url_snyk = f"https://snyk.io/vuln/npm:{software_name.lower().replace(' ', '-')}" page = requests.get(url_snyk, timeout=30) soup = BeautifulSoup(page.content, "html.parser") @@ -53,10 +49,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: if cve_code != " ": result["cve_vulnerabilities"].append( - { - "cve_code": cve_code, - "Vuln_text": parsed_info["Vuln_text"], - } + {"cve_code": cve_code, "Vuln_text": parsed_info["Vuln_text"]} ) else: result["table_vulnerabilities"].append(parsed_info) diff --git a/boefjes/boefjes/plugins/kat_snyk/normalize.py b/boefjes/boefjes/plugins/kat_snyk/normalize.py index cd8b5746234..72e1538e612 100755 --- a/boefjes/boefjes/plugins/kat_snyk/normalize.py +++ b/boefjes/boefjes/plugins/kat_snyk/normalize.py @@ -27,19 +27,11 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: for vuln in results["table_vulnerabilities"]: snyk_ft = SnykFindingType(id=vuln.get("Vuln_href")) yield snyk_ft - yield Finding( - finding_type=snyk_ft.reference, - ooi=pk_ooi, - description=vuln.get("Vuln_text"), - ) + yield Finding(finding_type=snyk_ft.reference, ooi=pk_ooi, description=vuln.get("Vuln_text")) for vuln in results["cve_vulnerabilities"]: cve_ft = CVEFindingType(id=vuln.get("cve_code")) yield cve_ft - yield Finding( - finding_type=cve_ft.reference, - ooi=pk_ooi, - description=vuln.get("Vuln_text"), - ) + yield Finding(finding_type=cve_ft.reference, ooi=pk_ooi, description=vuln.get("Vuln_text")) # Check for latest version latest_version = "" diff --git a/boefjes/boefjes/plugins/kat_ssl_certificates/main.py b/boefjes/boefjes/plugins/kat_ssl_certificates/main.py index b33dcb1c652..58a506e99bc 100644 --- a/boefjes/boefjes/plugins/kat_ssl_certificates/main.py +++ b/boefjes/boefjes/plugins/kat_ssl_certificates/main.py @@ -19,17 +19,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: try: output = client.containers.run( OPENSSL_IMAGE, - [ - "s_client", - "-host", - ip_address, - "-port", - port, - "-prexit", - "-showcerts", - "-servername", - hostname, - ], + ["s_client", "-host", ip_address, "-port", port, "-prexit", "-showcerts", "-servername", hostname], remove=True, ) except docker.errors.ContainerError as e: diff --git a/boefjes/boefjes/plugins/kat_ssl_certificates/normalize.py b/boefjes/boefjes/plugins/kat_ssl_certificates/normalize.py index bf20fa48421..52d2e7a9ae8 100644 --- a/boefjes/boefjes/plugins/kat_ssl_certificates/normalize.py +++ b/boefjes/boefjes/plugins/kat_ssl_certificates/normalize.py @@ -99,9 +99,7 @@ def read_certificates( certificate_subject_alternative_names = [] hostnames = [] for m in re.finditer( - r"(?<=-----BEGIN CERTIFICATE-----).*?(?=-----END CERTIFICATE-----)", - contents, - flags=re.DOTALL, + r"(?<=-----BEGIN CERTIFICATE-----).*?(?=-----END CERTIFICATE-----)", contents, flags=re.DOTALL ): pem_contents = f"-----BEGIN CERTIFICATE-----{m.group()}-----END CERTIFICATE-----" @@ -122,10 +120,7 @@ def read_certificates( pk_algorithm = "" pk_size = cert.public_key().key_size logging.info("Parsing certificate of type %s", type(cert.public_key())) - if isinstance( - cert.public_key(), - rsa.RSAPublicKey, - ): + if isinstance(cert.public_key(), rsa.RSAPublicKey): pk_algorithm = str(AlgorithmType.RSA) pk_number = cert.public_key().public_numbers().n.to_bytes(pk_size // 8, "big").hex() elif isinstance(cert.public_key(), ec.EllipticCurvePublicKey): diff --git a/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/main.py b/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/main.py index 7fea6fe6b60..7c8ecef47e0 100644 --- a/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/main.py +++ b/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/main.py @@ -24,18 +24,10 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: timeout = getenv("TIMEOUT", 30) - environment_vars = { - "OPENSSL_TIMEOUT": timeout, - "CONNECT_TIMEOUT": timeout, - } + environment_vars = {"OPENSSL_TIMEOUT": timeout, "CONNECT_TIMEOUT": timeout} client = docker.from_env() - container = client.containers.run( - SSL_TEST_IMAGE, - args, - detach=True, - environment=environment_vars, - ) + container = client.containers.run(SSL_TEST_IMAGE, args, detach=True, environment=environment_vars) try: container.wait(timeout=300) diff --git a/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/normalize.py b/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/normalize.py index b173df5ee56..5c280671f2c 100644 --- a/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/normalize.py +++ b/boefjes/boefjes/plugins/kat_testssl_sh_ciphers/normalize.py @@ -30,11 +30,7 @@ def parse_cipher(cipher: dict) -> dict | None: ) else: cipher_dict[parts[0]].update( - { - "encryption_algorithm": parts[4], - "bits": int(parts[5]), - "cipher_suite_alias": parts[6], - } + {"encryption_algorithm": parts[4], "bits": int(parts[5]), "cipher_suite_alias": parts[6]} ) return cipher_dict diff --git a/boefjes/boefjes/plugins/kat_webpage_analysis/find_images_in_html/normalize.py b/boefjes/boefjes/plugins/kat_webpage_analysis/find_images_in_html/normalize.py index 4f58e6bd084..f78e5c2962f 100644 --- a/boefjes/boefjes/plugins/kat_webpage_analysis/find_images_in_html/normalize.py +++ b/boefjes/boefjes/plugins/kat_webpage_analysis/find_images_in_html/normalize.py @@ -23,7 +23,4 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: if not validators.url(img): img = urljoin(url, img) - yield URL( - network=Network(name=network_name).reference, - raw=img, - ) + yield URL(network=Network(name=network_name).reference, raw=img) diff --git a/boefjes/boefjes/plugins/kat_webpage_analysis/headers/normalize.py b/boefjes/boefjes/plugins/kat_webpage_analysis/headers/normalize.py index 426f7005165..3d2a320a5f3 100644 --- a/boefjes/boefjes/plugins/kat_webpage_analysis/headers/normalize.py +++ b/boefjes/boefjes/plugins/kat_webpage_analysis/headers/normalize.py @@ -11,8 +11,4 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: resource = Reference.from_str(input_ooi["primary_key"]) for key, value in json.loads(raw).items(): - yield HTTPHeader( - resource=resource, - key=key, - value=value, - ) + yield HTTPHeader(resource=resource, key=key, value=value) diff --git a/boefjes/boefjes/plugins/kat_webpage_analysis/main.py b/boefjes/boefjes/plugins/kat_webpage_analysis/main.py index e8accd74d7b..f9fef3921fa 100644 --- a/boefjes/boefjes/plugins/kat_webpage_analysis/main.py +++ b/boefjes/boefjes/plugins/kat_webpage_analysis/main.py @@ -39,15 +39,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: else: url_parts = url_parts._replace(netloc=f"[{ip}]") if addr.version == 6 else url_parts._replace(netloc=ip) - uri = urlunsplit( - [ - url_parts.scheme, - url_parts.netloc, - url_parts.path, - url_parts.query, - url_parts.fragment, - ] - ) + uri = urlunsplit([url_parts.scheme, url_parts.netloc, url_parts.path, url_parts.query, url_parts.fragment]) body_mimetypes = {"openkat-http/body"} try: @@ -97,10 +89,7 @@ def create_response_object(response: requests.Response) -> dict: def do_request(hostname: str, session: Session, uri: str, useragent: str): response = session.get( - uri, - headers={"Host": hostname, "User-Agent": useragent}, - verify=False, - allow_redirects=False, + uri, headers={"Host": hostname, "User-Agent": useragent}, verify=False, allow_redirects=False ) return response diff --git a/boefjes/boefjes/plugins/kat_wpscan/main.py b/boefjes/boefjes/plugins/kat_wpscan/main.py index 1056198d886..a117322013e 100644 --- a/boefjes/boefjes/plugins/kat_wpscan/main.py +++ b/boefjes/boefjes/plugins/kat_wpscan/main.py @@ -27,14 +27,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: url = f"{scheme}://{hostname}{path}" - argv = [ - "--url", - url, - "--format", - "json", - "--plugins-version-detection", - "aggressive", - ] + argv = ["--url", url, "--format", "json", "--plugins-version-detection", "aggressive"] if wpscan_api_token := getenv("WP_SCAN_API"): argv += ["--api-token", wpscan_api_token] @@ -43,11 +36,7 @@ def run(boefje_meta: BoefjeMeta) -> list[tuple[set, bytes | str]]: client.images.pull(WPSCAN_IMAGE) # since WPScan can give positive exit codes on completion, docker-py's run() can fail on this - container = client.containers.run( - WPSCAN_IMAGE, - argv, - detach=True, - ) + container = client.containers.run(WPSCAN_IMAGE, argv, detach=True) try: # wait for container to exit, read its output in the logs and remove container diff --git a/boefjes/boefjes/plugins/kat_wpscan/normalize.py b/boefjes/boefjes/plugins/kat_wpscan/normalize.py index 88cbfa79178..6acb682fd76 100644 --- a/boefjes/boefjes/plugins/kat_wpscan/normalize.py +++ b/boefjes/boefjes/plugins/kat_wpscan/normalize.py @@ -24,8 +24,6 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: ft = CVEFindingType(id=id_) yield ft finding = Finding( - finding_type=ft.reference, - ooi=url_reference, - description=alert.splitlines()[0], + finding_type=ft.reference, ooi=url_reference, description=alert.splitlines()[0] ) yield finding diff --git a/boefjes/boefjes/sql/db_models.py b/boefjes/boefjes/sql/db_models.py index 04a284a9dba..68307f81d95 100644 --- a/boefjes/boefjes/sql/db_models.py +++ b/boefjes/boefjes/sql/db_models.py @@ -33,11 +33,7 @@ class RepositoryInDB(SQL_BASE): class SettingsInDB(SQL_BASE): __tablename__ = "settings" __table_args__ = ( - UniqueConstraint( - "organisation_pk", - "plugin_id", - name="unique_settings_per_organisation_per_plugin", - ), + UniqueConstraint("organisation_pk", "plugin_id", name="unique_settings_per_organisation_per_plugin"), ) id = Column(Integer, primary_key=True, autoincrement=True) @@ -51,12 +47,7 @@ class SettingsInDB(SQL_BASE): class PluginStateInDB(SQL_BASE): __tablename__ = "plugin_state" __table_args__ = ( - UniqueConstraint( - "plugin_id", - "organisation_pk", - "repository_pk", - name="unique_plugin_per_repo_per_org", - ), + UniqueConstraint("plugin_id", "organisation_pk", "repository_pk", name="unique_plugin_per_repo_per_org"), ) id = Column(Integer, primary_key=True, autoincrement=True) diff --git a/boefjes/boefjes/sql/organisation_storage.py b/boefjes/boefjes/sql/organisation_storage.py index d772b890e3e..64b90edecf0 100644 --- a/boefjes/boefjes/sql/organisation_storage.py +++ b/boefjes/boefjes/sql/organisation_storage.py @@ -36,11 +36,7 @@ def create(self, organisation: Organisation) -> None: self.session.add(organisation_in_db) def add_repository(self, organisation_id: str, repository_id: str) -> None: - logger.info( - "Adding repository to organisation: %s -> %s", - organisation_id, - repository_id, - ) + logger.info("Adding repository to organisation: %s -> %s", organisation_id, repository_id) organisation_in_db = self._db_instance_by_id(organisation_id) repo_in_db = self._db_repo_instance_by_id(repository_id) @@ -76,17 +72,11 @@ def _db_repo_instance_by_id(self, repository_id: str) -> RepositoryInDB: @staticmethod def to_organisation_in_db(organisation: Organisation) -> OrganisationInDB: - return OrganisationInDB( - id=organisation.id, - name=organisation.name, - ) + return OrganisationInDB(id=organisation.id, name=organisation.name) @staticmethod def to_organisation(organisation_in_db: OrganisationInDB) -> Organisation: - return Organisation( - id=organisation_in_db.id, - name=organisation_in_db.name, - ) + return Organisation(id=organisation_in_db.id, name=organisation_in_db.name) def create_organisation_storage(session) -> SQLOrganisationStorage: diff --git a/boefjes/boefjes/sql/plugin_enabled_storage.py b/boefjes/boefjes/sql/plugin_enabled_storage.py index 677b68e1dc3..00f622f2d3f 100644 --- a/boefjes/boefjes/sql/plugin_enabled_storage.py +++ b/boefjes/boefjes/sql/plugin_enabled_storage.py @@ -80,10 +80,7 @@ def _db_instance_by_id(self, plugin_id: str, repository_id: str, organisation_id if instance is None: raise PluginNotFound(plugin_id, repository_id, organisation_id) from ObjectNotFoundException( - PluginStateInDB, - plugin_id=plugin_id, - organisation_id=organisation_id, - repository_id=repository_id, + PluginStateInDB, plugin_id=plugin_id, organisation_id=organisation_id, repository_id=repository_id ) return instance @@ -106,16 +103,11 @@ def to_plugin_state_in_db( ) return PluginStateInDB( - plugin_id=plugin_id, - enabled=enabled, - organisation_pk=organisation.pk, - repository_pk=repository.pk, + plugin_id=plugin_id, enabled=enabled, organisation_pk=organisation.pk, repository_pk=repository.pk ) -def create_plugin_enabled_storage( - session: Session | None = None, -) -> SQLPluginEnabledStorage: +def create_plugin_enabled_storage(session: Session | None = None) -> SQLPluginEnabledStorage: if not session: session = sessionmaker(bind=get_engine())() diff --git a/boefjes/boefjes/sql/repository_storage.py b/boefjes/boefjes/sql/repository_storage.py index 9ba65769630..abfe7aa68d9 100644 --- a/boefjes/boefjes/sql/repository_storage.py +++ b/boefjes/boefjes/sql/repository_storage.py @@ -52,19 +52,11 @@ def _db_instance_by_id(self, repository_id: str) -> RepositoryInDB: @staticmethod def to_repository_in_db(repository: Repository) -> RepositoryInDB: - return RepositoryInDB( - id=repository.id, - name=repository.name, - base_url=str(repository.base_url), - ) + return RepositoryInDB(id=repository.id, name=repository.name, base_url=str(repository.base_url)) @staticmethod def to_repository(repository_in_db: RepositoryInDB) -> Repository: - return Repository( - id=repository_in_db.id, - name=repository_in_db.name, - base_url=repository_in_db.base_url, - ) + return Repository(id=repository_in_db.id, name=repository_in_db.name, base_url=repository_in_db.base_url) def create_repository_storage(session) -> SQLRepositoryStorage: diff --git a/boefjes/boefjes/sql/setting_storage.py b/boefjes/boefjes/sql/setting_storage.py index f7c8d182fdc..c2056d8b0ac 100644 --- a/boefjes/boefjes/sql/setting_storage.py +++ b/boefjes/boefjes/sql/setting_storage.py @@ -29,11 +29,7 @@ def upsert(self, values: dict, organisation_id: str, plugin_id: str) -> None: except SettingsNotFound: organisation = self.session.query(OrganisationInDB).filter(OrganisationInDB.id == organisation_id).first() - setting_in_db = SettingsInDB( - values=encrypted_values, - plugin_id=plugin_id, - organisation_pk=organisation.pk, - ) + setting_in_db = SettingsInDB(values=encrypted_values, plugin_id=plugin_id, organisation_pk=organisation.pk) self.session.add(setting_in_db) def get_all(self, organisation_id: str, plugin_id: str) -> dict: @@ -76,8 +72,7 @@ def create_encrypter(): encrypter: EncryptMiddleware = IdentityMiddleware() if get_context().env.encryption_middleware == EncryptionMiddleware.NACL_SEALBOX: encrypter = NaclBoxMiddleware( - get_context().env.katalogus_private_key_b64, - get_context().env.katalogus_public_key_b64, + get_context().env.katalogus_private_key_b64, get_context().env.katalogus_public_key_b64 ) return encrypter diff --git a/boefjes/tests/loading.py b/boefjes/tests/loading.py index 3eae8bc6b5a..b3cc401780a 100644 --- a/boefjes/tests/loading.py +++ b/boefjes/tests/loading.py @@ -28,8 +28,7 @@ def get_boefje_meta( def get_normalizer_meta( - boefje_meta: BoefjeMeta = get_boefje_meta(), - raw_file_id: UUID = UUID("2c9f47db-dfca-4928-b29f-368e64b3c779"), + boefje_meta: BoefjeMeta = get_boefje_meta(), raw_file_id: UUID = UUID("2c9f47db-dfca-4928-b29f-368e64b3c779") ) -> NormalizerMeta: return NormalizerMeta( id=UUID("203eedee-a590-43e1-8f80-6d18ffe529f5"), @@ -41,11 +40,8 @@ def get_normalizer_meta( def get_raw_data_meta( - raw_file_id: UUID = UUID("2c9f47db-dfca-4928-b29f-368e64b3c779"), - boefje_meta: BoefjeMeta = get_boefje_meta(), + raw_file_id: UUID = UUID("2c9f47db-dfca-4928-b29f-368e64b3c779"), boefje_meta: BoefjeMeta = get_boefje_meta() ) -> RawDataMeta: return RawDataMeta( - id=raw_file_id, - boefje_meta=boefje_meta, - mime_types=[{"value": "boefje_id/test"}, {"value": "text/plain"}], + id=raw_file_id, boefje_meta=boefje_meta, mime_types=[{"value": "boefje_id/test"}, {"value": "text/plain"}] ) diff --git a/boefjes/tests/modules/dummy_bad_normalizer_dict_structure/normalize.py b/boefjes/tests/modules/dummy_bad_normalizer_dict_structure/normalize.py index 98ee732bb71..f82dce28c0f 100644 --- a/boefjes/tests/modules/dummy_bad_normalizer_dict_structure/normalize.py +++ b/boefjes/tests/modules/dummy_bad_normalizer_dict_structure/normalize.py @@ -4,7 +4,4 @@ def run(input_ooi: dict, raw: bytes) -> Iterable[NormalizerOutput]: - yield { - "I": "write", - "bad": "normalizers", - } + yield {"I": "write", "bad": "normalizers"} diff --git a/boefjes/tests/test_cve_finding_types.py b/boefjes/tests/test_cve_finding_types.py index a67ac217e20..c526c132882 100644 --- a/boefjes/tests/test_cve_finding_types.py +++ b/boefjes/tests/test_cve_finding_types.py @@ -13,12 +13,7 @@ class CVETest(TestCase): def test_cve_with_cvss(self): input_ooi = {"id": "CVE-2021-46882"} - oois = list( - run( - input_ooi, - get_dummy_data("inputs/cve-result-with-cvss.json"), - ) - ) + oois = list(run(input_ooi, get_dummy_data("inputs/cve-result-with-cvss.json"))) expected = [ NormalizerAffirmation( @@ -38,12 +33,7 @@ def test_cve_with_cvss(self): def test_cve_with_cvss2(self): input_ooi = {"id": "CVE-2016-0616"} - oois = list( - run( - input_ooi, - get_dummy_data("inputs/cve-result-with-cvss2.json"), - ) - ) + oois = list(run(input_ooi, get_dummy_data("inputs/cve-result-with-cvss2.json"))) expected = [ NormalizerAffirmation( @@ -64,12 +54,7 @@ def test_cve_with_cvss2(self): def test_cve_without_cvss(self): input_ooi = {"id": "CVE-2021-46882"} - oois = list( - run( - input_ooi, - get_dummy_data("inputs/cve-result-without-cvss.json"), - ) - ) + oois = list(run(input_ooi, get_dummy_data("inputs/cve-result-without-cvss.json"))) expected = [ NormalizerAffirmation( @@ -83,7 +68,7 @@ def test_cve_without_cvss(self): source="https://cve.circl.lu/cve/CVE-2021-46882", risk_severity=RiskLevelSeverity.UNKNOWN, risk_score=None, - ), + ) ) ] diff --git a/boefjes/tests/test_dns.py b/boefjes/tests/test_dns.py index 555db001734..72de3c639f3 100644 --- a/boefjes/tests/test_dns.py +++ b/boefjes/tests/test_dns.py @@ -39,50 +39,24 @@ def test_dns_normalizer(self): IPAddressV4(network=internet.reference, address=IPv4Address("94.198.159.36")), ] dns_a_records = [ - DNSARecord( - hostname=zone_hostname.reference, - value=str(ip.address), - address=ip.reference, - ttl=14364, - ) + DNSARecord(hostname=zone_hostname.reference, value=str(ip.address), address=ip.reference, ttl=14364) for ip in ip_v4_addresses ] ip_v6_addresses = [ - IPAddressV6( - network=internet.reference, - address=IPv6Address("2a00:d78:0:712:94:198:159:35"), - ), - IPAddressV6( - network=internet.reference, - address=IPv6Address("2a00:d78:0:712:94:198:159:36"), - ), + IPAddressV6(network=internet.reference, address=IPv6Address("2a00:d78:0:712:94:198:159:35")), + IPAddressV6(network=internet.reference, address=IPv6Address("2a00:d78:0:712:94:198:159:36")), ] dns_aaaa_records = [ - DNSAAAARecord( - hostname=zone_hostname.reference, - value=str(ip.address), - address=ip.reference, - ttl=14400, - ) + DNSAAAARecord(hostname=zone_hostname.reference, value=str(ip.address), address=ip.reference, ttl=14400) for ip in ip_v6_addresses ] dns_txt_records = [ - DNSTXTRecord( - hostname=zone_hostname.reference, - value="v=spf1 redirect=spf-a.example.nl", - ttl=14400, - ) + DNSTXTRecord(hostname=zone_hostname.reference, value="v=spf1 redirect=spf-a.example.nl", ttl=14400) ] mx_hostnames = [ - Hostname( - network=internet.reference, - name="mail.example.nl", - ), - Hostname( - network=internet.reference, - name="mail2.example.nl", - ), + Hostname(network=internet.reference, name="mail.example.nl"), + Hostname(network=internet.reference, name="mail2.example.nl"), ] dns_mx_records = [ DNSMXRecord( @@ -97,12 +71,7 @@ def test_dns_normalizer(self): ns_hostnames = [ Hostname(name=value, network=internet.reference) - for value in [ - "ns3.examplenl.org", - "ns1.examplenl.nl", - "ns2.examplenl.eu", - "ns0.examplenl.com", - ] + for value in ["ns3.examplenl.org", "ns1.examplenl.nl", "ns2.examplenl.eu", "ns0.examplenl.com"] ] ns_records = [ @@ -115,11 +84,7 @@ def test_dns_normalizer(self): for ns_hostname in ns_hostnames ] - soa_hostname = Hostname( - network=internet.reference, - name="ns1.examplenl.nl", - dns_zone=zone.reference, - ) + soa_hostname = Hostname(network=internet.reference, name="ns1.examplenl.nl", dns_zone=zone.reference) soa_record = DNSSOARecord( hostname=zone_hostname.reference, value="ns1.examplenl.nl. hostmaster.sidn.nl. 2021111101 14400 7200 1209600 86400", @@ -161,24 +126,12 @@ def test_dns_normalizer(self): def test_dns_normalizer_cname(self): internet = Network(name="internet") - zone_hostname = Hostname( - network=internet.reference, - name="example.nl", - ) - zone = DNSZone( - hostname=zone_hostname.reference, - ) + zone_hostname = Hostname(network=internet.reference, name="example.nl") + zone = DNSZone(hostname=zone_hostname.reference) zone_hostname.dns_zone = zone.reference - input_hostname = Hostname( - network=internet.reference, - name="www.example.nl", - dns_zone=zone.reference, - ) - cname_target = Hostname( - network=internet.reference, - name="webredir.examplenl.nl", - ) + input_hostname = Hostname(network=internet.reference, name="www.example.nl", dns_zone=zone.reference) + cname_target = Hostname(network=internet.reference, name="webredir.examplenl.nl") soa_hostname = Hostname(network=internet.reference, name="ns1.examplenl.nl") soa_record = DNSSOARecord( @@ -202,10 +155,7 @@ def test_dns_normalizer_cname(self): ip_address = IPAddressV4(network=internet.reference, address=IPv4Address("94.198.159.35")) dns_a_record = DNSARecord( - hostname=cname_target.reference, - address=ip_address.reference, - value=str(ip_address.address), - ttl=10800, + hostname=cname_target.reference, address=ip_address.reference, value=str(ip_address.address), ttl=10800 ) expected = [ @@ -230,10 +180,7 @@ def test_dns_normalizer_cname(self): boefje=Boefje(id="dns-records"), organization="_dev", input_ooi="Hostname|internet|www.example.nl", - arguments={ - "domain": "www.example.nl.", - "input": {"name": "www.example.nl."}, - }, + arguments={"domain": "www.example.nl.", "input": {"name": "www.example.nl."}}, ), mime_types=[{"value": "boefje/dns-records"}], ), @@ -258,10 +205,7 @@ def test_parse_record_null_mx_record(self): boefje=Boefje(id="dns-records"), organization="_dev", input_ooi="Hostname|internet|english.example.nl", - arguments={ - "domain": "english.example.nl", - "input": {"name": "english.example.nl"}, - }, + arguments={"domain": "english.example.nl", "input": {"name": "english.example.nl"}}, ), mime_types=[{"value": "boefje/dns-records"}], ), @@ -270,24 +214,13 @@ def test_parse_record_null_mx_record(self): answer = get_dummy_data("inputs/dns-result-mx-example.nl.json") internet = Network(name="internet") - input_hostname = Hostname( - network=internet.reference, - name="english.example.nl", - ) + input_hostname = Hostname(network=internet.reference, name="english.example.nl") cname_target = Hostname(network=internet.reference, name="redir.example.nl") cname_record = DNSCNAMERecord( - hostname=input_hostname.reference, - value="redir.example.nl.", - ttl=60, - target_hostname=cname_target.reference, - ) - mx_record = DNSMXRecord( - hostname=cname_target.reference, - value="0 .", - ttl=14400, - preference=0, + hostname=input_hostname.reference, value="redir.example.nl.", ttl=60, target_hostname=cname_target.reference ) + mx_record = DNSMXRecord(hostname=cname_target.reference, value="0 .", ttl=14400, preference=0) local_repository = LocalPluginRepository(Path(__file__).parent.parent / "boefjes" / "plugins") runner = LocalNormalizerJobRunner(local_repository) @@ -301,41 +234,22 @@ def test_parse_record_null_mx_record(self): def test_parse_cname_soa(self): internet = Network(name="internet") - zone_hostname = Hostname( - network=internet.reference, - name="example.com", - ) - zone = DNSZone( - hostname=zone_hostname.reference, - ) + zone_hostname = Hostname(network=internet.reference, name="example.com") + zone = DNSZone(hostname=zone_hostname.reference) zone_hostname.dns_zone = zone.reference - input_hostname = Hostname( - network=internet.reference, - name="www.example.com", - dns_zone=zone.reference, - ) + input_hostname = Hostname(network=internet.reference, name="www.example.com", dns_zone=zone.reference) cname_record = DNSCNAMERecord( - hostname=input_hostname.reference, - value="example.com.", - ttl=60, - target_hostname=zone_hostname.reference, + hostname=input_hostname.reference, value="example.com.", ttl=60, target_hostname=zone_hostname.reference ) ip_address = IPAddressV4(network=internet.reference, address=IPv4Address("94.198.159.35")) a_record = DNSARecord( - hostname=zone_hostname.reference, - address=ip_address.reference, - value=str(ip_address.address), - ttl=60, + hostname=zone_hostname.reference, address=ip_address.reference, value=str(ip_address.address), ttl=60 ) soa_hostname = Hostname(network=internet.reference, name="ns.icann.org") ns_hostnames = [ - Hostname(name=value, network=internet.reference) - for value in [ - "a.iana-servers.net", - "b.iana-servers.net", - ] + Hostname(name=value, network=internet.reference) for value in ["a.iana-servers.net", "b.iana-servers.net"] ] ns_records = [ DNSNSRecord( @@ -357,11 +271,7 @@ def test_parse_cname_soa(self): expire=1209600, minimum=3600, ) - txt_record = DNSTXTRecord( - hostname=zone_hostname.reference, - value="v=spf1 -all", - ttl=60, - ) + txt_record = DNSTXTRecord(hostname=zone_hostname.reference, value="v=spf1 -all", ttl=60) mx_record = DNSMXRecord( hostname=zone_hostname.reference, value="0 example.com.", @@ -380,10 +290,7 @@ def test_parse_cname_soa(self): boefje=Boefje(id="dns-records"), organization="_dev", input_ooi="Hostname|internet|www.example.com", - arguments={ - "domain": "www.example.com", - "input": {"name": "www.example.com"}, - }, + arguments={"domain": "www.example.com", "input": {"name": "www.example.com"}}, ), mime_types=[{"value": "boefje/dns-records"}], ), @@ -415,25 +322,14 @@ def test_parse_cname_soa(self): def test_find_parent_dns_zone(self): internet = Network(name="internet") - requested_zone = DNSZone( - hostname=Hostname( - network=internet.reference, - name="sub.example.nl", - ).reference - ) - parent_zone_hostname = Hostname( - network=internet.reference, - name="example.nl", - ) + requested_zone = DNSZone(hostname=Hostname(network=internet.reference, name="sub.example.nl").reference) + parent_zone_hostname = Hostname(network=internet.reference, name="example.nl") parent_zone = DNSZone(hostname=parent_zone_hostname.reference) parent_zone_hostname.dns_zone = parent_zone.reference requested_zone.parent = parent_zone.reference - name_server_hostname = Hostname( - network=internet.reference, - name="ns1.examplenl.nl", - ) + name_server_hostname = Hostname(network=internet.reference, name="ns1.examplenl.nl") soa_record = DNSSOARecord( hostname=parent_zone_hostname.reference, @@ -448,12 +344,7 @@ def test_find_parent_dns_zone(self): ) input_ = serialize_ooi( - DNSZone( - hostname=Hostname( - network=Reference.from_str("Network|internet"), - name="sub.example.nl", - ).reference - ) + DNSZone(hostname=Hostname(network=Reference.from_str("Network|internet"), name="sub.example.nl").reference) ) meta = NormalizerMeta( @@ -475,13 +366,7 @@ def test_find_parent_dns_zone(self): runner = LocalNormalizerJobRunner(local_repository) results = runner.run(meta, get_dummy_data("inputs/dns-zone-result-sub.example.nl.txt")) - expected = [ - requested_zone, - parent_zone, - parent_zone_hostname, - name_server_hostname, - soa_record, - ] + expected = [requested_zone, parent_zone, parent_zone_hostname, name_server_hostname, soa_record] self.assertCountEqual( map(BaseModel.model_dump, expected), map(BaseModel.model_dump, results.observations[0].results) ) diff --git a/boefjes/tests/test_fierce.py b/boefjes/tests/test_fierce.py index 697c18bf19a..de13388da03 100644 --- a/boefjes/tests/test_fierce.py +++ b/boefjes/tests/test_fierce.py @@ -13,21 +13,16 @@ def test_fierce(): expected = { ResolvedHostname( - hostname=Reference("Hostname|internet|www.example.com"), - address=Reference("IPAddressV4|internet|192.0.2.2"), + hostname=Reference("Hostname|internet|www.example.com"), address=Reference("IPAddressV4|internet|192.0.2.2") ), ResolvedHostname( hostname=Reference("Hostname|internet|subdomain.example.com"), address=Reference("IPAddressV4|internet|192.0.2.3"), ), ResolvedHostname( - hostname=Reference("Hostname|internet|ipv6.example.com"), - address=Reference("IPAddressV6|internet|ff02::1"), - ), - Hostname( - network=Reference("Network|internet"), - name="example.com", + hostname=Reference("Hostname|internet|ipv6.example.com"), address=Reference("IPAddressV6|internet|ff02::1") ), + Hostname(network=Reference("Network|internet"), name="example.com"), Hostname( registered_domain=Reference("Hostname|internet|example.com"), network=Reference("Network|internet"), @@ -43,18 +38,9 @@ def test_fierce(): network=Reference("Network|internet"), name="ipv6.example.com", ), - IPAddressV4( - address=IPv4Address("192.0.2.3"), - network=Reference("Network|internet"), - ), - IPAddressV4( - address=IPv4Address("192.0.2.2"), - network=Reference("Network|internet"), - ), - IPAddressV6( - address=IPv6Address("ff02::1"), - network=Reference("Network|internet"), - ), + IPAddressV4(address=IPv4Address("192.0.2.3"), network=Reference("Network|internet")), + IPAddressV4(address=IPv4Address("192.0.2.2"), network=Reference("Network|internet")), + IPAddressV6(address=IPv6Address("ff02::1"), network=Reference("Network|internet")), Network(name="internet"), } diff --git a/boefjes/tests/test_manual.py b/boefjes/tests/test_manual.py index 9dfcfd215fa..dfc1b484a86 100644 --- a/boefjes/tests/test_manual.py +++ b/boefjes/tests/test_manual.py @@ -129,9 +129,7 @@ def test_parse_url_csv(self): "network": Reference("Network|internet"), "object_type": "URL", "primary_key": "URL|internet|https://example.com/", - "raw": Url( - "https://example.com/", - ), + "raw": Url("https://example.com/"), "scan_profile": None, "web_url": None, }, @@ -145,9 +143,7 @@ def test_parse_url_csv(self): "network": Reference("Network|internet"), "object_type": "URL", "primary_key": "URL|internet|https://example.com/", - "raw": Url( - "https://example.com/", - ), + "raw": Url("https://example.com/"), "scan_profile": None, "web_url": None, }, diff --git a/boefjes/tests/test_rdns.py b/boefjes/tests/test_rdns.py index 2f0980766ce..aa16faf0e93 100644 --- a/boefjes/tests/test_rdns.py +++ b/boefjes/tests/test_rdns.py @@ -4,10 +4,7 @@ from octopoes.models.ooi.dns.zone import Hostname from tests.loading import get_dummy_data -input_ooi = { - "primary_key": "IPAddressV4|internet|192.0.2.1", - "network": {"name": "internet"}, -} +input_ooi = {"primary_key": "IPAddressV4|internet|192.0.2.1", "network": {"name": "internet"}} def test_rdns_nxdomain(): diff --git a/boefjes/tests/test_report_data.py b/boefjes/tests/test_report_data.py index ea6434c5c74..bfb1bbf2822 100644 --- a/boefjes/tests/test_report_data.py +++ b/boefjes/tests/test_report_data.py @@ -17,12 +17,7 @@ def test_report_data(): ooi_dict = json.loads(raw) declaration = NormalizerDeclaration( - ooi={ - "object_type": "ReportData", - "scan_profile": None, - "primary_key": "ReportData|test", - **ooi_dict, - } + ooi={"object_type": "ReportData", "scan_profile": None, "primary_key": "ReportData|test", **ooi_dict} ) assert output.observations == [] diff --git a/boefjes/tests/test_security_txt.py b/boefjes/tests/test_security_txt.py index 8241c8656db..49fe2a9d546 100644 --- a/boefjes/tests/test_security_txt.py +++ b/boefjes/tests/test_security_txt.py @@ -30,12 +30,7 @@ class SecurityTXTTest(TestCase): maxDiff = None def test_security_txt_same_website(self): - oois = list( - run( - input_ooi, - get_dummy_data("inputs/security_txt_result_same_website.json"), - ) - ) + oois = list(run(input_ooi, get_dummy_data("inputs/security_txt_result_same_website.json"))) expected = [] expected.append( @@ -55,12 +50,7 @@ def test_security_txt_same_website(self): self.assertEqual(expected, oois) def test_security_txt_different_website(self): - oois = list( - run( - input_ooi, - get_dummy_data("inputs/security_txt_result_different_website.json"), - ) - ) + oois = list(run(input_ooi, get_dummy_data("inputs/security_txt_result_different_website.json"))) expected = [] url_original = URL( diff --git a/boefjes/tests/test_snyk.py b/boefjes/tests/test_snyk.py index 6de75179347..2976d7dc48c 100644 --- a/boefjes/tests/test_snyk.py +++ b/boefjes/tests/test_snyk.py @@ -8,25 +8,14 @@ from octopoes.models.types import CVEFindingType, Finding, Software from tests.loading import get_dummy_data -input_ooi = { - "primary_key": "Software|lodash|1.1.0|", - "software": { - "name": "lodash", - "version": "1.1.0", - }, -} +input_ooi = {"primary_key": "Software|lodash|1.1.0|", "software": {"name": "lodash", "version": "1.1.0"}} class SnykTest(TestCase): maxDiff = None def test_snyk_no_findings(self): - oois = list( - run( - input_ooi, - get_dummy_data("inputs/snyk-result-no-findings.json"), - ) - ) + oois = list(run(input_ooi, get_dummy_data("inputs/snyk-result-no-findings.json"))) # noinspection PyTypeChecker expected = () @@ -34,12 +23,7 @@ def test_snyk_no_findings(self): self.assertCountEqual(expected, oois) def test_snyk_findings(self): - oois = list( - run( - input_ooi, - get_dummy_data("inputs/snyk-result-findings.json"), - ) - ) + oois = list(run(input_ooi, get_dummy_data("inputs/snyk-result-findings.json"))) software = Software(name="lodash", version="1.1.0") @@ -66,23 +50,13 @@ def test_snyk_findings(self): snyk_ft = SnykFindingType(id=finding[0]) snyk_findingtypes.append(snyk_ft) snyk_findings.append( - Finding( - finding_type=snyk_ft.reference, - ooi=software.reference, - description=finding[1], - ) + Finding(finding_type=snyk_ft.reference, ooi=software.reference, description=finding[1]) ) for finding in cve_finding_data: cve_ft = CVEFindingType(id=finding[0]) cve_findingtypes.append(cve_ft) - cve_findings.append( - Finding( - finding_type=cve_ft.reference, - ooi=software.reference, - description=finding[1], - ) - ) + cve_findings.append(Finding(finding_type=cve_ft.reference, ooi=software.reference, description=finding[1])) # noinspection PyTypeChecker expected = snyk_findingtypes + snyk_findings + cve_findingtypes + cve_findings diff --git a/boefjes/tests/test_tasks.py b/boefjes/tests/test_tasks.py index d2973477d59..5bb7d89294e 100644 --- a/boefjes/tests/test_tasks.py +++ b/boefjes/tests/test_tasks.py @@ -20,29 +20,11 @@ class TaskTest(TestCase): def setUp(self) -> None: self.boefjes = [ + Boefje(id="test-boefje-1", repository_id="", consumes={"SomeOOI"}, produces=["test-boef-1", "test/text"]), + Boefje(id="test-boefje-2", repository_id="", consumes={"SomeOOI"}, produces=["test-boef-2", "test/text"]), + Boefje(id="test-boefje-3", repository_id="", consumes={"SomeOOI"}, produces=["test-boef-3", "test/plain"]), Boefje( - id="test-boefje-1", - repository_id="", - consumes={"SomeOOI"}, - produces=["test-boef-1", "test/text"], - ), - Boefje( - id="test-boefje-2", - repository_id="", - consumes={"SomeOOI"}, - produces=["test-boef-2", "test/text"], - ), - Boefje( - id="test-boefje-3", - repository_id="", - consumes={"SomeOOI"}, - produces=["test-boef-3", "test/plain"], - ), - Boefje( - id="test-boefje-4", - repository_id="", - consumes={"SomeOOI"}, - produces=["test-boef-4", "test/and-simple"], + id="test-boefje-4", repository_id="", consumes={"SomeOOI"}, produces=["test-boef-4", "test/and-simple"] ), ] self.normalizers = [ @@ -52,21 +34,10 @@ def setUp(self) -> None: consumes=["test-boef-3", "test/text"], produces=["SomeOOI", "OtherOOI"], ), - Normalizer( - id="test-normalizer-2", - repository_id="", - consumes=["test/text"], - produces=["SomeOtherOOI"], - ), + Normalizer(id="test-normalizer-2", repository_id="", consumes=["test/text"], produces=["SomeOtherOOI"]), ] self.bits = [ - Bit( - id="test-bit-1", - repository_id="", - consumes="SomeOOI", - produces=["SomeOOI"], - parameters=[], - ), + Bit(id="test-bit-1", repository_id="", consumes="SomeOOI", produces=["SomeOOI"], parameters=[]), Bit( id="test-bit-2", repository_id="", @@ -119,10 +90,7 @@ def test_handle_boefje_with_exception(self, mock_get_octopoes_api_connector, moc assert raw_call_args[0][0] == UUID("0dca59db-b339-47c4-bcc9-896fc18e2386") assert "Traceback (most recent call last)" in raw_call_args[0][1] assert "JobRuntimeError: Boefje failed" in raw_call_args[0][1] - assert raw_call_args[0][2] == { - "error/boefje", - "boefje/dummy_boefje_runtime_exception", - } + assert raw_call_args[0][2] == {"error/boefje", "boefje/dummy_boefje_runtime_exception"} def test_exception_raised_unsupported_return_type_normalizer(self): meta = NormalizerMeta.model_validate_json(get_dummy_data("dns-normalize.json")) diff --git a/boefjes/tests/test_testssl_sh.py b/boefjes/tests/test_testssl_sh.py index 67debdb735d..c693c7f924c 100644 --- a/boefjes/tests/test_testssl_sh.py +++ b/boefjes/tests/test_testssl_sh.py @@ -21,12 +21,7 @@ class TestsslSh(TestCase): maxDiff = None def test_cipherless_service(self): - oois = list( - run( - input_ooi, - get_dummy_data("inputs/testssl-sh-cipherless.json"), - ) - ) + oois = list(run(input_ooi, get_dummy_data("inputs/testssl-sh-cipherless.json"))) # noinspection PyTypeChecker expected = [] @@ -34,12 +29,7 @@ def test_cipherless_service(self): self.assertEqual(expected, oois) def test_ciphered_service(self): - oois = list( - run( - input_ooi, - get_dummy_data("inputs/testssl-sh-ciphered.json"), - ) - ) + oois = list(run(input_ooi, get_dummy_data("inputs/testssl-sh-ciphered.json"))) # noinspection PyTypeChecker expected_suites = { diff --git a/bytes/bytes/api/root.py b/bytes/bytes/api/root.py index d26468ee2e1..5fde88664ed 100644 --- a/bytes/bytes/api/root.py +++ b/bytes/bytes/api/root.py @@ -32,12 +32,7 @@ class ServiceHealth(BaseModel): def validation_exception_handler(_: Request, exc: RequestValidationError | ValidationError) -> JSONResponse: logger.critical(exc) - return JSONResponse( - { - "value": str(exc), - }, - status.HTTP_422_UNPROCESSABLE_ENTITY, - ) + return JSONResponse({"value": str(exc)}, status.HTTP_422_UNPROCESSABLE_ENTITY) @router.get("/", include_in_schema=False) @@ -63,8 +58,4 @@ def metrics(meta_repository: MetaDataRepository = Depends(create_meta_data_repos def login(form_data: OAuth2PasswordRequestForm = Depends()) -> TokenResponse: access_token, expire_time = get_access_token(form_data) - return TokenResponse( - access_token=access_token, - token_type="bearer", - expires_at=expire_time.isoformat(), - ) + return TokenResponse(access_token=access_token, token_type="bearer", expires_at=expire_time.isoformat()) diff --git a/bytes/bytes/api/router.py b/bytes/bytes/api/router.py index d6961c91dcd..e7177fd81e2 100644 --- a/bytes/bytes/api/router.py +++ b/bytes/bytes/api/router.py @@ -25,8 +25,7 @@ @router.post("/boefje_meta", tags=[BOEFJE_META_TAG]) def create_boefje_meta( - boefje_meta: BoefjeMeta, - meta_repository: MetaDataRepository = Depends(create_meta_data_repository), + boefje_meta: BoefjeMeta, meta_repository: MetaDataRepository = Depends(create_meta_data_repository) ) -> JSONResponse: try: with meta_repository: @@ -41,8 +40,7 @@ def create_boefje_meta( @router.get("/boefje_meta/{boefje_meta_id}", response_model=BoefjeMeta, tags=[BOEFJE_META_TAG]) def get_boefje_meta_by_id( - boefje_meta_id: UUID, - meta_repository: MetaDataRepository = Depends(create_meta_data_repository), + boefje_meta_id: UUID, meta_repository: MetaDataRepository = Depends(create_meta_data_repository) ) -> BoefjeMeta: with meta_repository: meta = meta_repository.get_boefje_meta_by_id(boefje_meta_id) @@ -86,8 +84,7 @@ def get_boefje_meta( @router.post("/normalizer_meta", tags=[NORMALIZER_META_TAG]) def create_normalizer_meta( - normalizer_meta: NormalizerMeta, - meta_repository: MetaDataRepository = Depends(create_meta_data_repository), + normalizer_meta: NormalizerMeta, meta_repository: MetaDataRepository = Depends(create_meta_data_repository) ) -> JSONResponse: try: with meta_repository: @@ -102,8 +99,7 @@ def create_normalizer_meta( @router.get("/normalizer_meta/{normalizer_meta_id}", response_model=NormalizerMeta, tags=[NORMALIZER_META_TAG]) def get_normalizer_meta_by_id( - normalizer_meta_id: UUID, - meta_repository: MetaDataRepository = Depends(create_meta_data_repository), + normalizer_meta_id: UUID, meta_repository: MetaDataRepository = Depends(create_meta_data_repository) ) -> NormalizerMeta: with meta_repository: return meta_repository.get_normalizer_meta_by_id(normalizer_meta_id) @@ -167,11 +163,7 @@ async def create_raw( event = RawFileReceived( organization=meta.organization, - raw_data=RawDataMeta( - id=raw_id, - boefje_meta=raw_data.boefje_meta, - mime_types=raw_data.mime_types, - ), + raw_data=RawDataMeta(id=raw_id, boefje_meta=raw_data.boefje_meta, mime_types=raw_data.mime_types), ) event_manager.publish(event) except Exception as error: @@ -182,10 +174,7 @@ async def create_raw( @router.get("/raw/{raw_id}", tags=[RAW_TAG]) -def get_raw_by_id( - raw_id: UUID, - meta_repository: MetaDataRepository = Depends(create_meta_data_repository), -) -> Response: +def get_raw_by_id(raw_id: UUID, meta_repository: MetaDataRepository = Depends(create_meta_data_repository)) -> Response: try: raw_data = meta_repository.get_raw_by_id(raw_id) except ObjectNotFoundException as error: @@ -196,8 +185,7 @@ def get_raw_by_id( @router.get("/raw/{raw_id}/meta", tags=[RAW_TAG]) def get_raw_meta_by_id( - raw_id: UUID, - meta_repository: MetaDataRepository = Depends(create_meta_data_repository), + raw_id: UUID, meta_repository: MetaDataRepository = Depends(create_meta_data_repository) ) -> RawDataMeta: try: raw_meta = meta_repository.get_raw_meta_by_id(raw_id) @@ -268,8 +256,7 @@ def ignore_arguments_key(meta_repository: MetaDataRepository, query_filter: RawD ) def cached_counts_per_mime_type(meta_repository: MetaDataRepository, query_filter: RawDataFilter) -> dict[str, int]: logger.debug( - "Metrics cache miss for cached_counts_per_mime_type, ttl set to %s seconds", - get_settings().metrics_ttl_seconds, + "Metrics cache miss for cached_counts_per_mime_type, ttl set to %s seconds", get_settings().metrics_ttl_seconds ) return meta_repository.get_raw_file_count_per_mime_type(query_filter) diff --git a/bytes/bytes/auth.py b/bytes/bytes/auth.py index 29e3808fe7e..348bd027a64 100644 --- a/bytes/bytes/auth.py +++ b/bytes/bytes/auth.py @@ -65,10 +65,7 @@ def _create_access_token( form_data: OAuth2PasswordRequestForm, secret: str, access_token_expire_minutes: float ) -> tuple[str, datetime]: expire_time = _get_expire_time(access_token_expire_minutes) - data = { - "sub": form_data.username, - "exp": expire_time, - } + data = {"sub": form_data.username, "exp": expire_time} access_token = jwt.encode(data.copy(), secret, algorithm=ALGORITHM) diff --git a/bytes/bytes/database/migrations/env.py b/bytes/bytes/database/migrations/env.py index 6c731d7d72f..f91d8947d18 100644 --- a/bytes/bytes/database/migrations/env.py +++ b/bytes/bytes/database/migrations/env.py @@ -38,10 +38,7 @@ def run_migrations_offline() -> None: """ url = config.get_main_option("sqlalchemy.url") context.configure( - url=url, - target_metadata=target_metadata, - literal_binds=True, - dialect_opts={"paramstyle": "named"}, + url=url, target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"} ) with context.begin_transaction(): @@ -56,17 +53,11 @@ def run_migrations_online() -> None: """ connectable = engine_from_config( - config.get_section(config.config_ini_section), - prefix="sqlalchemy.", - poolclass=pool.NullPool, + config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool ) with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=target_metadata, - compare_type=True, - ) + context.configure(connection=connection, target_metadata=target_metadata, compare_type=True) with context.begin_transaction(): context.run_migrations() diff --git a/bytes/bytes/database/migrations/versions/0001_initial.py b/bytes/bytes/database/migrations/versions/0001_initial.py index 2a0512abc30..4a3cbbd6c20 100644 --- a/bytes/bytes/database/migrations/versions/0001_initial.py +++ b/bytes/bytes/database/migrations/versions/0001_initial.py @@ -40,10 +40,7 @@ def upgrade() -> None: sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True), sa.Column("boefje_meta_id", postgresql.UUID(), nullable=False), - sa.ForeignKeyConstraint( - ["boefje_meta_id"], - ["boefje_meta.id"], - ), + sa.ForeignKeyConstraint(["boefje_meta_id"], ["boefje_meta.id"]), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("id"), ) @@ -51,10 +48,7 @@ def upgrade() -> None: "output_ooi", sa.Column("ooi_id", sa.String(), nullable=False), sa.Column("normalizer_meta_id", postgresql.UUID(), nullable=False), - sa.ForeignKeyConstraint( - ["normalizer_meta_id"], - ["normalizer_meta.id"], - ), + sa.ForeignKeyConstraint(["normalizer_meta_id"], ["normalizer_meta.id"]), sa.PrimaryKeyConstraint("ooi_id", "normalizer_meta_id"), ) # ### end Alembic commands ### diff --git a/bytes/bytes/database/sql_meta_repository.py b/bytes/bytes/database/sql_meta_repository.py index 4faac6e4cab..dac10cfbda2 100644 --- a/bytes/bytes/database/sql_meta_repository.py +++ b/bytes/bytes/database/sql_meta_repository.py @@ -282,10 +282,7 @@ def to_normalizer_meta(normalizer_meta_in_db: NormalizerMetaInDB) -> NormalizerM return NormalizerMeta( id=normalizer_meta_in_db.id, - normalizer=Normalizer( - id=normalizer_meta_in_db.normalizer_id, - version=normalizer_meta_in_db.normalizer_version, - ), + normalizer=Normalizer(id=normalizer_meta_in_db.normalizer_id, version=normalizer_meta_in_db.normalizer_version), started_at=normalizer_meta_in_db.started_at, ended_at=normalizer_meta_in_db.ended_at, raw_data=raw_meta, diff --git a/bytes/tests/client.py b/bytes/tests/client.py index 6996d1e821b..3d3d9f5078b 100644 --- a/bytes/tests/client.py +++ b/bytes/tests/client.py @@ -35,10 +35,7 @@ def __init__(self, base_url: str, username: str, password: str): self.client = httpx.Client( base_url=base_url, headers={"User-Agent": f"bytes-api-client/{BYTES_API_CLIENT_VERSION}"} ) - self._credentials = { - "username": username, - "password": password, - } + self._credentials = {"username": username, "password": password} def login(self) -> None: self.client.headers.update(self._get_authentication_headers()) @@ -52,9 +49,7 @@ def _get_authentication_headers(self) -> dict[str, str]: def _get_token(self) -> str: response = self.client.post( - "/token", - data=self._credentials, - headers={"content-type": "application/x-www-form-urlencoded"}, + "/token", data=self._credentials, headers={"content-type": "application/x-www-form-urlencoded"} ) return str(response.json()["access_token"]) diff --git a/bytes/tests/conftest.py b/bytes/tests/conftest.py index 69474441feb..4e24aa544a6 100644 --- a/bytes/tests/conftest.py +++ b/bytes/tests/conftest.py @@ -89,11 +89,7 @@ def bytes_api_client(settings) -> Iterator[BytesAPIClient]: alembicArgs = ["--config", "/app/bytes/bytes/alembic.ini", "--raiseerr", "upgrade", "head"] alembic.config.main(argv=alembicArgs) - client = BytesAPIClient( - "http://ci_bytes:8000", - settings.username, - settings.password, - ) + client = BytesAPIClient("http://ci_bytes:8000", settings.username, settings.password) client.login() yield client diff --git a/bytes/tests/integration/test_bytes_api.py b/bytes/tests/integration/test_bytes_api.py index 95122aaeea0..a7cd728aae0 100644 --- a/bytes/tests/integration/test_bytes_api.py +++ b/bytes/tests/integration/test_bytes_api.py @@ -79,10 +79,7 @@ def test_get_mime_type_count(bytes_api_client: BytesAPIClient) -> None: normalizer_meta = get_normalizer_meta(raw_id) bytes_api_client.save_normalizer_meta(normalizer_meta) - assert bytes_api_client.get_mime_type_count(RawDataFilter(organization="test")) == { - "boefje": 2, - "text/boefje": 1, - } + assert bytes_api_client.get_mime_type_count(RawDataFilter(organization="test")) == {"boefje": 2, "text/boefje": 1} assert bytes_api_client.get_mime_type_count(RawDataFilter(organization="test", normalized=True)) == { "boefje": 1, diff --git a/bytes/tests/loading.py b/bytes/tests/loading.py index a381d57d080..5281c7caf17 100644 --- a/bytes/tests/loading.py +++ b/bytes/tests/loading.py @@ -57,8 +57,4 @@ def get_raw_data() -> RawData: def get_raw_data_meta(raw_file_id: UUID = UUID("2c9f47db-dfca-4928-b29f-368e64b3c779")) -> RawDataMeta: raw_data = get_raw_data() - return RawDataMeta( - id=raw_file_id, - boefje_meta=raw_data.boefje_meta, - mime_types=raw_data.mime_types, - ) + return RawDataMeta(id=raw_file_id, boefje_meta=raw_data.boefje_meta, mime_types=raw_data.mime_types) diff --git a/bytes/tests/unit/test_hash.py b/bytes/tests/unit/test_hash.py index f8f6e84bd5a..1c815c95d46 100644 --- a/bytes/tests/unit/test_hash.py +++ b/bytes/tests/unit/test_hash.py @@ -32,14 +32,8 @@ def test_hash_sha224(self) -> None: secure_hash = hash_data(data=get_raw_data(), datetime=dt, hash_algo=HashingAlgorithm.SHA224) - self.assertEqual( - "sha224:27154a1b6301ba1bde0b78cc28590e19b8f15c660f13885765cc3d44", - secure_hash, - ) + self.assertEqual("sha224:27154a1b6301ba1bde0b78cc28590e19b8f15c660f13885765cc3d44", secure_hash) secure_hash = hash_data(data=get_raw_data(), datetime=dt, hash_algo=HashingAlgorithm.SHA224) - self.assertEqual( - "sha224:27154a1b6301ba1bde0b78cc28590e19b8f15c660f13885765cc3d44", - secure_hash, - ) + self.assertEqual("sha224:27154a1b6301ba1bde0b78cc28590e19b8f15c660f13885765cc3d44", secure_hash) diff --git a/docs/source/conf.py b/docs/source/conf.py index e1bd2ee3fc0..a5a14b667ce 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -46,10 +46,7 @@ html_logo = "_static/keiko-hero.jpg" html_favicon = "_static/favicon.svg" -html_theme_options = { - "collapse_navigation": False, - "style_nav_header_background": "#ca005d", -} +html_theme_options = {"collapse_navigation": False, "style_nav_header_background": "#ca005d"} html_context = { "display_github": True, @@ -60,11 +57,7 @@ } html_static_path = ["_static"] -html_css_files = [ - "openkat.css", -] +html_css_files = ["openkat.css"] mermaid_version = "" # Do not fetch from the CDN -html_js_files = [ - "mermaid-9.4.3.min.js", -] +html_js_files = ["mermaid-9.4.3.min.js"] diff --git a/keiko/keiko/cli.py b/keiko/keiko/cli.py index 3bc9a0bac84..60a66e04f2f 100644 --- a/keiko/keiko/cli.py +++ b/keiko/keiko/cli.py @@ -13,9 +13,7 @@ @click.command() @click.argument("sample", type=click.File("r")) -def main( - sample: TextIO, -) -> None: +def main(sample: TextIO) -> None: """ Click entry point. diff --git a/keiko/keiko/health.py b/keiko/keiko/health.py index 58191b0a746..a31f0dabbe0 100644 --- a/keiko/keiko/health.py +++ b/keiko/keiko/health.py @@ -22,8 +22,4 @@ class ServiceHealth(BaseModel): def get_health() -> ServiceHealth: """Determine health of Keiko service.""" - return ServiceHealth( - service="keiko", - healthy=True, - version=__version__, - ) + return ServiceHealth(service="keiko", healthy=True, version=__version__) diff --git a/keiko/keiko/keiko.py b/keiko/keiko/keiko.py index 8526113a34b..94959d8d337 100644 --- a/keiko/keiko/keiko.py +++ b/keiko/keiko/keiko.py @@ -80,12 +80,7 @@ def baretext(text: str) -> str: @tracer.start_as_current_span("generate_report") def generate_report( - template_name: str, - report_data: DataShapeBase, - glossary: str, - report_id: str, - debug: bool, - settings: Settings, + template_name: str, report_data: DataShapeBase, glossary: str, report_id: str, debug: bool, settings: Settings ) -> None: """Generate a preprocessed LateX file from a template, a JSON data file and a glossary CSV file.""" current_span = trace.get_current_span() @@ -94,11 +89,7 @@ def generate_report( data_shape_class = get_data_shape(template_name, settings) data = data_shape_class.parse_obj(report_data.dict()) current_span.add_event("Data shape validation successful") - logger.info( - "Data shape validation successful. [report_id=%s] [template=%s]", - report_id, - template_name, - ) + logger.info("Data shape validation successful. [report_id=%s] [template=%s]", report_id, template_name) # build glossary glossary_entries = read_glossary(glossary, settings) @@ -107,9 +98,7 @@ def generate_report( # init jinja2 template env = Environment( # noqa: S701 - loader=FileSystemLoader(settings.templates_folder), - variable_start_string="@@{", - variable_end_string="}@@", + loader=FileSystemLoader(settings.templates_folder), variable_start_string="@@{", variable_end_string="}@@" ) env.filters["latex_escape"] = latex_escape env.filters["to_text"] = to_text @@ -117,11 +106,7 @@ def generate_report( template = env.get_template(f"{template_name}/template.tex") if not template.filename: - logger.error( - "Template file not found. [report_id=%s] [template=%s]", - report_id, - template_name, - ) + logger.error("Template file not found. [report_id=%s] [template=%s]", report_id, template_name) ex = Exception("Template file %s not found", template_name) current_span.set_status(Status(StatusCode.ERROR)) current_span.record_exception(ex) @@ -154,10 +139,7 @@ def generate_report( with tempfile.TemporaryDirectory() as directory: current_span.add_event("Temporary folder created") logger.info( - "Temporary folder created. [report_id=%s] [template=%s] [directory=%s]", - report_id, - template_name, - directory, + "Temporary folder created. [report_id=%s] [template=%s] [directory=%s]", report_id, template_name, directory ) # copy assets @@ -177,22 +159,13 @@ def generate_report( # if debug is enabled copy preprocessed tex file and input data if debug or settings.debug: - shutil.copyfile( - preprocessed_tex_path, - tex_output_file_path, - ) + shutil.copyfile(preprocessed_tex_path, tex_output_file_path) json_output_file_path = output_file.with_suffix(".keiko.json") json_output_file_path.write_text(report_data.model_dump_json(indent=4)) # run pdflatex - cmd = [ - "latexmk", - "-xelatex", - "-synctex=1", - "-interaction=nonstopmode", - preprocessed_tex_path.as_posix(), - ] + cmd = ["latexmk", "-xelatex", "-synctex=1", "-interaction=nonstopmode", preprocessed_tex_path.as_posix()] env = {**os.environ, "TEXMFVAR": directory} def log_output(level, output): @@ -208,12 +181,7 @@ def log_output(level, output): cmd, cwd=directory, env=env, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) current_span.add_event("Completed latexmk") - logger.info( - "latexmk [report_id=%s] [template=%s] [command=%s]", - report_id, - template_name, - " ".join(cmd), - ) + logger.info("latexmk [report_id=%s] [template=%s] [command=%s]", report_id, template_name, " ".join(cmd)) log_output(DEBUG, output.stdout) except subprocess.CalledProcessError as ex: log_output(ERROR, ex.stdout) @@ -224,10 +192,7 @@ def log_output(level, output): raise err # copy result back to output folder - shutil.copyfile( - preprocessed_tex_path.with_suffix(".pdf"), - pdf_output_file_path, - ) + shutil.copyfile(preprocessed_tex_path.with_suffix(".pdf"), pdf_output_file_path) current_span.add_event("Report copied to reports folder") logger.info( "Report copied to reports folder. [report_id=%s] [template=%s] [output_file=%s]", diff --git a/keiko/keiko/logging.py b/keiko/keiko/logging.py index 9a01c8857e7..c679bdeab62 100644 --- a/keiko/keiko/logging.py +++ b/keiko/keiko/logging.py @@ -18,10 +18,7 @@ def setup_loggers(settings: Settings) -> None: except FileNotFoundError: logger.error("Logging configuration file not found. [log_cfg=%s]", settings.log_cfg) except JSONDecodeError: - logger.error( - "Logging configuration file is not a valid JSON file. [log_cfg=%s]", - settings.log_cfg, - ) + logger.error("Logging configuration file is not a valid JSON file. [log_cfg=%s]", settings.log_cfg) if settings.debug: # pylint fails on "Instance of 'RootLogger' has no 'loggerDict' member (no-member)" diff --git a/keiko/keiko/templates.py b/keiko/keiko/templates.py index 748ca6c68a9..dc226df6ba8 100644 --- a/keiko/keiko/templates.py +++ b/keiko/keiko/templates.py @@ -31,10 +31,7 @@ def get_templates(settings: Settings) -> set[str]: get_data_shape(template_folder, settings) templates.add(template_folder) except FileNotFoundError: - logger.warning( - "Template data shape definition not found. [template=%s]", - template_folder, - ) + logger.warning("Template data shape definition not found. [template=%s]", template_folder) return templates @@ -50,9 +47,7 @@ def get_data_shape(template: str, settings: Settings) -> BaseModel: loader = importlib.machinery.SourceFileLoader(f"{template}_model", str(model_path)) spec = importlib.util.spec_from_loader(loader.name, loader) if spec is None: - raise FileNotFoundError( - f"No such file or directory: {model_path}", - ) + raise FileNotFoundError(f"No such file or directory: {model_path}") module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -73,14 +68,8 @@ def get_samples(settings: Settings) -> dict[str, dict[str, Any]]: if sample_file.exists(): with sample_file.open() as sample: try: - samples[subfolder_name] = { - "summary": subfolder_name, - "value": json.load(sample), - } + samples[subfolder_name] = {"summary": subfolder_name, "value": json.load(sample)} except json.decoder.JSONDecodeError: - logger.warning( - "Could not load sample data for template %s. Invalid JSON", - subfolder_name, - ) + logger.warning("Could not load sample data for template %s. Invalid JSON", subfolder_name) return samples diff --git a/keiko/tests/test_api.py b/keiko/tests/test_api.py index 92b4683cb1d..6daaa75189e 100644 --- a/keiko/tests/test_api.py +++ b/keiko/tests/test_api.py @@ -24,13 +24,7 @@ def test_health(self): self.assertEqual(200, response.status_code) self.assertDictEqual( - { - "service": "keiko", - "healthy": True, - "version": "0.0.1.dev1", - "additional": None, - "results": [], - }, + {"service": "keiko", "healthy": True, "version": "0.0.1.dev1", "additional": None, "results": []}, response.json(), ) diff --git a/keiko/tests/test_report.py b/keiko/tests/test_report.py index d62015fcf7a..18ed0353710 100644 --- a/keiko/tests/test_report.py +++ b/keiko/tests/test_report.py @@ -103,7 +103,7 @@ }, } ], - }, + } }, "valid_time": "2022-08-26 08:23:58.373810+00:00", "report_source_type": "Hostname", diff --git a/mula/scheduler/alembic/env.py b/mula/scheduler/alembic/env.py index 080e35d2bbc..b7dab602acb 100644 --- a/mula/scheduler/alembic/env.py +++ b/mula/scheduler/alembic/env.py @@ -54,17 +54,12 @@ def run_migrations_online(): and associate a connection with the context. """ connectable = engine_from_config( - config.get_section(config.config_ini_section), - prefix="sqlalchemy.", - poolclass=pool.NullPool, + config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool ) with connectable.connect() as connection: context.configure( - connection=connection, - target_metadata=target_metadata, - compare_server_default=True, - compare_type=True, + connection=connection, target_metadata=target_metadata, compare_server_default=True, compare_type=True ) with context.begin_transaction(): diff --git a/mula/scheduler/alembic/versions/0005_size_limit_hash.py b/mula/scheduler/alembic/versions/0005_size_limit_hash.py index 8e52fb9024a..fce255cc13b 100644 --- a/mula/scheduler/alembic/versions/0005_size_limit_hash.py +++ b/mula/scheduler/alembic/versions/0005_size_limit_hash.py @@ -18,23 +18,11 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.alter_column( - "items", - "hash", - existing_type=sa.String(), - type_=sa.String(length=32), - existing_nullable=True, - ) + op.alter_column("items", "hash", existing_type=sa.String(), type_=sa.String(length=32), existing_nullable=True) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.alter_column( - "items", - "hash", - existing_type=sa.String(length=32), - type_=sa.String(), - existing_nullable=True, - ) + op.alter_column("items", "hash", existing_type=sa.String(length=32), type_=sa.String(), existing_nullable=True) # ### end Alembic commands ### diff --git a/mula/scheduler/app.py b/mula/scheduler/app.py index 4e0836bed12..35051f18d58 100644 --- a/mula/scheduler/app.py +++ b/mula/scheduler/app.py @@ -59,8 +59,7 @@ def __init__(self, ctx: context.AppContext) -> None: self.lock: threading.Lock = threading.Lock() self.schedulers: dict[ - str, - schedulers.Scheduler | schedulers.BoefjeScheduler | schedulers.NormalizerScheduler, + str, schedulers.Scheduler | schedulers.BoefjeScheduler | schedulers.NormalizerScheduler ] = {} self.server: server.Server | None = None @@ -108,11 +107,7 @@ def monitor_organisations(self) -> None: self.schedulers[scheduler_id].stop() if removals: - self.logger.debug( - "Removed %s organisations from scheduler", - len(removals), - removals=sorted(removals), - ) + self.logger.debug("Removed %s organisations from scheduler", len(removals), removals=sorted(removals)) # Add schedulers for organisation for org_id in additions: @@ -123,17 +118,11 @@ def monitor_organisations(self) -> None: continue scheduler_boefje = schedulers.BoefjeScheduler( - ctx=self.ctx, - scheduler_id=f"boefje-{org.id}", - organisation=org, - callback=self.remove_scheduler, + ctx=self.ctx, scheduler_id=f"boefje-{org.id}", organisation=org, callback=self.remove_scheduler ) scheduler_normalizer = schedulers.NormalizerScheduler( - ctx=self.ctx, - scheduler_id=f"normalizer-{org.id}", - organisation=org, - callback=self.remove_scheduler, + ctx=self.ctx, scheduler_id=f"normalizer-{org.id}", organisation=org, callback=self.remove_scheduler ) with self.lock: @@ -147,11 +136,7 @@ def monitor_organisations(self) -> None: # Flush katalogus caches when new organisations are added self.ctx.services.katalogus.flush_caches() - self.logger.debug( - "Added %s organisations to scheduler", - len(additions), - additions=sorted(additions), - ) + self.logger.debug("Added %s organisations to scheduler", len(additions), additions=sorted(additions)) @tracer.start_as_current_span("collect_metrics") def collect_metrics(self) -> None: @@ -161,20 +146,11 @@ def collect_metrics(self) -> None: """ with self.lock: for s in self.schedulers.copy().values(): - self.ctx.metrics_qsize.labels( - scheduler_id=s.scheduler_id, - ).set( - s.queue.qsize(), - ) + self.ctx.metrics_qsize.labels(scheduler_id=s.scheduler_id).set(s.queue.qsize()) status_counts = self.ctx.datastores.task_store.get_status_counts(s.scheduler_id) for status, count in status_counts.items(): - self.ctx.metrics_task_status_counts.labels( - scheduler_id=s.scheduler_id, - status=status, - ).set( - count, - ) + self.ctx.metrics_task_status_counts.labels(scheduler_id=s.scheduler_id, status=status).set(count) def start_schedulers(self) -> None: # Initialize the schedulers @@ -186,18 +162,12 @@ def start_schedulers(self) -> None: for org in orgs: boefje_scheduler = schedulers.BoefjeScheduler( - ctx=self.ctx, - scheduler_id=f"boefje-{org.id}", - organisation=org, - callback=self.remove_scheduler, + ctx=self.ctx, scheduler_id=f"boefje-{org.id}", organisation=org, callback=self.remove_scheduler ) self.schedulers[boefje_scheduler.scheduler_id] = boefje_scheduler normalizer_scheduler = schedulers.NormalizerScheduler( - ctx=self.ctx, - scheduler_id=f"normalizer-{org.id}", - organisation=org, - callback=self.remove_scheduler, + ctx=self.ctx, scheduler_id=f"normalizer-{org.id}", organisation=org, callback=self.remove_scheduler ) self.schedulers[normalizer_scheduler.scheduler_id] = normalizer_scheduler @@ -215,20 +185,12 @@ def start_monitors(self) -> None: def start_collectors(self) -> None: thread.ThreadRunner( - name="App-metrics_collector", - target=self.collect_metrics, - stop_event=self.stop_event, - interval=10, + name="App-metrics_collector", target=self.collect_metrics, stop_event=self.stop_event, interval=10 ).start() def start_server(self) -> None: self.server = server.Server(self.ctx, self.schedulers) - thread.ThreadRunner( - name="App-server", - target=self.server.run, - stop_event=self.stop_event, - loop=False, - ).start() + thread.ThreadRunner(name="App-server", target=self.server.run, stop_event=self.stop_event, loop=False).start() def run(self) -> None: """Start the main scheduler application, and run in threads the diff --git a/mula/scheduler/config/settings.py b/mula/scheduler/config/settings.py index 809f4dd00bb..4d9c1d56f39 100644 --- a/mula/scheduler/config/settings.py +++ b/mula/scheduler/config/settings.py @@ -14,10 +14,7 @@ class BackwardsCompatibleEnvSettings(PydanticBaseSettingsSource): - backwards_compatibility_mapping = { - "SCHEDULER_RABBITMQ_DSN": "QUEUE_URI", - "SCHEDULER_DB_DSN": "SCHEDULER_DB_URI", - } + backwards_compatibility_mapping = {"SCHEDULER_RABBITMQ_DSN": "QUEUE_URI", "SCHEDULER_DB_DSN": "SCHEDULER_DB_URI"} def get_field_value(self, field: fields.FieldInfo, field_name: str) -> tuple[Any, str, bool]: return super().get_field_value(field, field_name) @@ -48,37 +45,20 @@ class Settings(BaseSettings): model_config = SettingsConfigDict(env_prefix="SCHEDULER_") # Application settings - debug: bool = Field( - False, - alias="DEBUG", - description="Enables/disables global debugging mode", - ) + debug: bool = Field(False, alias="DEBUG", description="Enables/disables global debugging mode") - log_cfg: Path = Field( - BASE_DIR / "logging.json", - description="Path to the logging configuration file", - ) + log_cfg: Path = Field(BASE_DIR / "logging.json", description="Path to the logging configuration file") collect_metrics: bool = Field( - False, - description="Enables/disables the collection of metrics to be used with tools like Prometheus", + False, description="Enables/disables the collection of metrics to be used with tools like Prometheus" ) - json_logging: bool = Field( - False, - description="Enables/disables structured logging in json format", - ) + json_logging: bool = Field(False, description="Enables/disables structured logging in json format") # Server settings - api_host: str = Field( - "0.0.0.0", - description="Host address of the scheduler api server", - ) + api_host: str = Field("0.0.0.0", description="Host address of the scheduler api server") - api_port: int = Field( - 8000, - description="Host api server port", - ) + api_port: int = Field(8000, description="Host api server port") # Application settings monitor_organisations_interval: int = Field( @@ -91,39 +71,26 @@ class Settings(BaseSettings): ) # External services settings - octopoes_request_timeout: int = Field( - 10, - description="The timeout in seconds for the requests to the octopoes api", - ) + octopoes_request_timeout: int = Field(10, description="The timeout in seconds for the requests to the octopoes api") octopoes_pool_connections: int = Field( - 10, - description="The maximum number of connections to save in the pool for the octopoes api", + 10, description="The maximum number of connections to save in the pool for the octopoes api" ) - katalogus_cache_ttl: int = Field( - 30, - description="The lifetime of the katalogus cache in seconds", - ) + katalogus_cache_ttl: int = Field(30, description="The lifetime of the katalogus cache in seconds") katalogus_request_timeout: int = Field( - 10, - description="The timeout in seconds for the requests to the katalogus api", + 10, description="The timeout in seconds for the requests to the katalogus api" ) katalogus_pool_connections: int = Field( - 10, - description="The maximum number of connections to save in the pool for the katalogus api", + 10, description="The maximum number of connections to save in the pool for the katalogus api" ) - bytes_request_timeout: int = Field( - 10, - description="The timeout in seconds for the requests to the bytes api", - ) + bytes_request_timeout: int = Field(10, description="The timeout in seconds for the requests to the bytes api") bytes_pool_connections: int = Field( - 10, - description="The maximum number of connections to save in the pool for the bytes api", + 10, description="The maximum number of connections to save in the pool for the bytes api" ) rabbitmq_prefetch_count: int = Field( @@ -135,79 +102,46 @@ class Settings(BaseSettings): # External services settings host_katalogus: AnyHttpUrl = Field( - ..., - example="http://localhost:8003", - alias="KATALOGUS_API", - description="Katalogus API URL", + ..., example="http://localhost:8003", alias="KATALOGUS_API", description="Katalogus API URL" ) - host_bytes: AnyHttpUrl = Field( - ..., - example="http://localhost:8004", - alias="BYTES_API", - description="Bytes API URL", - ) + host_bytes: AnyHttpUrl = Field(..., example="http://localhost:8004", alias="BYTES_API", description="Bytes API URL") - host_bytes_user: str = Field( - ..., - example="test", - alias="BYTES_USERNAME", - description="Bytes JWT login username", - ) + host_bytes_user: str = Field(..., example="test", alias="BYTES_USERNAME", description="Bytes JWT login username") host_bytes_password: str = Field( - ..., - example="secret", - alias="BYTES_PASSWORD", - description="Bytes JWT login password", + ..., example="secret", alias="BYTES_PASSWORD", description="Bytes JWT login password" ) host_octopoes: AnyHttpUrl = Field( - ..., - example="http://localhost:8001", - alias="OCTOPOES_API", - description="Octopoes API URL", + ..., example="http://localhost:8001", alias="OCTOPOES_API", description="Octopoes API URL" ) host_mutation: AmqpDsn = Field( - ..., - example="amqp://", - alias="QUEUE_URI", - description="KAT queue URI for host mutations", + ..., example="amqp://", alias="QUEUE_URI", description="KAT queue URI for host mutations" ) host_raw_data: AmqpDsn = Field( - ..., - example="amqp://", - alias="QUEUE_URI", - description="KAT queue URI for host raw data", + ..., example="amqp://", alias="QUEUE_URI", description="KAT queue URI for host raw data" ) host_metrics: AnyHttpUrl | None = Field( - None, - alias="SPAN_EXPORT_GRPC_ENDPOINT", - description="OpenTelemetry endpoint", + None, alias="SPAN_EXPORT_GRPC_ENDPOINT", description="OpenTelemetry endpoint" ) # Queue settings - pq_maxsize: int = Field( - 1000, - description="How many items a priority queue can hold (0 is infinite)", - ) + pq_maxsize: int = Field(1000, description="How many items a priority queue can hold (0 is infinite)") pq_interval: int = Field( - 60, - description="Interval in seconds of the execution of the `` method of the `scheduler.Scheduler` class", + 60, description="Interval in seconds of the execution of the `` method of the `scheduler.Scheduler` class" ) pq_grace_period: int = Field( - 86400, - description="Grace period of when a job is considered to be running again in seconds", + 86400, description="Grace period of when a job is considered to be running again in seconds" ) pq_max_random_objects: int = Field( - 50, - description="The maximum number of random objects that can be added to the priority queue, per call", + 50, description="The maximum number of random objects that can be added to the priority queue, per call" ) # Database settings diff --git a/mula/scheduler/connectors/connector.py b/mula/scheduler/connectors/connector.py index 29f3cbc4351..857fd0ad1c4 100644 --- a/mula/scheduler/connectors/connector.py +++ b/mula/scheduler/connectors/connector.py @@ -69,12 +69,7 @@ def retry(self, func: Callable, *args, **kwargs) -> bool: return True self.logger.warning( - "Function %s, failed. Retry count: %d", - func.__name__, - i, - name=func.__name__, - args=args, - kwargs=kwargs, + "Function %s, failed. Retry count: %d", func.__name__, i, name=func.__name__, args=args, kwargs=kwargs ) time.sleep(10) diff --git a/mula/scheduler/connectors/errors.py b/mula/scheduler/connectors/errors.py index a76e4b47b34..e309641ec4b 100644 --- a/mula/scheduler/connectors/errors.py +++ b/mula/scheduler/connectors/errors.py @@ -29,8 +29,7 @@ def inner_function(*args, **kwargs): return func(*args, **kwargs) except httpx.HTTPStatusError as exc: raise ExternalServiceResponseError( - f"External service returned an error: {str(exc)}", - response=exc.response, + f"External service returned an error: {str(exc)}", response=exc.response ) from exc except httpx.ConnectError as exc: raise ExternalServiceConnectionError("External service is not available.") from exc diff --git a/mula/scheduler/connectors/services/bytes.py b/mula/scheduler/connectors/services/bytes.py index ec92dcec2df..03250e6342d 100644 --- a/mula/scheduler/connectors/services/bytes.py +++ b/mula/scheduler/connectors/services/bytes.py @@ -36,15 +36,7 @@ class Bytes(HTTPService): name = "bytes" - def __init__( - self, - host: str, - source: str, - user: str, - password: str, - timeout: int, - pool_connections: int, - ): + def __init__(self, host: str, source: str, user: str, password: str, timeout: int, pool_connections: int): """Initialize the Bytes service. Args: @@ -54,10 +46,7 @@ def __init__( password: A string representing the password. timeout: An integer representing the timeout. """ - self.credentials: dict[str, str] = { - "username": user, - "password": password, - } + self.credentials: dict[str, str] = {"username": user, "password": password} self.lock: threading.Lock = threading.Lock() @@ -73,10 +62,7 @@ def _verify_response(response: httpx.Response) -> None: def get_token(self) -> str: url = f"{self.host}/token" - response = self.post( - url=url, - payload=self.credentials, - ) + response = self.post(url=url, payload=self.credentials) self._verify_response(response) @@ -108,14 +94,7 @@ def get_last_run_boefje(self, boefje_id: str, input_ooi: str, organization_id: s @exception_handler def get_last_run_boefje_by_organisation_id(self, organization_id: str) -> BoefjeMeta | None: url = f"{self.host}/bytes/boefje_meta" - response = self.get( - url=url, - params={ - "organization": organization_id, - "limit": 1, - "descending": "true", - }, - ) + response = self.get(url=url, params={"organization": organization_id, "limit": 1, "descending": "true"}) self._verify_response(response) diff --git a/mula/scheduler/connectors/services/octopoes.py b/mula/scheduler/connectors/services/octopoes.py index 90688db5a58..4ca4ca08654 100644 --- a/mula/scheduler/connectors/services/octopoes.py +++ b/mula/scheduler/connectors/services/octopoes.py @@ -19,14 +19,7 @@ class Octopoes(HTTPService): name = "octopoes" health_endpoint = None - def __init__( - self, - host: str, - source: str, - orgs: list[Organisation], - pool_connections: int, - timeout: int = 10, - ): + def __init__(self, host: str, source: str, orgs: list[Organisation], pool_connections: int, timeout: int = 10): self.orgs: list[Organisation] = orgs super().__init__(host, source, timeout, pool_connections) @@ -75,11 +68,7 @@ def get_random_objects(self, organisation_id: str, n: int, scan_level: list[int] url = f"{self.host}/{organisation_id}/objects/random" - params = { - "amount": str(n), - "scan_level": [s for s in scan_level], - "valid_time": datetime.now(timezone.utc), - } + params = {"amount": str(n), "scan_level": [s for s in scan_level], "valid_time": datetime.now(timezone.utc)} response = self.get(url, params=params) @@ -89,10 +78,7 @@ def get_random_objects(self, organisation_id: str, n: int, scan_level: list[int] def get_object(self, organisation_id: str, reference: str) -> OOI: """Get an ooi from octopoes""" url = f"{self.host}/{organisation_id}" - response = self.get( - url, - params={"reference": reference, "valid_time": datetime.now(timezone.utc)}, - ) + response = self.get(url, params={"reference": reference, "valid_time": datetime.now(timezone.utc)}) return OOI(**response.json()) def is_healthy(self) -> bool: diff --git a/mula/scheduler/connectors/services/services.py b/mula/scheduler/connectors/services/services.py index b539b454836..b874b6a05e0 100644 --- a/mula/scheduler/connectors/services/services.py +++ b/mula/scheduler/connectors/services/services.py @@ -39,14 +39,7 @@ class HTTPService(Connector): name: str | None = None health_endpoint: str | None = "health" - def __init__( - self, - host: str, - source: str, - timeout: int = 10, - pool_connections: int = 10, - retries: int = 5, - ): + def __init__(self, host: str, source: str, timeout: int = 10, pool_connections: int = 10, retries: int = 5): """Initializer of the HTTPService class. During initialization the host will be checked if it is available and healthy. @@ -81,11 +74,7 @@ def __init__( self._do_checks() - def get( - self, - url: str, - params: dict[str, Any] | None = None, - ) -> httpx.Response: + def get(self, url: str, params: dict[str, Any] | None = None) -> httpx.Response: """Execute a HTTP GET request Args: @@ -97,27 +86,12 @@ def get( Returns: A request.Response object """ - response = self.session.get( - url, - headers=self.headers, - params=params, - timeout=self.timeout, - ) - self.logger.debug( - "Made GET request to %s.", - url, - name=self.name, - url=url, - ) + response = self.session.get(url, headers=self.headers, params=params, timeout=self.timeout) + self.logger.debug("Made GET request to %s.", url, name=self.name, url=url) return response - def post( - self, - url: str, - payload: dict[str, Any], - params: dict[str, Any] | None = None, - ) -> httpx.Response: + def post(self, url: str, payload: dict[str, Any], params: dict[str, Any] | None = None) -> httpx.Response: """Execute a HTTP POST request Args: @@ -129,20 +103,8 @@ def post( Returns: A request.Response object """ - response = self.session.post( - url, - headers=self.headers, - params=params, - data=payload, - timeout=self.timeout, - ) - self.logger.debug( - "Made POST request to %s.", - url, - name=self.name, - url=url, - payload=payload, - ) + response = self.session.post(url, headers=self.headers, params=params, data=payload, timeout=self.timeout) + self.logger.debug("Made POST request to %s.", url, name=self.name, url=url, payload=payload) self._verify_response(response) @@ -165,11 +127,7 @@ def _do_checks(self) -> None: port = 80 if parsed_url.scheme == "http" else 443 if hostname is None or port is None: - self.logger.warning( - "Not able to parse hostname and port from %s", - self.host, - host=self.host, - ) + self.logger.warning("Not able to parse hostname and port from %s", self.host, host=self.host) return if self.host is not None and self.retry(self.is_host_available, hostname, port) is False: diff --git a/mula/scheduler/context/context.py b/mula/scheduler/context/context.py index 4369132a51a..2e7c7fab4ee 100644 --- a/mula/scheduler/context/context.py +++ b/mula/scheduler/context/context.py @@ -162,9 +162,7 @@ def __init__(self) -> None: self.metrics_registry: CollectorRegistry = CollectorRegistry() Info( - name="app_settings", - documentation="Scheduler configuration settings", - registry=self.metrics_registry, + name="app_settings", documentation="Scheduler configuration settings", registry=self.metrics_registry ).info( { "pq_maxsize": str(self.config.pq_maxsize), diff --git a/mula/scheduler/models/queue.py b/mula/scheduler/models/queue.py index d71df6e455e..014d999e702 100644 --- a/mula/scheduler/models/queue.py +++ b/mula/scheduler/models/queue.py @@ -48,18 +48,9 @@ class PrioritizedItemDB(Base): data = Column(JSONB, nullable=False) - created_at = Column( - DateTime(timezone=True), - nullable=False, - server_default=func.now(), - ) - - modified_at = Column( - DateTime(timezone=True), - nullable=False, - server_default=func.now(), - onupdate=func.now(), - ) + created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + modified_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()) class Queue(BaseModel): diff --git a/mula/scheduler/models/tasks.py b/mula/scheduler/models/tasks.py index 52c229ecd39..f54a2a6c65e 100644 --- a/mula/scheduler/models/tasks.py +++ b/mula/scheduler/models/tasks.py @@ -76,32 +76,13 @@ class TaskDB(Base): p_item = Column(JSONB, nullable=False) - status = Column( - Enum(TaskStatus), - nullable=False, - default=TaskStatus.PENDING, - ) - - created_at = Column( - DateTime(timezone=True), - nullable=False, - server_default=func.now(), - ) - - modified_at = Column( - DateTime(timezone=True), - nullable=False, - server_default=func.now(), - onupdate=func.now(), - ) - - __table_args__ = ( - Index( - "ix_p_item_hash", - text("(p_item->>'hash')"), - created_at.desc(), - ), - ) + status = Column(Enum(TaskStatus), nullable=False, default=TaskStatus.PENDING) + + created_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + modified_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()) + + __table_args__ = (Index("ix_p_item_hash", text("(p_item->>'hash')"), created_at.desc()),) class NormalizerTask(BaseModel): diff --git a/mula/scheduler/queues/normalizer.py b/mula/scheduler/queues/normalizer.py index 90cc7b86fa1..6a56bcbafd8 100644 --- a/mula/scheduler/queues/normalizer.py +++ b/mula/scheduler/queues/normalizer.py @@ -17,19 +17,10 @@ def create_hash(self, p_item: models.PrioritizedItem) -> str: Returns: A string representing the hash. """ - normalizer_id = dict_utils.deep_get( - p_item.model_dump(), - ["data", "normalizer", "id"], - ) - - boefje_meta_id = dict_utils.deep_get( - p_item.model_dump(), - ["data", "raw_data", "boefje_meta", "id"], - ) - - organization = dict_utils.deep_get( - p_item.model_dump(), - ["data", "raw_data", "boefje_meta", "organization"], - ) + normalizer_id = dict_utils.deep_get(p_item.model_dump(), ["data", "normalizer", "id"]) + + boefje_meta_id = dict_utils.deep_get(p_item.model_dump(), ["data", "raw_data", "boefje_meta", "id"]) + + organization = dict_utils.deep_get(p_item.model_dump(), ["data", "raw_data", "boefje_meta", "organization"]) return mmh3.hash_bytes(f"{normalizer_id}-{boefje_meta_id}-{organization}").hex() diff --git a/mula/scheduler/schedulers/boefje.py b/mula/scheduler/schedulers/boefje.py index 516ed777f26..683a37a73ff 100644 --- a/mula/scheduler/schedulers/boefje.py +++ b/mula/scheduler/schedulers/boefje.py @@ -55,16 +55,9 @@ def __init__( pq_store=ctx.datastores.pq_store, ) - super().__init__( - ctx=ctx, - queue=self.queue, - scheduler_id=scheduler_id, - callback=callback, - ) + super().__init__(ctx=ctx, queue=self.queue, scheduler_id=scheduler_id, callback=callback) - self.ranker = rankers.BoefjeRanker( - ctx=self.ctx, - ) + self.ranker = rankers.BoefjeRanker(ctx=self.ctx) def run(self) -> None: """The run method is called when the scheduler is started. It will @@ -108,9 +101,7 @@ def run(self) -> None: # Random OOI's from Octopoes self.run_in_thread( - name=f"BoefjeScheduler-{self.scheduler_id}-random", - target=self.push_tasks_for_random_objects, - interval=60.0, + name=f"BoefjeScheduler-{self.scheduler_id}-random", target=self.push_tasks_for_random_objects, interval=60.0 ) self.logger.info( @@ -144,9 +135,7 @@ def push_tasks_for_scan_profile_mutations(self, body: bytes) -> None: ooi = mutation.value if ooi is None: self.logger.debug( - "Mutation value is None, skipping", - organisation_id=self.organisation.id, - scheduler_id=self.scheduler_id, + "Mutation value is None, skipping", organisation_id=self.organisation.id, scheduler_id=self.scheduler_id ) return @@ -156,24 +145,14 @@ def push_tasks_for_scan_profile_mutations(self, body: bytes) -> None: items, _ = self.ctx.datastores.pq_store.get_items( scheduler_id=self.scheduler_id, filters=filters.FilterRequest( - filters=[ - filters.Filter( - column="data", - field="input_ooi", - operator="eq", - value=ooi.primary_key, - ), - ], + filters=[filters.Filter(column="data", field="input_ooi", operator="eq", value=ooi.primary_key)] ), ) # Delete all items for this ooi, update all tasks for this ooi # to cancelled. for item in items: - self.ctx.datastores.pq_store.remove( - scheduler_id=self.scheduler_id, - item_id=item.id, - ) + self.ctx.datastores.pq_store.remove(scheduler_id=self.scheduler_id, item_id=item.id) if item.hash is None: continue @@ -202,12 +181,7 @@ def push_tasks_for_scan_profile_mutations(self, body: bytes) -> None: thread_name_prefix=f"BoefjeScheduler-TPE-{self.scheduler_id}-mutations" ) as executor: for boefje in boefjes: - executor.submit( - self.push_task, - boefje, - ooi, - self.push_tasks_for_scan_profile_mutations.__name__, - ) + executor.submit(self.push_task, boefje, ooi, self.push_tasks_for_scan_profile_mutations.__name__) @tracer.start_as_current_span("boefje_push_tasks_for_new_boefjes") def push_tasks_for_new_boefjes(self) -> None: @@ -245,9 +219,7 @@ def push_tasks_for_new_boefjes(self) -> None: oois_by_object_type: list[OOI] = [] try: oois_by_object_type = self.ctx.services.octopoes.get_objects_by_object_types( - self.organisation.id, - boefje.consumes, - list(range(boefje.scan_level, 5)), + self.organisation.id, boefje.consumes, list(range(boefje.scan_level, 5)) ) except ExternalServiceError as exc: self.logger.error( @@ -263,12 +235,7 @@ def push_tasks_for_new_boefjes(self) -> None: thread_name_prefix=f"BoefjeScheduler-TPE-{self.scheduler_id}-new_boefjes" ) as executor: for ooi in oois_by_object_type: - executor.submit( - self.push_task, - boefje, - ooi, - self.push_tasks_for_new_boefjes.__name__, - ) + executor.submit(self.push_task, boefje, ooi, self.push_tasks_for_new_boefjes.__name__) @tracer.start_as_current_span("boefje_push_tasks_for_random_objects") def push_tasks_for_random_objects(self) -> None: @@ -284,9 +251,7 @@ def push_tasks_for_random_objects(self) -> None: try: random_oois = self.ctx.services.octopoes.get_random_objects( - organisation_id=self.organisation.id, - n=self.ctx.config.pq_max_random_objects, - scan_level=[1, 2, 3, 4], + organisation_id=self.organisation.id, n=self.ctx.config.pq_max_random_objects, scan_level=[1, 2, 3, 4] ) except ExternalServiceError: self.logger.exception( @@ -330,12 +295,7 @@ def push_tasks_for_random_objects(self) -> None: thread_name_prefix=f"BoefjeScheduler-TPE-{self.scheduler_id}-random" ) as executor: for boefje in boefjes: - executor.submit( - self.push_task, - boefje, - ooi, - self.push_tasks_for_random_objects.__name__, - ) + executor.submit(self.push_task, boefje, ooi, self.push_tasks_for_random_objects.__name__) @tracer.start_as_current_span("boefje_is_task_allowed_to_run") def is_task_allowed_to_run(self, boefje: Plugin, ooi: OOI) -> bool: @@ -435,10 +395,7 @@ def is_task_running(self, task: BoefjeTask) -> bool: ) raise exc_db - if task_db is not None and task_db.status not in [ - TaskStatus.FAILED, - TaskStatus.COMPLETED, - ]: + if task_db is not None and task_db.status not in [TaskStatus.FAILED, TaskStatus.COMPLETED]: self.logger.debug( "Task is still running, according to the datastore", task_id=task_db.id, @@ -450,9 +407,7 @@ def is_task_running(self, task: BoefjeTask) -> bool: # Is task running according to bytes? try: task_bytes = self.ctx.services.bytes.get_last_run_boefje( - boefje_id=task.boefje.id, - input_ooi=task.input_ooi, - organization_id=task.organization, + boefje_id=task.boefje.id, input_ooi=task.input_ooi, organization_id=task.organization ) except ExternalServiceError as exc: self.logger.error( @@ -549,9 +504,7 @@ def push_task(self, boefje: Plugin, ooi: OOI, caller: str = "") -> None: """ task = BoefjeTask( - boefje=Boefje.parse_obj(boefje.dict()), - input_ooi=ooi.primary_key, - organization=self.organisation.id, + boefje=Boefje.parse_obj(boefje.dict()), input_ooi=ooi.primary_key, organization=self.organisation.id ) if not self.is_task_allowed_to_run(boefje, ooi): @@ -665,21 +618,12 @@ def push_task(self, boefje: Plugin, ooi: OOI, caller: str = "") -> None: return prior_tasks = self.ctx.datastores.task_store.get_tasks_by_hash(task.hash) - score = self.ranker.rank( - SimpleNamespace( - prior_tasks=prior_tasks, - task=task, - ) - ) + score = self.ranker.rank(SimpleNamespace(prior_tasks=prior_tasks, task=task)) # We need to create a PrioritizedItem for this task, to push # it to the priority queue. p_item = PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler_id, - priority=score, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler_id, priority=score, data=task.model_dump(), hash=task.hash ) try: @@ -751,9 +695,7 @@ def has_grace_period_passed(self, task: BoefjeTask) -> bool: try: task_bytes = self.ctx.services.bytes.get_last_run_boefje( - boefje_id=task.boefje.id, - input_ooi=task.input_ooi, - organization_id=task.organization, + boefje_id=task.boefje.id, input_ooi=task.input_ooi, organization_id=task.organization ) except ExternalServiceError as exc_bytes: self.logger.error( @@ -792,10 +734,7 @@ def get_boefjes_for_ooi(self, ooi) -> list[Plugin]: A list of Plugin of type Boefje that can be run on the ooi. """ try: - boefjes = self.ctx.services.katalogus.get_boefjes_by_type_and_org_id( - ooi.object_type, - self.organisation.id, - ) + boefjes = self.ctx.services.katalogus.get_boefjes_by_type_and_org_id(ooi.object_type, self.organisation.id) except ExternalServiceError: self.logger.error( "Could not get boefjes for object_type: %s", diff --git a/mula/scheduler/schedulers/normalizer.py b/mula/scheduler/schedulers/normalizer.py index cc32f98572d..523aa8e0d18 100644 --- a/mula/scheduler/schedulers/normalizer.py +++ b/mula/scheduler/schedulers/normalizer.py @@ -52,16 +52,9 @@ def __init__( pq_store=ctx.datastores.pq_store, ) - super().__init__( - ctx=ctx, - queue=self.queue, - scheduler_id=scheduler_id, - callback=callback, - ) + super().__init__(ctx=ctx, queue=self.queue, scheduler_id=scheduler_id, callback=callback) - self.ranker = rankers.NormalizerRanker( - ctx=self.ctx, - ) + self.ranker = rankers.NormalizerRanker(ctx=self.ctx) def run(self) -> None: """The run method is called when the scheduler is started. It will @@ -152,10 +145,7 @@ def push_tasks_for_received_raw_data(self, body: bytes) -> None: ) as executor: for normalizer in normalizers.values(): executor.submit( - self.push_task, - normalizer, - latest_raw_data.raw_data, - self.push_tasks_for_received_raw_data.__name__, + self.push_task, normalizer, latest_raw_data.raw_data, self.push_tasks_for_received_raw_data.__name__ ) @tracer.start_as_current_span("normalizer_push_task") @@ -168,10 +158,7 @@ def push_task(self, normalizer: Plugin, raw_data: RawData, caller: str = "") -> raw_data: The raw data to create a task for. caller: The name of the function that called this function, used for logging. """ - task = NormalizerTask( - normalizer=Normalizer(id=normalizer.id), - raw_data=raw_data, - ) + task = NormalizerTask(normalizer=Normalizer(id=normalizer.id), raw_data=raw_data) if not self.is_task_allowed_to_run(normalizer): self.logger.debug( @@ -218,21 +205,12 @@ def push_task(self, normalizer: Plugin, raw_data: RawData, caller: str = "") -> ) return - score = self.ranker.rank( - SimpleNamespace( - raw_data=raw_data, - task=task, - ), - ) + score = self.ranker.rank(SimpleNamespace(raw_data=raw_data, task=task)) # We need to create a PrioritizedItem for this task, to # push it to the priority queue. p_item = PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler_id, - priority=score, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler_id, priority=score, data=task.model_dump(), hash=task.hash ) try: @@ -273,8 +251,7 @@ def get_normalizers_for_mime_type(self, mime_type: str) -> list[Plugin]: """ try: normalizers = self.ctx.services.katalogus.get_normalizers_by_org_id_and_type( - self.organisation.id, - mime_type, + self.organisation.id, mime_type ) except ExternalServiceError: self.logger.warning( @@ -356,10 +333,7 @@ def is_task_running(self, task: NormalizerTask) -> bool: raise exc_db # Is task still running according to the datastore? - if task_db is not None and task_db.status not in [ - TaskStatus.COMPLETED, - TaskStatus.FAILED, - ]: + if task_db is not None and task_db.status not in [TaskStatus.COMPLETED, TaskStatus.FAILED]: self.logger.debug( "Task is still running, according to the datastore " "[task_id=%s, task_hash=%s, organisation_id=%s, scheduler_id=%s]", diff --git a/mula/scheduler/schedulers/scheduler.py b/mula/scheduler/schedulers/scheduler.py index 446f02c8605..c36a2a4f9d9 100644 --- a/mula/scheduler/schedulers/scheduler.py +++ b/mula/scheduler/schedulers/scheduler.py @@ -283,10 +283,7 @@ def push_items_to_queue(self, p_items: list[models.PrioritizedItem]) -> None: count += 1 def push_item_to_queue_with_timeout( - self, - p_item: models.PrioritizedItem, - max_tries: int = 5, - timeout: int = 1, + self, p_item: models.PrioritizedItem, max_tries: int = 5, timeout: int = 1 ) -> None: """Push an item to the queue, with a timeout. @@ -315,12 +312,7 @@ def push_item_to_queue_with_timeout( self.push_item_to_queue(p_item) def run_in_thread( - self, - name: str, - target: Callable[[], Any], - interval: float = 0.01, - daemon: bool = False, - loop: bool = True, + self, name: str, target: Callable[[], Any], interval: float = 0.01, daemon: bool = False, loop: bool = True ) -> None: """Make a function run in a thread, and add it to the dict of threads. @@ -332,12 +324,7 @@ def run_in_thread( loop: Whether the thread should loop. """ t = utils.ThreadRunner( - name=name, - target=target, - stop_event=self.stop_event_threads, - interval=interval, - daemon=daemon, - loop=loop, + name=name, target=target, stop_event=self.stop_event_threads, interval=interval, daemon=daemon, loop=loop ) t.start() @@ -379,8 +366,7 @@ def disable(self) -> None: # Get all tasks that were on the queue and set them to CANCELLED tasks, _ = self.ctx.datastores.task_store.get_tasks( - scheduler_id=self.scheduler_id, - status=models.TaskStatus.QUEUED, + scheduler_id=self.scheduler_id, status=models.TaskStatus.QUEUED ) task_ids = [task.id for task in tasks] self.ctx.datastores.task_store.cancel_tasks(scheduler_id=self.scheduler_id, task_ids=task_ids) diff --git a/mula/scheduler/server/server.py b/mula/scheduler/server/server.py index 319b6c62bfd..c6a85dd4c00 100644 --- a/mula/scheduler/server/server.py +++ b/mula/scheduler/server/server.py @@ -32,11 +32,7 @@ class Server: api: A fastapi.FastAPI object used for exposing API endpoints. """ - def __init__( - self, - ctx: context.AppContext, - s: dict[str, schedulers.Scheduler], - ): + def __init__(self, ctx: context.AppContext, s: dict[str, schedulers.Scheduler]): """Initializer of the Server class. Args: @@ -68,11 +64,7 @@ def __init__( self.logger.debug("Finished setting up OpenTelemetry instrumentation") self.api.add_api_route( - path="/", - endpoint=self.root, - methods=["GET"], - status_code=status.HTTP_200_OK, - description="Root endpoint", + path="/", endpoint=self.root, methods=["GET"], status_code=status.HTTP_200_OK, description="Root endpoint" ) self.api.add_api_route( @@ -212,11 +204,7 @@ def root(self) -> Any: return None def health(self, externals: bool = False) -> Any: - response = models.ServiceHealth( - service="scheduler", - healthy=True, - version=version.__version__, - ) + response = models.ServiceHealth(service="scheduler", healthy=True, version=version.__version__) if externals: for service in self.ctx.services.__dict__.values(): @@ -235,28 +223,19 @@ def get_schedulers(self) -> Any: def get_scheduler(self, scheduler_id: str) -> Any: s = self.schedulers.get(scheduler_id) if s is None: - raise fastapi.HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="scheduler not found", - ) + raise fastapi.HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="scheduler not found") return models.Scheduler(**s.dict()) def patch_scheduler(self, scheduler_id: str, item: models.Scheduler) -> Any: s = self.schedulers.get(scheduler_id) if s is None: - raise fastapi.HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="scheduler not found", - ) + raise fastapi.HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="scheduler not found") stored_scheduler_model = models.Scheduler(**s.dict()) patch_data = item.model_dump(exclude_unset=True) if len(patch_data) == 0: - raise fastapi.HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="no data to patch", - ) + raise fastapi.HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="no data to patch") updated_scheduler = stored_scheduler_model.model_copy(update=patch_data) @@ -267,8 +246,7 @@ def patch_scheduler(self, scheduler_id: str, item: models.Scheduler) -> Any: setattr(s, attr, value) except AttributeError as exc: raise fastapi.HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="attribute not found", + status_code=status.HTTP_400_BAD_REQUEST, detail="attribute not found" ) from exc # Enable or disable the scheduler if needed. @@ -295,8 +273,7 @@ def list_tasks( ) -> Any: if (min_created_at is not None and max_created_at is not None) and min_created_at > max_created_at: raise fastapi.HTTPException( - status_code=fastapi.status.HTTP_400_BAD_REQUEST, - detail="min_date must be less than max_date", + status_code=fastapi.status.HTTP_400_BAD_REQUEST, detail="min_date must be less than max_date" ) # FIXME: deprecated; backwards compatibility for rocky that uses the @@ -306,12 +283,7 @@ def list_tasks( if task_type == "boefje": f_ooi = { "and": [ - storage.filters.Filter( - column="p_item", - field="data__input_ooi", - operator="eq", - value=input_ooi, - ) + storage.filters.Filter(column="p_item", field="data__input_ooi", operator="eq", value=input_ooi) ] } elif task_type == "normalizer": @@ -329,10 +301,7 @@ def list_tasks( f_ooi = { "or": [ storage.filters.Filter( - column="p_item", - field="data__input_ooi", - operator="eq", - value=input_ooi, + column="p_item", field="data__input_ooi", operator="eq", value=input_ooi ), storage.filters.Filter( column="p_item", @@ -350,34 +319,22 @@ def list_tasks( f_plugin = { "and": [ storage.filters.Filter( - column="p_item", - field="data__boefje__id", - operator="eq", - value=plugin_id, + column="p_item", field="data__boefje__id", operator="eq", value=plugin_id ) ] } elif task_type == "normalizer": f_plugin = storage.filters.Filter( - column="p_item", - field="data__normalizer__id", - operator="eq", - value=plugin_id, + column="p_item", field="data__normalizer__id", operator="eq", value=plugin_id ) else: f_plugin = { "or": [ storage.filters.Filter( - column="p_item", - field="data__boefje__id", - operator="eq", - value=plugin_id, + column="p_item", field="data__boefje__id", operator="eq", value=plugin_id ), storage.filters.Filter( - column="p_item", - field="data__normalizer__id", - operator="eq", - value=plugin_id, + column="p_item", field="data__normalizer__id", operator="eq", value=plugin_id ), ] } @@ -396,20 +353,13 @@ def list_tasks( filters=f_req, ) except storage.filters.errors.FilterError as exc: - raise fastapi.HTTPException( - status_code=fastapi.status.HTTP_400_BAD_REQUEST, - detail=str(exc), - ) from exc + raise fastapi.HTTPException(status_code=fastapi.status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc except ValueError as exc: - raise fastapi.HTTPException( - status_code=fastapi.status.HTTP_400_BAD_REQUEST, - detail=str(exc), - ) from exc + raise fastapi.HTTPException(status_code=fastapi.status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc except Exception as exc: self.logger.exception(exc) raise fastapi.HTTPException( - status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="failed to get tasks", + status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR, detail="failed to get tasks" ) from exc return paginate(request, results, count=count, offset=offset, limit=limit) @@ -418,45 +368,31 @@ def get_task(self, task_id: str) -> Any: try: task = self.ctx.datastores.task_store.get_task_by_id(task_id) except ValueError as exc: - raise fastapi.HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(exc), - ) from exc + raise fastapi.HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc except Exception as exc: self.logger.exception(exc) raise fastapi.HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="failed to get task", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="failed to get task" ) from exc if task is None: - raise fastapi.HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="task not found", - ) + raise fastapi.HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="task not found") return models.Task(**task.model_dump()) def patch_task(self, task_id: str, item: dict) -> Any: if len(item) == 0: - raise fastapi.HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="no data to patch", - ) + raise fastapi.HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="no data to patch") try: task_db = self.ctx.datastores.task_store.get_task_by_id(task_id) except Exception as exc: raise fastapi.HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"failed to get task [exception: {exc}]", + status_code=status.HTTP_400_BAD_REQUEST, detail=f"failed to get task [exception: {exc}]" ) from exc if task_db is None: - raise fastapi.HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="task not found", - ) + raise fastapi.HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="task not found") updated_task = task_db.model_copy(update=item) @@ -466,8 +402,7 @@ def patch_task(self, task_id: str, item: dict) -> Any: except Exception as exc: self.logger.error(exc) raise fastapi.HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="failed to update task", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="failed to update task" ) from exc return updated_task @@ -478,8 +413,7 @@ def get_task_stats(self, scheduler_id: str | None = None) -> dict[str, dict[str, except Exception as exc: self.logger.exception(exc) raise fastapi.HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="failed to get task stats", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="failed to get task stats" ) from exc return stats @@ -491,26 +425,19 @@ def get_queue(self, queue_id: str) -> Any: s = self.schedulers.get(queue_id) if s is None: raise fastapi.HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="scheduler not found, by queue_id", + status_code=status.HTTP_404_NOT_FOUND, detail="scheduler not found, by queue_id" ) q = s.queue if q is None: - raise fastapi.HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="queue not found", - ) + raise fastapi.HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="queue not found") return models.Queue(**q.dict()) def pop_queue(self, queue_id: str, filters: storage.filters.FilterRequest | None = None) -> Any: s = self.schedulers.get(queue_id) if s is None: - raise fastapi.HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="queue not found", - ) + raise fastapi.HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="queue not found") try: p_item = s.pop_item_from_queue(filters) @@ -519,8 +446,7 @@ def pop_queue(self, queue_id: str, filters: storage.filters.FilterRequest | None if p_item is None: raise fastapi.HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="could not pop item from queue, check your filters", + status_code=status.HTTP_404_NOT_FOUND, detail="could not pop item from queue, check your filters" ) return models.PrioritizedItem(**p_item.model_dump()) @@ -528,10 +454,7 @@ def pop_queue(self, queue_id: str, filters: storage.filters.FilterRequest | None def push_queue(self, queue_id: str, item: models.PrioritizedItemRequest) -> Any: s = self.schedulers.get(queue_id) if s is None: - raise fastapi.HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="queue not found", - ) + raise fastapi.HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="queue not found") try: # Load default values @@ -553,36 +476,22 @@ def push_queue(self, queue_id: str, item: models.PrioritizedItemRequest) -> Any: p_item.data = item.data except Exception as exc: self.logger.exception(exc) - raise fastapi.HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=str(exc), - ) from exc + raise fastapi.HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc try: s.push_item_to_queue(p_item) except ValueError as exc_value: - raise fastapi.HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="malformed item", - ) from exc_value + raise fastapi.HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="malformed item") from exc_value except queues.QueueFullError as exc_full: raise fastapi.HTTPException( - status_code=status.HTTP_429_TOO_MANY_REQUESTS, - detail="queue is full", + status_code=status.HTTP_429_TOO_MANY_REQUESTS, detail="queue is full" ) from exc_full except queues.errors.NotAllowedError as exc_not_allowed: raise fastapi.HTTPException( - headers={"Retry-After": "60"}, - status_code=status.HTTP_409_CONFLICT, - detail=str(exc_not_allowed), + headers={"Retry-After": "60"}, status_code=status.HTTP_409_CONFLICT, detail=str(exc_not_allowed) ) from exc_not_allowed return p_item def run(self) -> None: - uvicorn.run( - self.api, - host=str(self.ctx.config.api_host), - port=self.ctx.config.api_port, - log_config=None, - ) + uvicorn.run(self.api, host=str(self.ctx.config.api_host), port=self.ctx.config.api_port, log_config=None) diff --git a/mula/scheduler/storage/filters/operators.py b/mula/scheduler/storage/filters/operators.py index 5db0e5566df..fb2dfca3f59 100644 --- a/mula/scheduler/storage/filters/operators.py +++ b/mula/scheduler/storage/filters/operators.py @@ -1,7 +1,3 @@ from sqlalchemy import and_, not_, or_ -FILTER_OPERATORS = { - "and": and_, - "or": or_, - "not": not_, -} +FILTER_OPERATORS = {"and": and_, "or": or_, "not": not_} diff --git a/mula/scheduler/storage/pq_store.py b/mula/scheduler/storage/pq_store.py index 3cb300102f2..32104604089 100644 --- a/mula/scheduler/storage/pq_store.py +++ b/mula/scheduler/storage/pq_store.py @@ -111,11 +111,7 @@ def qsize(self, scheduler_id: str) -> int: return count @retry() - def get_items( - self, - scheduler_id: str, - filters: FilterRequest | None, - ) -> tuple[list[models.PrioritizedItem], int]: + def get_items(self, scheduler_id: str, filters: FilterRequest | None) -> tuple[list[models.PrioritizedItem], int]: with self.dbconn.session.begin() as session: query = session.query(models.PrioritizedItemDB).filter( models.PrioritizedItemDB.scheduler_id == scheduler_id diff --git a/mula/scheduler/storage/storage.py b/mula/scheduler/storage/storage.py index 405090b72d3..7e9b36cdbb5 100644 --- a/mula/scheduler/storage/storage.py +++ b/mula/scheduler/storage/storage.py @@ -23,20 +23,14 @@ def __init__(self, dsn: str) -> None: logger.info("Connecting to database %s with pool size %s...", db_uri_redacted, pool_size) self.engine = sqlalchemy.create_engine( - dsn, - pool_pre_ping=True, - pool_size=pool_size, - pool_recycle=300, - json_serializer=serializer, + dsn, pool_pre_ping=True, pool_size=pool_size, pool_recycle=300, json_serializer=serializer ) logger.info("Connected to database %s.", db_uri_redacted) if self.engine is None: raise Exception("Invalid datastore type") - self.session = sqlalchemy.orm.sessionmaker( - bind=self.engine, - ) + self.session = sqlalchemy.orm.sessionmaker(bind=self.engine) def retry(max_retries: int = 3, retry_delay: float = 5.0): @@ -46,10 +40,7 @@ def wrapper(*args, **kwargs): for i in range(max_retries): try: return func(*args, **kwargs) - except ( - sqlalchemy.exc.OperationalError, - sqlalchemy.exc.InternalError, - ) as e: + except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.InternalError) as e: if i == max_retries - 1: raise e diff --git a/mula/scheduler/storage/task_store.py b/mula/scheduler/storage/task_store.py index c70842652c3..524e1fd549c 100644 --- a/mula/scheduler/storage/task_store.py +++ b/mula/scheduler/storage/task_store.py @@ -125,10 +125,7 @@ def cancel_tasks(self, scheduler_id: str, task_ids: list[str]) -> None: ).update({"status": models.TaskStatus.CANCELLED.name}) @retry() - def get_status_count_per_hour( - self, - scheduler_id: str | None = None, - ) -> dict[str, dict[str, int]] | None: + def get_status_count_per_hour(self, scheduler_id: str | None = None) -> dict[str, dict[str, int]] | None: with self.dbconn.session.begin() as session: query = ( session.query( @@ -136,9 +133,7 @@ def get_status_count_per_hour( models.TaskDB.status, func.count(models.TaskDB.id).label("count"), ) - .filter( - models.TaskDB.modified_at >= datetime.now(timezone.utc) - timedelta(hours=24), - ) + .filter(models.TaskDB.modified_at >= datetime.now(timezone.utc) - timedelta(hours=24)) .group_by("hour", models.TaskDB.status) .order_by("hour", models.TaskDB.status) ) diff --git a/mula/tests/factories/ooi.py b/mula/tests/factories/ooi.py index 54b5ac4f10f..c24ff9b43d6 100644 --- a/mula/tests/factories/ooi.py +++ b/mula/tests/factories/ooi.py @@ -8,10 +8,7 @@ class Meta: level: int = fuzzy.FuzzyInteger(0, 4) - scan_profile_type: str = Faker( - "random_element", - elements=["declared", "empty", "inherited"], - ) + scan_profile_type: str = Faker("random_element", elements=["declared", "empty", "inherited"]) reference: str = Faker("uuid4") @@ -24,9 +21,6 @@ class Meta: scan_profile: ScanProfile - object_type: str = Faker( - "random_element", - elements=["Hostname", "Network"], - ) + object_type: str = Faker("random_element", elements=["Hostname", "Network"]) organisation_id: str = Faker("uuid4") diff --git a/mula/tests/integration/test_api.py b/mula/tests/integration/test_api.py index 6b7c29b0e68..c151fbc019c 100644 --- a/mula/tests/integration/test_api.py +++ b/mula/tests/integration/test_api.py @@ -45,11 +45,7 @@ def setUp(self): pq_store=self.mock_ctx.datastores.pq_store, ) - self.scheduler = mock_scheduler.MockScheduler( - ctx=self.mock_ctx, - scheduler_id=self.organisation.id, - queue=queue, - ) + self.scheduler = mock_scheduler.MockScheduler(ctx=self.mock_ctx, scheduler_id=self.organisation.id, queue=queue) # API server and Test Client self.server = server.Server(self.mock_ctx, {self.scheduler.scheduler_id: self.scheduler}) @@ -91,10 +87,7 @@ def test_patch_scheduler_disable(self): # Try to push to queue item = create_p_item_request(0) - response = self.client.post( - f"/queues/{self.scheduler.scheduler_id}/push", - data=item.model_dump_json(), - ) + response = self.client.post(f"/queues/{self.scheduler.scheduler_id}/push", data=item.model_dump_json()) self.assertNotEqual(response.status_code, 201) self.assertEqual(0, self.scheduler.queue.qsize()) @@ -167,10 +160,7 @@ def test_push_queue_full_high_priority(self): # Add one task to the queue first_item = create_p_item_request(1) - response = self.client.post( - f"/queues/{self.scheduler.scheduler_id}/push", - data=first_item.model_dump_json(), - ) + response = self.client.post(f"/queues/{self.scheduler.scheduler_id}/push", data=first_item.model_dump_json()) self.assertEqual(response.status_code, 201) self.assertEqual(1, self.scheduler.queue.qsize()) @@ -191,18 +181,12 @@ def test_push_replace_not_allowed(self): # Add one task to the queue initial_item = create_p_item_request(1) - response = self.client.post( - f"/queues/{self.scheduler.scheduler_id}/push", - data=initial_item.model_dump_json(), - ) + response = self.client.post(f"/queues/{self.scheduler.scheduler_id}/push", data=initial_item.model_dump_json()) self.assertEqual(response.status_code, 201) self.assertEqual(1, self.scheduler.queue.qsize()) # Add the same item again through the api - response = self.client.post( - f"/queues/{self.scheduler.scheduler_id}/push", - data=initial_item.model_dump_json(), - ) + response = self.client.post(f"/queues/{self.scheduler.scheduler_id}/push", data=initial_item.model_dump_json()) # The queue should still have one item self.assertEqual(response.status_code, 409) @@ -266,10 +250,7 @@ def test_push_updates_allowed(self): # Add one task to the queue initial_item = create_p_item_request(1) - response = self.client.post( - f"/queues/{self.scheduler.scheduler_id}/push", - data=initial_item.model_dump_json(), - ) + response = self.client.post(f"/queues/{self.scheduler.scheduler_id}/push", data=initial_item.model_dump_json()) self.assertEqual(response.status_code, 201) self.assertEqual(1, self.scheduler.queue.qsize()) @@ -355,10 +336,7 @@ def test_update_priority_lower(self): # Add one task to the queue initial_item = create_p_item_request(1) - response = self.client.post( - f"/queues/{self.scheduler.scheduler_id}/push", - data=initial_item.model_dump_json(), - ) + response = self.client.post(f"/queues/{self.scheduler.scheduler_id}/push", data=initial_item.model_dump_json()) self.assertEqual(response.status_code, 201) # Update priority of the item @@ -378,10 +356,7 @@ def test_update_priority_lower(self): def test_pop_queue(self): # Add one task to the queue initial_item = create_p_item_request(1) - response = self.client.post( - f"/queues/{self.scheduler.scheduler_id}/push", - data=initial_item.model_dump_json(), - ) + response = self.client.post(f"/queues/{self.scheduler.scheduler_id}/push", data=initial_item.model_dump_json()) initial_item_id = response.json().get("id") self.assertEqual(response.status_code, 201) self.assertEqual(1, self.scheduler.queue.qsize()) @@ -394,10 +369,7 @@ def test_pop_queue(self): def test_pop_queue_filters(self): # Add one task to the queue first_item = create_p_item_request(1, data=functions.TestModel(id="123", name="test")) - response = self.client.post( - f"/queues/{self.scheduler.scheduler_id}/push", - data=first_item.model_dump_json(), - ) + response = self.client.post(f"/queues/{self.scheduler.scheduler_id}/push", data=first_item.model_dump_json()) first_item_id = response.json().get("id") self.assertEqual(response.status_code, 201) self.assertEqual(1, self.scheduler.queue.qsize()) @@ -449,11 +421,7 @@ def setUp(self): # Add one task to the queue first_item = create_p_item_request( 1, - data=functions.TestModel( - id="123", - name="test", - child=functions.TestModel(id="123.123", name="test.child"), - ), + data=functions.TestModel(id="123", name="test", child=functions.TestModel(id="123.123", name="test.child")), ) response = self.client.post(f"/queues/{self.scheduler.scheduler_id}/push", data=first_item.model_dump_json()) initial_item_id = response.json().get("id") @@ -463,10 +431,7 @@ def setUp(self): self.first_item_api = self.client.get(f"/tasks/{initial_item_id}").json() # Add second item to the queue - second_item = create_p_item_request( - 1, - data=functions.TestModel(id="456", name="test"), - ) + second_item = create_p_item_request(1, data=functions.TestModel(id="456", name="test")) response = self.client.post(f"/queues/{self.scheduler.scheduler_id}/push", data=second_item.model_dump_json()) second_item_id = response.json().get("id") self.assertEqual(response.status_code, 201) @@ -493,10 +458,7 @@ def test_get_task(self): # First add a task item = create_p_item_request(1) - response_post = self.client.post( - f"/queues/{self.scheduler.scheduler_id}/push", - data=item.model_dump_json(), - ) + response_post = self.client.post(f"/queues/{self.scheduler.scheduler_id}/push", data=item.model_dump_json()) self.assertEqual(201, response_post.status_code) initial_item_id = response_post.json().get("id") diff --git a/mula/tests/integration/test_app.py b/mula/tests/integration/test_app.py index 827f23cb194..1e9a8ed177b 100644 --- a/mula/tests/integration/test_app.py +++ b/mula/tests/integration/test_app.py @@ -125,9 +125,7 @@ def test_monitor_orgs_add_and_remove(self): def test_shutdown(self): """Test that the app shuts down gracefully""" # Arrange - self.mock_ctx.services.katalogus.organisations = { - "org-1": OrganisationFactory(id="org-1"), - } + self.mock_ctx.services.katalogus.organisations = {"org-1": OrganisationFactory(id="org-1")} self.app.start_schedulers() self.app.start_monitors() diff --git a/mula/tests/integration/test_boefje_scheduler.py b/mula/tests/integration/test_boefje_scheduler.py index c5fb014928e..3ac5bee9766 100644 --- a/mula/tests/integration/test_boefje_scheduler.py +++ b/mula/tests/integration/test_boefje_scheduler.py @@ -24,31 +24,21 @@ def setUp(self): self.mock_ctx.config = config.settings.Settings() # Mock connectors: octopoes - self.mock_octopoes = mock.create_autospec( - spec=connectors.services.Octopoes, - spec_set=True, - ) + self.mock_octopoes = mock.create_autospec(spec=connectors.services.Octopoes, spec_set=True) self.mock_ctx.services.octopoes = self.mock_octopoes # Mock connectors: Scan profile mutation self.mock_scan_profile_mutation = mock.create_autospec( - spec=connectors.listeners.ScanProfileMutation, - spec_set=True, + spec=connectors.listeners.ScanProfileMutation, spec_set=True ) self.mock_ctx.services.scan_profile_mutation = self.mock_scan_profile_mutation # Mock connectors: Katalogus - self.mock_katalogus = mock.create_autospec( - spec=connectors.services.Katalogus, - spec_set=True, - ) + self.mock_katalogus = mock.create_autospec(spec=connectors.services.Katalogus, spec_set=True) self.mock_ctx.services.katalogus = self.mock_katalogus # Mock connectors: Bytes - self.mock_bytes = mock.create_autospec( - spec=connectors.services.Bytes, - spec_set=True, - ) + self.mock_bytes = mock.create_autospec(spec=connectors.services.Bytes, spec_set=True) self.mock_ctx.services.bytes = self.mock_bytes # Database @@ -66,9 +56,7 @@ def setUp(self): # Scheduler self.organisation = OrganisationFactory() self.scheduler = schedulers.BoefjeScheduler( - ctx=self.mock_ctx, - scheduler_id=self.organisation.id, - organisation=self.organisation, + ctx=self.mock_ctx, scheduler_id=self.organisation.id, organisation=self.organisation ) def tearDown(self): @@ -123,11 +111,7 @@ def test_is_task_not_running(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) # Mock self.mock_get_latest_task_by_hash.return_value = None @@ -147,18 +131,10 @@ def test_is_task_running_datastore_running(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = models.PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler.scheduler_id, - priority=1, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler.scheduler_id, priority=1, data=task.model_dump(), hash=task.hash ) task_db = models.Task( @@ -189,18 +165,10 @@ def test_is_task_running_datastore_not_running(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = models.PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler.scheduler_id, - priority=1, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler.scheduler_id, priority=1, data=task.model_dump(), hash=task.hash ) task_db_first = models.Task( @@ -223,17 +191,10 @@ def test_is_task_running_datastore_not_running(self): modified_at=datetime.now(timezone.utc), ) - last_run_boefje = BoefjeMetaFactory( - boefje=boefje, - input_ooi=ooi.primary_key, - ended_at=datetime.utcnow(), - ) + last_run_boefje = BoefjeMetaFactory(boefje=boefje, input_ooi=ooi.primary_key, ended_at=datetime.utcnow()) # Mock - self.mock_get_latest_task_by_hash.side_effect = [ - task_db_first, - task_db_second, - ] + self.mock_get_latest_task_by_hash.side_effect = [task_db_first, task_db_second] self.mock_get_last_run_boefje.return_value = last_run_boefje # First run @@ -249,11 +210,7 @@ def test_is_task_running_datastore_exception(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) # Mock self.mock_get_latest_task_by_hash.side_effect = Exception("Something went wrong") @@ -270,16 +227,8 @@ def test_is_task_running_bytes_running(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) - last_run_boefje = BoefjeMetaFactory( - boefje=boefje, - input_ooi=ooi.primary_key, - ended_at=None, - ) + task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) + last_run_boefje = BoefjeMetaFactory(boefje=boefje, input_ooi=ooi.primary_key, ended_at=None) # Mock self.mock_get_latest_task_by_hash.return_value = None @@ -298,16 +247,8 @@ def test_is_task_running_bytes_not_running(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) - last_run_boefje = BoefjeMetaFactory( - boefje=boefje, - input_ooi=ooi.primary_key, - ended_at=datetime.utcnow(), - ) + task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) + last_run_boefje = BoefjeMetaFactory(boefje=boefje, input_ooi=ooi.primary_key, ended_at=datetime.utcnow()) # Mock self.mock_get_latest_task_by_hash.return_value = None @@ -324,11 +265,7 @@ def test_is_task_running_bytes_exception(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) # Mock self.mock_get_latest_task_by_hash.return_value = None @@ -342,18 +279,10 @@ def test_is_task_running_stalled_before_grace_period(self): # Arrange scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) - task = models.BoefjeTask( - boefje=BoefjeFactory(), - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=BoefjeFactory(), input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = models.PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler.scheduler_id, - priority=1, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler.scheduler_id, priority=1, data=task.model_dump(), hash=task.hash ) task_db = models.Task( @@ -377,18 +306,10 @@ def test_is_task_running_stalled_after_grace_period(self): # Arrange scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) - task = models.BoefjeTask( - boefje=BoefjeFactory(), - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=BoefjeFactory(), input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = models.PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler.scheduler_id, - priority=1, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler.scheduler_id, priority=1, data=task.model_dump(), hash=task.hash ) task_db = models.Task( @@ -415,18 +336,10 @@ def test_is_task_running_mismatch_before_grace_period(self): # Arrange scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) - task = models.BoefjeTask( - boefje=BoefjeFactory(), - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=BoefjeFactory(), input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = models.PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler.scheduler_id, - priority=1, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler.scheduler_id, priority=1, data=task.model_dump(), hash=task.hash ) task_db = models.Task( @@ -456,18 +369,10 @@ def test_is_task_running_mismatch_after_grace_period(self): # Arrange scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) - task = models.BoefjeTask( - boefje=BoefjeFactory(), - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=BoefjeFactory(), input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = models.PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler.scheduler_id, - priority=1, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler.scheduler_id, priority=1, data=task.model_dump(), hash=task.hash ) task_db = models.Task( @@ -492,18 +397,10 @@ def test_has_grace_period_passed_datastore_passed(self): # Arrange scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) - task = models.BoefjeTask( - boefje=BoefjeFactory(), - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=BoefjeFactory(), input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = models.PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler.scheduler_id, - priority=1, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler.scheduler_id, priority=1, data=task.model_dump(), hash=task.hash ) task_db = models.Task( @@ -531,18 +428,10 @@ def test_has_grace_period_passed_datastore_not_passed(self): # Arrange scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) - task = models.BoefjeTask( - boefje=BoefjeFactory(), - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=BoefjeFactory(), input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = models.PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler.scheduler_id, - priority=1, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler.scheduler_id, priority=1, data=task.model_dump(), hash=task.hash ) task_db = models.Task( @@ -570,18 +459,10 @@ def test_has_grace_period_passed_bytes_passed(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = models.PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler.scheduler_id, - priority=1, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler.scheduler_id, priority=1, data=task.model_dump(), hash=task.hash ) task_db = models.Task( @@ -615,18 +496,10 @@ def test_has_grace_period_passed_bytes_not_passed(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = models.PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler.scheduler_id, - priority=1, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler.scheduler_id, priority=1, data=task.model_dump(), hash=task.hash ) task_db = models.Task( @@ -640,9 +513,7 @@ def test_has_grace_period_passed_bytes_not_passed(self): ) last_run_boefje = BoefjeMetaFactory( - boefje=boefje, - input_ooi=ooi.primary_key, - ended_at=datetime.now(timezone.utc), + boefje=boefje, input_ooi=ooi.primary_key, ended_at=datetime.now(timezone.utc) ) # Mock @@ -717,16 +588,8 @@ def test_push_task_stalled( ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) - p_item = functions.create_p_item( - scheduler_id=self.organisation.id, - priority=1, - data=task, - ) + task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) + p_item = functions.create_p_item(scheduler_id=self.organisation.id, priority=1, data=task) # Act self.scheduler.push_item_to_queue(p_item) @@ -780,16 +643,8 @@ def test_post_push(self): # Arrange scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) - task = models.BoefjeTask( - boefje=BoefjeFactory(), - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) - p_item = functions.create_p_item( - scheduler_id=self.organisation.id, - priority=1, - data=task, - ) + task = models.BoefjeTask(boefje=BoefjeFactory(), input_ooi=ooi.primary_key, organization=self.organisation.id) + p_item = functions.create_p_item(scheduler_id=self.organisation.id, priority=1, data=task) # Act self.scheduler.push_item_to_queue(p_item) @@ -810,16 +665,8 @@ def test_post_pop(self): # Arrange scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) - task = models.BoefjeTask( - boefje=BoefjeFactory(), - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) - p_item = functions.create_p_item( - scheduler_id=self.organisation.id, - priority=1, - data=task, - ) + task = models.BoefjeTask(boefje=BoefjeFactory(), input_ooi=ooi.primary_key, organization=self.organisation.id) + p_item = functions.create_p_item(scheduler_id=self.organisation.id, priority=1, data=task) # Act self.scheduler.push_item_to_queue(p_item) @@ -850,16 +697,8 @@ def test_disable_scheduler(self): # Arrange: add tasks scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) - task = models.BoefjeTask( - boefje=BoefjeFactory(), - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) - p_item = functions.create_p_item( - scheduler_id=self.organisation.id, - priority=1, - data=task, - ) + task = models.BoefjeTask(boefje=BoefjeFactory(), input_ooi=ooi.primary_key, organization=self.organisation.id) + p_item = functions.create_p_item(scheduler_id=self.organisation.id, priority=1, data=task) self.scheduler.push_item_to_queue(p_item) # Assert: task should be on priority queue @@ -1001,23 +840,18 @@ def setUp(self): super().setUp() self.mock_is_task_running = mock.patch( - "scheduler.schedulers.BoefjeScheduler.is_task_running", - return_value=False, + "scheduler.schedulers.BoefjeScheduler.is_task_running", return_value=False ).start() self.mock_is_task_allowed_to_run = mock.patch( - "scheduler.schedulers.BoefjeScheduler.is_task_allowed_to_run", - return_value=True, + "scheduler.schedulers.BoefjeScheduler.is_task_allowed_to_run", return_value=True ).start() self.mock_has_grace_period_passed = mock.patch( - "scheduler.schedulers.BoefjeScheduler.has_grace_period_passed", - return_value=True, + "scheduler.schedulers.BoefjeScheduler.has_grace_period_passed", return_value=True ).start() - self.mock_get_boefjes_for_ooi = mock.patch( - "scheduler.schedulers.BoefjeScheduler.get_boefjes_for_ooi", - ).start() + self.mock_get_boefjes_for_ooi = mock.patch("scheduler.schedulers.BoefjeScheduler.get_boefjes_for_ooi").start() def test_push_tasks_for_scan_profile_mutations(self): """Scan level change""" @@ -1154,9 +988,7 @@ def test_push_task_for_scan_profile_mutations_delete(self): boefje = PluginFactory(scan_level=0, consumes=[ooi.object_type]) mutation1 = models.ScanProfileMutation( - operation=models.MutationOperationType.DELETE, - primary_key=ooi.primary_key, - value=ooi, + operation=models.MutationOperationType.DELETE, primary_key=ooi.primary_key, value=ooi ).model_dump_json() # Mocks @@ -1178,21 +1010,15 @@ def test_push_task_for_scan_profile_mutations_delete_on_queue(self): boefje = PluginFactory(scan_level=0, consumes=[ooi.object_type]) mutation1 = models.ScanProfileMutation( - operation=models.MutationOperationType.CREATE, - primary_key=ooi.primary_key, - value=ooi, + operation=models.MutationOperationType.CREATE, primary_key=ooi.primary_key, value=ooi ).model_dump_json() mutation2 = models.ScanProfileMutation( - operation=models.MutationOperationType.CREATE, - primary_key=ooi.primary_key, - value=ooi, + operation=models.MutationOperationType.CREATE, primary_key=ooi.primary_key, value=ooi ).model_dump_json() models.ScanProfileMutation( - operation=models.MutationOperationType.CREATE, - primary_key=ooi.primary_key, - value=ooi, + operation=models.MutationOperationType.CREATE, primary_key=ooi.primary_key, value=ooi ).model_dump_json() # Mocks @@ -1210,9 +1036,7 @@ def test_push_task_for_scan_profile_mutations_delete_on_queue(self): # Arrange ooi.scan_profile.level = 1 mutation2 = models.ScanProfileMutation( - operation=models.MutationOperationType.DELETE, - primary_key=ooi.primary_key, - value=ooi, + operation=models.MutationOperationType.DELETE, primary_key=ooi.primary_key, value=ooi ).model_dump_json() # Act @@ -1233,18 +1057,15 @@ def setUp(self): super().setUp() self.mock_is_task_running = mock.patch( - "scheduler.schedulers.BoefjeScheduler.is_task_running", - return_value=False, + "scheduler.schedulers.BoefjeScheduler.is_task_running", return_value=False ).start() self.mock_is_task_allowed_to_run = mock.patch( - "scheduler.schedulers.BoefjeScheduler.is_task_allowed_to_run", - return_value=True, + "scheduler.schedulers.BoefjeScheduler.is_task_allowed_to_run", return_value=True ).start() self.mock_has_grace_period_passed = mock.patch( - "scheduler.schedulers.BoefjeScheduler.has_grace_period_passed", - return_value=True, + "scheduler.schedulers.BoefjeScheduler.has_grace_period_passed", return_value=True ).start() self.mock_get_new_boefjes_by_org_id = mock.patch( @@ -1421,18 +1242,15 @@ def setUp(self): super().setUp() self.mock_is_task_running = mock.patch( - "scheduler.schedulers.BoefjeScheduler.is_task_running", - return_value=False, + "scheduler.schedulers.BoefjeScheduler.is_task_running", return_value=False ).start() self.mock_is_task_allowed_to_run = mock.patch( - "scheduler.schedulers.BoefjeScheduler.is_task_allowed_to_run", - return_value=True, + "scheduler.schedulers.BoefjeScheduler.is_task_allowed_to_run", return_value=True ).start() self.mock_has_grace_period_passed = mock.patch( - "scheduler.schedulers.BoefjeScheduler.has_grace_period_passed", - return_value=True, + "scheduler.schedulers.BoefjeScheduler.has_grace_period_passed", return_value=True ).start() self.mock_get_boefjes_for_ooi = mock.patch("scheduler.schedulers.BoefjeScheduler.get_boefjes_for_ooi").start() @@ -1523,18 +1341,10 @@ def test_push_tasks_for_random_objects_prior_tasks(self, mock_get_tasks_by_hash) scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = models.PrioritizedItem( - id=task.id, - scheduler_id=self.scheduler.scheduler_id, - priority=1, - data=task.model_dump(), - hash=task.hash, + id=task.id, scheduler_id=self.scheduler.scheduler_id, priority=1, data=task.model_dump(), hash=task.hash ) task_db = models.Task( diff --git a/mula/tests/integration/test_normalizer_scheduler.py b/mula/tests/integration/test_normalizer_scheduler.py index 5631a0ff690..b5b754676a6 100644 --- a/mula/tests/integration/test_normalizer_scheduler.py +++ b/mula/tests/integration/test_normalizer_scheduler.py @@ -40,9 +40,7 @@ def setUp(self): # Scheduler self.organisation = OrganisationFactory() self.scheduler = schedulers.NormalizerScheduler( - ctx=self.mock_ctx, - scheduler_id=self.organisation.id, - organisation=self.organisation, + ctx=self.mock_ctx, scheduler_id=self.organisation.id, organisation=self.organisation ) def tearDown(self): @@ -177,13 +175,11 @@ def setUp(self): super().setUp() self.mock_is_task_running = mock.patch( - "scheduler.schedulers.NormalizerScheduler.is_task_running", - return_value=False, + "scheduler.schedulers.NormalizerScheduler.is_task_running", return_value=False ).start() self.mock_is_task_allowed_to_run = mock.patch( - "scheduler.schedulers.NormalizerScheduler.is_task_allowed_to_run", - return_value=True, + "scheduler.schedulers.NormalizerScheduler.is_task_allowed_to_run", return_value=True ).start() self.mock_get_normalizers_for_mime_type = mock.patch( @@ -195,35 +191,22 @@ def test_push_tasks_for_received_raw_file(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - boefje_task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + boefje_task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1, data=boefje_task) task = functions.create_task(p_item) self.mock_ctx.datastores.task_store.create_task(task) - boefje_meta = BoefjeMetaFactory( - id=p_item.id, - boefje=boefje, - input_ooi=ooi.primary_key, - ) + boefje_meta = BoefjeMetaFactory(id=p_item.id, boefje=boefje, input_ooi=ooi.primary_key) raw_data_event = models.RawDataReceivedEvent( - raw_data=RawDataFactory( - boefje_meta=boefje_meta, - mime_types=[{"value": "text/plain"}], - ), + raw_data=RawDataFactory(boefje_meta=boefje_meta, mime_types=[{"value": "text/plain"}]), organization=self.organisation.name, created_at=datetime.datetime.now(), ).model_dump_json() # Mocks - self.mock_get_normalizers_for_mime_type.return_value = [ - NormalizerFactory(), - ] + self.mock_get_normalizers_for_mime_type.return_value = [NormalizerFactory()] # Act self.scheduler.push_tasks_for_received_raw_data(raw_data_event) @@ -242,27 +225,16 @@ def test_push_tasks_for_received_raw_file_no_normalizers_found(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - boefje_task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + boefje_task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1, data=boefje_task) task = functions.create_task(p_item) self.mock_ctx.datastores.task_store.create_task(task) - boefje_meta = BoefjeMetaFactory( - id=p_item.id, - boefje=boefje, - input_ooi=ooi.primary_key, - ) + boefje_meta = BoefjeMetaFactory(id=p_item.id, boefje=boefje, input_ooi=ooi.primary_key) raw_data_event = models.RawDataReceivedEvent( - raw_data=RawDataFactory( - boefje_meta=boefje_meta, - mime_types=[{"value": "text/plain"}], - ), + raw_data=RawDataFactory(boefje_meta=boefje_meta, mime_types=[{"value": "text/plain"}]), organization=self.organisation.name, created_at=datetime.datetime.now(), ).model_dump_json() @@ -281,35 +253,22 @@ def test_push_tasks_for_received_raw_file_not_allowed_to_run(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - boefje_task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + boefje_task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1, data=boefje_task) task = functions.create_task(p_item) self.mock_ctx.datastores.task_store.create_task(task) - boefje_meta = BoefjeMetaFactory( - id=p_item.id, - boefje=boefje, - input_ooi=ooi.primary_key, - ) + boefje_meta = BoefjeMetaFactory(id=p_item.id, boefje=boefje, input_ooi=ooi.primary_key) # Mocks raw_data_event = models.RawDataReceivedEvent( - raw_data=RawDataFactory( - boefje_meta=boefje_meta, - mime_types=[{"value": "text/plain"}], - ), + raw_data=RawDataFactory(boefje_meta=boefje_meta, mime_types=[{"value": "text/plain"}]), organization=self.organisation.name, created_at=datetime.datetime.now(), ).model_dump_json() - self.mock_get_normalizers_for_mime_type.return_value = [ - NormalizerFactory(), - ] + self.mock_get_normalizers_for_mime_type.return_value = [NormalizerFactory()] self.mock_is_task_allowed_to_run.return_value = False # Act @@ -323,35 +282,22 @@ def test_push_tasks_for_received_raw_file_still_running(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - boefje_task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + boefje_task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1, data=boefje_task) task = functions.create_task(p_item) self.mock_ctx.datastores.task_store.create_task(task) - boefje_meta = BoefjeMetaFactory( - id=p_item.id, - boefje=boefje, - input_ooi=ooi.primary_key, - ) + boefje_meta = BoefjeMetaFactory(id=p_item.id, boefje=boefje, input_ooi=ooi.primary_key) # Mocks raw_data_event = models.RawDataReceivedEvent( - raw_data=RawDataFactory( - boefje_meta=boefje_meta, - mime_types=[{"value": "text/plain"}], - ), + raw_data=RawDataFactory(boefje_meta=boefje_meta, mime_types=[{"value": "text/plain"}]), organization=self.organisation.name, created_at=datetime.datetime.now(), ).model_dump_json() - self.mock_get_normalizers_for_mime_type.return_value = [ - NormalizerFactory(), - ] + self.mock_get_normalizers_for_mime_type.return_value = [NormalizerFactory()] self.mock_is_task_allowed_to_run.return_value = True self.mock_is_task_running.return_value = True @@ -366,35 +312,22 @@ def test_push_tasks_for_received_raw_file_still_running_exception(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - boefje_task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + boefje_task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1, data=boefje_task) task = functions.create_task(p_item) self.mock_ctx.datastores.task_store.create_task(task) - boefje_meta = BoefjeMetaFactory( - id=p_item.id, - boefje=boefje, - input_ooi=ooi.primary_key, - ) + boefje_meta = BoefjeMetaFactory(id=p_item.id, boefje=boefje, input_ooi=ooi.primary_key) # Mocks raw_data_event = models.RawDataReceivedEvent( - raw_data=RawDataFactory( - boefje_meta=boefje_meta, - mime_types=[{"value": "text/plain"}], - ), + raw_data=RawDataFactory(boefje_meta=boefje_meta, mime_types=[{"value": "text/plain"}]), organization=self.organisation.name, created_at=datetime.datetime.now(), ).model_dump_json() - self.mock_get_normalizers_for_mime_type.return_value = [ - NormalizerFactory(), - ] + self.mock_get_normalizers_for_mime_type.return_value = [NormalizerFactory()] self.mock_is_task_allowed_to_run.return_value = True self.mock_is_task_running.side_effect = Exception("Something went wrong") @@ -409,44 +342,28 @@ def test_push_tasks_for_received_raw_file_item_on_queue(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - boefje_task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + boefje_task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1, data=boefje_task) task = functions.create_task(p_item) self.mock_ctx.datastores.task_store.create_task(task) - boefje_meta = BoefjeMetaFactory( - id=p_item.id, - boefje=boefje, - input_ooi=ooi.primary_key, - ) + boefje_meta = BoefjeMetaFactory(id=p_item.id, boefje=boefje, input_ooi=ooi.primary_key) # Mocks raw_data_event1 = models.RawDataReceivedEvent( - raw_data=RawDataFactory( - boefje_meta=boefje_meta, - mime_types=[{"value": "text/plain"}], - ), + raw_data=RawDataFactory(boefje_meta=boefje_meta, mime_types=[{"value": "text/plain"}]), organization=self.organisation.name, created_at=datetime.datetime.now(), ).model_dump_json() raw_data_event2 = models.RawDataReceivedEvent( - raw_data=RawDataFactory( - boefje_meta=boefje_meta, - mime_types=[{"value": "text/plain"}], - ), + raw_data=RawDataFactory(boefje_meta=boefje_meta, mime_types=[{"value": "text/plain"}]), organization=self.organisation.name, created_at=datetime.datetime.now(), ).model_dump_json() - self.mock_get_normalizers_for_mime_type.return_value = [ - NormalizerFactory(), - ] + self.mock_get_normalizers_for_mime_type.return_value = [NormalizerFactory()] # Act self.scheduler.push_tasks_for_received_raw_data(raw_data_event1) @@ -466,27 +383,16 @@ def test_push_tasks_for_received_raw_file_error_mimetype(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - boefje_task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + boefje_task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1, data=boefje_task) task = functions.create_task(p_item) self.mock_ctx.datastores.task_store.create_task(task) - boefje_meta = BoefjeMetaFactory( - id=p_item.id, - boefje=boefje, - input_ooi=ooi.primary_key, - ) + boefje_meta = BoefjeMetaFactory(id=p_item.id, boefje=boefje, input_ooi=ooi.primary_key) raw_data_event = models.RawDataReceivedEvent( - raw_data=RawDataFactory( - boefje_meta=boefje_meta, - mime_types=[{"value": "error/unknown"}], - ), + raw_data=RawDataFactory(boefje_meta=boefje_meta, mime_types=[{"value": "error/unknown"}]), organization=self.organisation.name, created_at=datetime.datetime.now(), ).model_dump_json() @@ -504,27 +410,16 @@ def test_push_tasks_for_received_raw_file_queue_full(self): scan_profile = ScanProfileFactory(level=0) ooi = OOIFactory(scan_profile=scan_profile) boefje = BoefjeFactory() - boefje_task = models.BoefjeTask( - boefje=boefje, - input_ooi=ooi.primary_key, - organization=self.organisation.id, - ) + boefje_task = models.BoefjeTask(boefje=boefje, input_ooi=ooi.primary_key, organization=self.organisation.id) p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1, data=boefje_task) task = functions.create_task(p_item) self.mock_ctx.datastores.task_store.create_task(task) - boefje_meta = BoefjeMetaFactory( - id=p_item.id, - boefje=boefje, - input_ooi=ooi.primary_key, - ) + boefje_meta = BoefjeMetaFactory(id=p_item.id, boefje=boefje, input_ooi=ooi.primary_key) raw_data_event = models.RawDataReceivedEvent( - raw_data=RawDataFactory( - boefje_meta=boefje_meta, - mime_types=[{"value": "text/plain"}], - ), + raw_data=RawDataFactory(boefje_meta=boefje_meta, mime_types=[{"value": "text/plain"}]), organization=self.organisation.name, created_at=datetime.datetime.now(), ).model_dump_json() @@ -535,9 +430,7 @@ def test_push_tasks_for_received_raw_file_queue_full(self): self.scheduler.max_tries = 1 # Mocks - self.mock_get_normalizers_for_mime_type.return_value = [ - NormalizerFactory(), - ] + self.mock_get_normalizers_for_mime_type.return_value = [NormalizerFactory()] # Act self.scheduler.push_tasks_for_received_raw_data(events[0]) diff --git a/mula/tests/integration/test_scheduler.py b/mula/tests/integration/test_scheduler.py index 9727414bc16..1843b623114 100644 --- a/mula/tests/integration/test_scheduler.py +++ b/mula/tests/integration/test_scheduler.py @@ -39,11 +39,7 @@ def setUp(self): pq_store=self.mock_ctx.datastores.pq_store, ) - self.scheduler = mock_scheduler.MockScheduler( - ctx=self.mock_ctx, - scheduler_id=identifier, - queue=queue, - ) + self.scheduler = mock_scheduler.MockScheduler(ctx=self.mock_ctx, scheduler_id=identifier, queue=queue) def tearDown(self): self.scheduler.stop() @@ -53,10 +49,7 @@ def tearDown(self): def test_post_push(self): """When a task is added to the queue, it should be added to the database""" # Arrange - p_item = functions.create_p_item( - scheduler_id=self.scheduler.scheduler_id, - priority=1, - ) + p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1) # Act self.scheduler.push_item_to_queue(p_item) @@ -74,10 +67,7 @@ def test_post_push(self): def test_post_pop(self): """When a task is popped from the queue, it should be removed from the database""" # Arrange - p_item = functions.create_p_item( - scheduler_id=self.scheduler.scheduler_id, - priority=1, - ) + p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1) # Act self.scheduler.push_item_to_queue(p_item) @@ -106,10 +96,7 @@ def test_disable_scheduler(self): self.scheduler.run() # Arrange: add tasks - p_item = functions.create_p_item( - scheduler_id=self.scheduler.scheduler_id, - priority=1, - ) + p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1) self.scheduler.push_item_to_queue(p_item) # Assert: task should be on priority queue @@ -156,10 +143,7 @@ def test_enable_scheduler(self): self.scheduler.run() # Arrange: add tasks - p_item = functions.create_p_item( - scheduler_id=self.scheduler.scheduler_id, - priority=1, - ) + p_item = functions.create_p_item(scheduler_id=self.scheduler.scheduler_id, priority=1) self.scheduler.push_item_to_queue(p_item) # Assert: listeners should be running diff --git a/mula/tests/integration/test_services.py b/mula/tests/integration/test_services.py index ba92d5a30fa..a02a245c5ff 100644 --- a/mula/tests/integration/test_services.py +++ b/mula/tests/integration/test_services.py @@ -29,20 +29,12 @@ def test_login(self): @unittest.skip def test_expired_token_refresh(self): - self.service_bytes.get_last_run_boefje( - boefje_id="boefje-1", - input_ooi="ooi-1", - organization_id="org-1", - ) + self.service_bytes.get_last_run_boefje(boefje_id="boefje-1", input_ooi="ooi-1", organization_id="org-1") initial_token = id(self.service_bytes.headers.get("Authorization")) time.sleep(70) - self.service_bytes.get_last_run_boefje( - boefje_id="boefje-1", - input_ooi="ooi-1", - organization_id="org-1", - ) + self.service_bytes.get_last_run_boefje(boefje_id="boefje-1", input_ooi="ooi-1", organization_id="org-1") refresh_token = id(self.service_bytes.headers.get("Authorization")) self.assertNotEqual(initial_token, refresh_token) diff --git a/mula/tests/integration/test_task_store.py b/mula/tests/integration/test_task_store.py index af193d82b60..1c9018b4e73 100644 --- a/mula/tests/integration/test_task_store.py +++ b/mula/tests/integration/test_task_store.py @@ -45,13 +45,7 @@ def test_get_status_counts(self): twenty_five_hours = datetime.now(timezone.utc) - timedelta(hours=25) for r, status, modified_at in zip( - ( - range(2), - range(2), - range(2), - range(2), - range(2), - ), + (range(2), range(2), range(2), range(2), range(2)), ( models.TaskStatus.QUEUED, models.TaskStatus.COMPLETED, @@ -59,13 +53,7 @@ def test_get_status_counts(self): models.TaskStatus.DISPATCHED, models.TaskStatus.DISPATCHED, ), - ( - one_hour, - four_hours, - one_hour, - twenty_five_hours, - twenty_three_hours, - ), + (one_hour, four_hours, one_hour, twenty_five_hours, twenty_three_hours), ): for _ in r: p_item = functions.create_p_item(self.organisation.id, 1) @@ -97,13 +85,7 @@ def test_get_status_count_per_hour(self): twenty_five_hours = datetime.now(timezone.utc) - timedelta(hours=25) for r, status, modified_at in zip( - ( - range(2), - range(2), - range(2), - range(2), - range(2), - ), + (range(2), range(2), range(2), range(2), range(2)), ( models.TaskStatus.QUEUED, models.TaskStatus.COMPLETED, @@ -111,13 +93,7 @@ def test_get_status_count_per_hour(self): models.TaskStatus.DISPATCHED, models.TaskStatus.DISPATCHED, ), - ( - one_hour, - four_hours, - one_hour, - twenty_five_hours, - twenty_three_hours, - ), + (one_hour, four_hours, one_hour, twenty_five_hours, twenty_three_hours), ): for _ in r: p_item = functions.create_p_item(self.organisation.id, 1) @@ -153,20 +129,10 @@ def test_get_tasks_filter_multiple_and(self): f_req = filters.FilterRequest( filters={ "and": [ - filters.Filter( - column="id", - field=None, - operator="eq", - value=first_task.id.hex, - ), - filters.Filter( - column="p_item", - field="data__id", - operator="eq", - value=first_p_item.data.get("id"), - ), + filters.Filter(column="id", field=None, operator="eq", value=first_task.id.hex), + filters.Filter(column="p_item", field="data__id", operator="eq", value=first_p_item.data.get("id")), ] - }, + } ) # Act @@ -190,20 +156,10 @@ def test_get_tasks_filter_multiple_or(self): f_req = filters.FilterRequest( filters={ "or": [ - filters.Filter( - column="id", - field=None, - operator="eq", - value=first_task.id.hex, - ), - filters.Filter( - column="id", - field=None, - operator="eq", - value=second_task.id.hex, - ), + filters.Filter(column="id", field=None, operator="eq", value=first_task.id.hex), + filters.Filter(column="id", field=None, operator="eq", value=second_task.id.hex), ] - }, + } ) # Act @@ -244,28 +200,13 @@ def test_get_tasks_filter_multiple_not(self): f_req = filters.FilterRequest( filters={ "and": [ + filters.Filter(column="id", field=None, operator="eq", value=first_task.id.hex), filters.Filter( - column="id", - field=None, - operator="eq", - value=first_task.id.hex, - ), - filters.Filter( - column="p_item", - field="data", - operator="@>", - value=json.dumps({"categories": ["test-a"]}), + column="p_item", field="data", operator="@>", value=json.dumps({"categories": ["test-a"]}) ), ], - "not": [ - filters.Filter( - column="p_item", - field="data__count", - operator=">", - value=1, - ), - ], - }, + "not": [filters.Filter(column="p_item", field="data__count", operator=">", value=1)], + } ) # Act @@ -287,14 +228,7 @@ def test_get_tasks_filter_eq(self): self.mock_ctx.datastores.task_store.create_task(second_task) f_req = filters.FilterRequest( - filters=[ - filters.Filter( - column="id", - field=None, - operator="eq", - value=first_task.id.hex, - ) - ], + filters=[filters.Filter(column="id", field=None, operator="eq", value=first_task.id.hex)] ) # Act @@ -316,14 +250,7 @@ def test_get_tasks_filter_ne(self): self.mock_ctx.datastores.task_store.create_task(second_task) f_req = filters.FilterRequest( - filters=[ - filters.Filter( - column="id", - field=None, - operator="ne", - value=first_task.id.hex, - ) - ], + filters=[filters.Filter(column="id", field=None, operator="ne", value=first_task.id.hex)] ) # Act @@ -346,13 +273,8 @@ def test_get_tasks_filter_json_eq(self): f_req = filters.FilterRequest( filters=[ - filters.Filter( - column="p_item", - field="data__id", - operator="eq", - value=first_p_item.data.get("id"), - ) - ], + filters.Filter(column="p_item", field="data__id", operator="eq", value=first_p_item.data.get("id")) + ] ) # Act @@ -366,13 +288,8 @@ def test_get_tasks_filter_json_eq(self): # Arrange f_req = filters.FilterRequest( filters=[ - filters.Filter( - column="p_item", - field="data__id", - operator="==", - value=first_p_item.data.get("id"), - ) - ], + filters.Filter(column="p_item", field="data__id", operator="==", value=first_p_item.data.get("id")) + ] ) # Act @@ -395,13 +312,8 @@ def test_get_tasks_filter_json_ne(self): f_req = filters.FilterRequest( filters=[ - filters.Filter( - column="p_item", - field="data__id", - operator="ne", - value=second_p_item.data.get("id"), - ) - ], + filters.Filter(column="p_item", field="data__id", operator="ne", value=second_p_item.data.get("id")) + ] ) # Act @@ -429,12 +341,9 @@ def test_get_tasks_filter_json_gt(self): f_req = filters.FilterRequest( filters=[ filters.Filter( - column="p_item", - field="data__count", - operator="gt", - value=first_p_item.data.get("count"), + column="p_item", field="data__count", operator="gt", value=first_p_item.data.get("count") ) - ], + ] ) # Act @@ -448,13 +357,8 @@ def test_get_tasks_filter_json_gt(self): # Arrange f_req = filters.FilterRequest( filters=[ - filters.Filter( - column="p_item", - field="data__count", - operator=">", - value=first_p_item.data.get("count"), - ) - ], + filters.Filter(column="p_item", field="data__count", operator=">", value=first_p_item.data.get("count")) + ] ) # Act @@ -482,12 +386,9 @@ def test_get_tasks_filter_json_gte(self): f_req = filters.FilterRequest( filters=[ filters.Filter( - column="p_item", - field="data__count", - operator="gte", - value=first_p_item.data.get("count"), + column="p_item", field="data__count", operator="gte", value=first_p_item.data.get("count") ) - ], + ] ) # Act @@ -503,12 +404,9 @@ def test_get_tasks_filter_json_gte(self): f_req = filters.FilterRequest( filters=[ filters.Filter( - column="p_item", - field="data__count", - operator=">=", - value=first_p_item.data.get("count"), + column="p_item", field="data__count", operator=">=", value=first_p_item.data.get("count") ) - ], + ] ) # Act @@ -537,12 +435,9 @@ def test_get_tasks_filter_json_lt(self): f_req = filters.FilterRequest( filters=[ filters.Filter( - column="p_item", - field="data__count", - operator="lt", - value=first_p_item.data.get("count"), + column="p_item", field="data__count", operator="lt", value=first_p_item.data.get("count") ) - ], + ] ) # Act @@ -556,13 +451,8 @@ def test_get_tasks_filter_json_lt(self): # Arrange f_req = filters.FilterRequest( filters=[ - filters.Filter( - column="p_item", - field="data__count", - operator="<", - value=first_p_item.data.get("count"), - ) - ], + filters.Filter(column="p_item", field="data__count", operator="<", value=first_p_item.data.get("count")) + ] ) # Act @@ -590,12 +480,9 @@ def test_get_tasks_filter_json_lte(self): f_req = filters.FilterRequest( filters=[ filters.Filter( - column="p_item", - field="data__count", - operator="lte", - value=first_p_item.data.get("count"), + column="p_item", field="data__count", operator="lte", value=first_p_item.data.get("count") ) - ], + ] ) # Act @@ -611,12 +498,9 @@ def test_get_tasks_filter_json_lte(self): f_req = filters.FilterRequest( filters=[ filters.Filter( - column="p_item", - field="data__count", - operator="<=", - value=first_p_item.data.get("count"), + column="p_item", field="data__count", operator="<=", value=first_p_item.data.get("count") ) - ], + ] ) # Act @@ -646,7 +530,7 @@ def test_get_tasks_filter_json_like(self): operator="like", value=f"%{first_p_item.data.get('name')[0:10]}%", ) - ], + ] ) # Act @@ -675,7 +559,7 @@ def test_get_tasks_filter_json_not_like(self): operator="not_like", value=f"%{first_p_item.data.get('name')[0:10]}%", ) - ], + ] ) # Act @@ -704,7 +588,7 @@ def test_get_tasks_filter_json_ilike(self): operator="ilike", value=f"%{first_p_item.data.get('name')[0:10].upper()}%", ) - ], + ] ) # Act @@ -733,7 +617,7 @@ def test_get_tasks_filter_json_not_ilike(self): operator="not_ilike", value=f"%{first_p_item.data.get('name')[0:10].upper()}%", ) - ], + ] ) # Act @@ -762,7 +646,7 @@ def test_get_tasks_filter_json_in(self): operator="in", value=[first_p_item.data.get("name"), second_p_item.data.get("name")], ) - ], + ] ) # Act @@ -787,12 +671,9 @@ def test_get_tasks_filter_json_not_in(self): f_req = filters.FilterRequest( filters=[ filters.Filter( - column="p_item", - field="data__name", - operator="not_in", - value=[first_p_item.data.get("name")], + column="p_item", field="data__name", operator="not_in", value=[first_p_item.data.get("name")] ) - ], + ] ) # Act @@ -816,12 +697,9 @@ def test_get_tasks_filter_json_contains(self): f_req = filters.FilterRequest( filters=[ filters.Filter( - column="p_item", - field="data__name", - operator="contains", - value=first_p_item.data.get("name")[0:10], + column="p_item", field="data__name", operator="contains", value=first_p_item.data.get("name")[0:10] ) - ], + ] ) # Act @@ -850,7 +728,7 @@ def test_get_tasks_filter_json_starts_with(self): operator="starts_with", value=first_p_item.data.get("name")[0:10], ) - ], + ] ) # Act @@ -890,12 +768,9 @@ def test_get_tasks_filter_jsonb_contains(self): f_req = filters.FilterRequest( filters=[ filters.Filter( - column="p_item", - field="data", - operator="@>", - value=json.dumps({"categories": ["test-a"]}), + column="p_item", field="data", operator="@>", value=json.dumps({"categories": ["test-a"]}) ) - ], + ] ) # Act @@ -918,14 +793,7 @@ def test_get_tasks_filter_json_mismatch(self): self.mock_ctx.datastores.task_store.create_task(second_task) f_req = filters.FilterRequest( - filters=[ - filters.Filter( - column="p_item", - field="data__id", - operator="eq", - value=False, - ) - ], + filters=[filters.Filter(column="p_item", field="data__id", operator="eq", value=False)] ) # Act diff --git a/mula/tests/mocks/scheduler.py b/mula/tests/mocks/scheduler.py index 1aaae2d212f..e6b91c7e400 100644 --- a/mula/tests/mocks/scheduler.py +++ b/mula/tests/mocks/scheduler.py @@ -9,18 +9,10 @@ class MockScheduler(schedulers.Scheduler): def run(self) -> None: # Listener self.listeners["mock-listener"] = listener.MockListener() - self.run_in_thread( - name="mock-listener", - target=self.listeners["mock-listener"].listen, - loop=True, - ) + self.run_in_thread(name="mock-listener", target=self.listeners["mock-listener"].listen, loop=True) # Thread - self.run_in_thread( - name="mock-scheduler", - target=self._run, - loop=True, - ) + self.run_in_thread(name="mock-scheduler", target=self._run, loop=True) def _run(self) -> None: time.sleep(1) diff --git a/mula/tests/scripts/benchmark.py b/mula/tests/scripts/benchmark.py index 4449af1dff1..ddd36afdfe0 100644 --- a/mula/tests/scripts/benchmark.py +++ b/mula/tests/scripts/benchmark.py @@ -14,10 +14,7 @@ def are_tasks_done() -> bool: - response = httpx.get( - url=f"{SCHEDULER_API}/tasks/stats", - timeout=30, - ) + response = httpx.get(url=f"{SCHEDULER_API}/tasks/stats", timeout=30) try: response.raise_for_status() @@ -31,10 +28,7 @@ def are_tasks_done() -> bool: def parse_stats() -> None: - resp_tasks_stats = httpx.get( - url=f"{SCHEDULER_API}/tasks/stats", - timeout=30, - ) + resp_tasks_stats = httpx.get(url=f"{SCHEDULER_API}/tasks/stats", timeout=30) try: resp_tasks_stats.raise_for_status() @@ -49,25 +43,13 @@ def parse_stats() -> None: failed = tasks_stats[hour].get("failed") completed = tasks_stats[hour].get("completed") - logger.info( - "HOUR %s, QUEUED %s, RUNNING %s, FAILED %s, COMPLETED %s", - hour, - queued, - running, - failed, - completed, - ) + logger.info("HOUR %s, QUEUED %s, RUNNING %s, FAILED %s, COMPLETED %s", hour, queued, running, failed, completed) def capture_logs(container_id: str, output_file: str) -> None: # Capture logs with Path.open(output_file, "w", encoding="utf-8") as file: - subprocess.run( - ["docker", "logs", container_id], - stdout=file, - stderr=file, - check=True, - ) + subprocess.run(["docker", "logs", container_id], stdout=file, stderr=file, check=True) def parse_logs(path: str) -> None: @@ -148,11 +130,7 @@ def run(container_id: str) -> None: parser.add_argument("--verbose", "-v", action="store_true", help="Set to enable verbose logging.") parser.add_argument( - "--container-id", - "-c", - type=str, - required=False, - help="The container id of the process to monitor.", + "--container-id", "-c", type=str, required=False, help="The container id of the process to monitor." ) # Parse arguments @@ -165,9 +143,6 @@ def run(container_id: str) -> None: if args.verbose: default_loglevel = logging.DEBUG - logging.basicConfig( - level=level, - format="%(asctime)s %(name)-10s %(levelname)-8s %(message)s", - ) + logging.basicConfig(level=level, format="%(asctime)s %(name)-10s %(levelname)-8s %(message)s") run(args.container_id) diff --git a/mula/tests/scripts/load.py b/mula/tests/scripts/load.py index 01f3b02b598..0b58ea1d466 100644 --- a/mula/tests/scripts/load.py +++ b/mula/tests/scripts/load.py @@ -16,17 +16,10 @@ def run(org_num: int = 1): # Create organisations orgs: list[dict[str, Any]] = [] for n in range(0, org_num): - org = { - "id": f"org-{n}", - "name": f"Organisation {n}", - } + org = {"id": f"org-{n}", "name": f"Organisation {n}"} orgs.append(org) - resp_katalogus = httpx.post( - url=f"{KATALOGUS_API}/v1/organisations/", - json=org, - timeout=30, - ) + resp_katalogus = httpx.post(url=f"{KATALOGUS_API}/v1/organisations/", json=org, timeout=30) try: resp_katalogus.raise_for_status() @@ -39,10 +32,7 @@ def run(org_num: int = 1): print("Organisation already exists in katalogus", org) try: - httpx.post( - url=f"{OCTOPOES_API}/{org.get('id')}/node/", - timeout=30, - ) + httpx.post(url=f"{OCTOPOES_API}/{org.get('id')}/node/", timeout=30) except httpx.HTTPError: print("Error creating organisation ", org) raise diff --git a/mula/tests/simulation/test_simulation.py b/mula/tests/simulation/test_simulation.py index cbaaf00dde8..3eb70c4c63f 100644 --- a/mula/tests/simulation/test_simulation.py +++ b/mula/tests/simulation/test_simulation.py @@ -33,9 +33,7 @@ def create_normalizer_scheduler_for_organisation(self, organisation): allow_priority_updates=True, ) - normalizer_ranker = rankers.NormalizerRanker( - ctx=self.mock_ctx, - ) + normalizer_ranker = rankers.NormalizerRanker(ctx=self.mock_ctx) return schedulers.NormalizerScheduler( ctx=self.mock_ctx, @@ -53,9 +51,7 @@ def create_boefje_scheduler_for_organisation(self, organisation): allow_priority_updates=True, ) - boefje_ranker = rankers.BoefjeRanker( - ctx=self.mock_ctx, - ) + boefje_ranker = rankers.BoefjeRanker(ctx=self.mock_ctx) return schedulers.BoefjeScheduler( ctx=self.mock_ctx, @@ -121,9 +117,7 @@ def test_simulation_normalizer_queue(self, mock_get_latest_raw_data, mock_create queues.PrioritizedItem( 0, models.NormalizerTask( - id=uuid.uuid4(), - normalizer=PluginFactory(type="normalizer"), - boefje_meta=raw_file.boefje_meta, + id=uuid.uuid4(), normalizer=PluginFactory(type="normalizer"), boefje_meta=raw_file.boefje_meta ), ) ] diff --git a/mula/tests/unit/test_filter.py b/mula/tests/unit/test_filter.py index 8a5f447ea71..079565499fd 100644 --- a/mula/tests/unit/test_filter.py +++ b/mula/tests/unit/test_filter.py @@ -88,7 +88,7 @@ def test_apply_filter_nested_fields(self): Filter(column="name", operator="eq", value="Alice"), Filter(column="data", field="foo", operator="eq", value="bar"), ] - }, + } ) query = session.query(TestModel) @@ -162,11 +162,7 @@ def test_apply_filter_and(self): self.assertEqual(results[0].name, "Alice") def test_apply_filter_not(self): - filter_request = FilterRequest( - filters={ - "not": [Filter(column="name", operator="eq", value="Alice")], - } - ) + filter_request = FilterRequest(filters={"not": [Filter(column="name", operator="eq", value="Alice")]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -231,13 +227,7 @@ def test_apply_filter_boolean(self): self.assertEqual(results[1].name, "Charlie") def test_apply_filter_eq(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="name", operator="eq", value="Alice"), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="name", operator="eq", value="Alice")]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -247,13 +237,7 @@ def test_apply_filter_eq(self): self.assertEqual(len(results), 1) self.assertEqual(results[0].name, "Alice") - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="name", operator="==", value="Alice"), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="name", operator="==", value="Alice")]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -265,11 +249,7 @@ def test_apply_filter_eq(self): def test_apply_filter_json_eq(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="foo", operator="eq", value="bar"), - ] - } + filters={"and": [Filter(column="data", field="foo", operator="eq", value="bar")]} ) query = session.query(TestModel) @@ -282,11 +262,7 @@ def test_apply_filter_json_eq(self): self.assertEqual(results[1].name, "Charlie") filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="foo", operator="==", value="bar"), - ] - } + filters={"and": [Filter(column="data", field="foo", operator="==", value="bar")]} ) query = session.query(TestModel) @@ -299,13 +275,7 @@ def test_apply_filter_json_eq(self): self.assertEqual(results[1].name, "Charlie") def test_apply_filter_ne(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="name", operator="ne", value="Alice"), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="name", operator="ne", value="Alice")]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -316,13 +286,7 @@ def test_apply_filter_ne(self): self.assertEqual(results[0].name, "Bob") self.assertEqual(results[1].name, "Charlie") - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="name", operator="!=", value="Alice"), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="name", operator="!=", value="Alice")]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -334,11 +298,7 @@ def test_apply_filter_ne(self): def test_apply_filter_json_ne(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="foo", operator="ne", value="bar"), - ] - } + filters={"and": [Filter(column="data", field="foo", operator="ne", value="bar")]} ) query = session.query(TestModel) @@ -350,11 +310,7 @@ def test_apply_filter_json_ne(self): self.assertEqual(results[0].name, "Bob") filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="foo", operator="!=", value="bar"), - ] - } + filters={"and": [Filter(column="data", field="foo", operator="!=", value="bar")]} ) query = session.query(TestModel) @@ -366,11 +322,7 @@ def test_apply_filter_json_ne(self): self.assertEqual(results[0].name, "Bob") def test_apply_filter_is(self): - filter_request = FilterRequest( - filters=[ - Filter(column="is_active", operator="is", value=True), - ] - ) + filter_request = FilterRequest(filters=[Filter(column="is_active", operator="is", value=True)]) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -381,11 +333,7 @@ def test_apply_filter_is(self): self.assertIn("Charlie", [r.name for r in results]) def test_apply_filter_is_not(self): - filter_request = FilterRequest( - filters=[ - Filter(column="is_active", operator="is_not", value=True), - ] - ) + filter_request = FilterRequest(filters=[Filter(column="is_active", operator="is_not", value=True)]) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -395,13 +343,7 @@ def test_apply_filter_is_not(self): self.assertIn("Bob", [r.name for r in results]) def test_apply_filter_gt(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="age", operator="gt", value=25), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="age", operator="gt", value=25)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -412,13 +354,7 @@ def test_apply_filter_gt(self): self.assertIn("Bob", [r.name for r in results]) self.assertIn("Charlie", [r.name for r in results]) - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="age", operator=">", value=25), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="age", operator=">", value=25)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -430,13 +366,7 @@ def test_apply_filter_gt(self): self.assertIn("Charlie", [r.name for r in results]) def test_apply_filter_json_gt(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="score", operator="gt", value=25), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="data", field="score", operator="gt", value=25)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -446,13 +376,7 @@ def test_apply_filter_json_gt(self): self.assertEqual(len(results), 1) self.assertEqual(results[0].name, "Charlie") - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="score", operator=">", value=25), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="data", field="score", operator=">", value=25)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -463,13 +387,7 @@ def test_apply_filter_json_gt(self): self.assertEqual(results[0].name, "Charlie") def test_apply_filter_gte(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="age", operator="gte", value=25), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="age", operator="gte", value=25)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -481,13 +399,7 @@ def test_apply_filter_gte(self): self.assertIn("Bob", [r.name for r in results]) self.assertIn("Charlie", [r.name for r in results]) - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="age", operator=">=", value=25), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="age", operator=">=", value=25)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -501,11 +413,7 @@ def test_apply_filter_gte(self): def test_apply_filter_json_gte(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="score", operator="gte", value=25), - ] - } + filters={"and": [Filter(column="data", field="score", operator="gte", value=25)]} ) query = session.query(TestModel) @@ -517,13 +425,7 @@ def test_apply_filter_json_gte(self): self.assertIn("Bob", [r.name for r in results]) self.assertIn("Charlie", [r.name for r in results]) - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="score", operator=">=", value=25), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="data", field="score", operator=">=", value=25)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -535,13 +437,7 @@ def test_apply_filter_json_gte(self): self.assertIn("Charlie", [r.name for r in results]) def test_apply_filter_lt(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="age", operator="lt", value=28), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="age", operator="lt", value=28)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -551,13 +447,7 @@ def test_apply_filter_lt(self): self.assertEqual(len(results), 1) self.assertEqual(results[0].name, "Alice") - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="age", operator="<", value=28), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="age", operator="<", value=28)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -568,13 +458,7 @@ def test_apply_filter_lt(self): self.assertEqual(results[0].name, "Alice") def test_apply_filter_json_lt(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="score", operator="lt", value=25), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="data", field="score", operator="lt", value=25)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -584,13 +468,7 @@ def test_apply_filter_json_lt(self): self.assertEqual(len(results), 1) self.assertEqual(results[0].name, "Alice") - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="score", operator="<", value=25), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="data", field="score", operator="<", value=25)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -601,13 +479,7 @@ def test_apply_filter_json_lt(self): self.assertEqual(results[0].name, "Alice") def test_apply_filter_lte(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="age", operator="lte", value=28), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="age", operator="lte", value=28)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -618,13 +490,7 @@ def test_apply_filter_lte(self): self.assertIn("Alice", [r.name for r in results]) self.assertIn("Charlie", [r.name for r in results]) - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="age", operator="<=", value=28), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="age", operator="<=", value=28)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -637,11 +503,7 @@ def test_apply_filter_lte(self): def test_apply_filter_json_lte(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="score", operator="lte", value=25), - ] - } + filters={"and": [Filter(column="data", field="score", operator="lte", value=25)]} ) query = session.query(TestModel) @@ -653,13 +515,7 @@ def test_apply_filter_json_lte(self): self.assertIn("Alice", [r.name for r in results]) self.assertIn("Bob", [r.name for r in results]) - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="score", operator="<=", value=25), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="data", field="score", operator="<=", value=25)]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -671,13 +527,7 @@ def test_apply_filter_json_lte(self): self.assertIn("Bob", [r.name for r in results]) def test_apply_filter_like(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="name", operator="like", value="B%"), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="name", operator="like", value="B%")]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -689,11 +539,7 @@ def test_apply_filter_like(self): def test_apply_filter_json_like(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="foo", operator="like", value="%ar"), - ] - } + filters={"and": [Filter(column="data", field="foo", operator="like", value="%ar")]} ) query = session.query(TestModel) @@ -705,13 +551,7 @@ def test_apply_filter_json_like(self): self.assertEqual(results[1].name, "Charlie") def test_apply_filter_not_like(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="name", operator="not_like", value="B%"), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="name", operator="not_like", value="B%")]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -724,11 +564,7 @@ def test_apply_filter_not_like(self): def test_apply_filter_json_not_like(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="foo", operator="not_like", value="%ar"), - ] - } + filters={"and": [Filter(column="data", field="foo", operator="not_like", value="%ar")]} ) query = session.query(TestModel) @@ -739,13 +575,7 @@ def test_apply_filter_json_not_like(self): self.assertEqual(results[0].name, "Bob") def test_apply_filter_ilike(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="name", operator="ilike", value="B%"), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="name", operator="ilike", value="B%")]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -757,11 +587,7 @@ def test_apply_filter_ilike(self): def test_apply_filter_json_ilike(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="foo", operator="ilike", value="%AR"), - ] - } + filters={"and": [Filter(column="data", field="foo", operator="ilike", value="%AR")]} ) query = session.query(TestModel) @@ -773,13 +599,7 @@ def test_apply_filter_json_ilike(self): self.assertIn("Charlie", [r.name for r in results]) def test_apply_filter_not_ilike(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="name", operator="not_ilike", value="B%"), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="name", operator="not_ilike", value="B%")]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -792,11 +612,7 @@ def test_apply_filter_not_ilike(self): def test_apply_filter_json_not_ilike(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="foo", operator="not_ilike", value="%AR"), - ] - } + filters={"and": [Filter(column="data", field="foo", operator="not_ilike", value="%AR")]} ) query = session.query(TestModel) @@ -807,13 +623,7 @@ def test_apply_filter_json_not_ilike(self): self.assertEqual(results[0].name, "Bob") def test_apply_filter_in(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="name", operator="in", value=["Alice", "Bob"]), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="name", operator="in", value=["Alice", "Bob"])]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -826,11 +636,7 @@ def test_apply_filter_in(self): def test_apply_filter_json_in(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="foo", operator="in", value=["bar", "baz"]), - ] - } + filters={"and": [Filter(column="data", field="foo", operator="in", value=["bar", "baz"])]} ) query = session.query(TestModel) @@ -841,11 +647,7 @@ def test_apply_filter_json_in(self): def test_apply_filter_not_in(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="name", operator="not_in", value=["Alice", "Bob"]), - ] - } + filters={"and": [Filter(column="name", operator="not_in", value=["Alice", "Bob"])]} ) query = session.query(TestModel) @@ -858,11 +660,7 @@ def test_apply_filter_not_in(self): def test_apply_filter_json_not_in(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="foo", operator="not_in", value=["bar"]), - ] - } + filters={"and": [Filter(column="data", field="foo", operator="not_in", value=["bar"])]} ) query = session.query(TestModel) @@ -873,13 +671,7 @@ def test_apply_filter_json_not_in(self): self.assertEqual(results[0].name, "Bob") def test_apply_filter_contains(self): - filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="name", operator="contains", value="li"), - ] - } - ) + filter_request = FilterRequest(filters={"and": [Filter(column="name", operator="contains", value="li")]}) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) @@ -891,11 +683,7 @@ def test_apply_filter_contains(self): def test_apply_filter_json_contains(self): filter_request = FilterRequest( - filters={ - "and": [ - Filter(column="data", field="foo", operator="contains", value="ar"), - ] - } + filters={"and": [Filter(column="data", field="foo", operator="contains", value="ar")]} ) query = session.query(TestModel) @@ -907,11 +695,7 @@ def test_apply_filter_json_contains(self): self.assertIn("Charlie", [r.name for r in results]) def test_apply_filter_jsonb_contains(self): - filter_request = FilterRequest( - filters=[ - Filter(column="data", operator="@>", value=json.dumps({"foo": "bar"})), - ] - ) + filter_request = FilterRequest(filters=[Filter(column="data", operator="@>", value=json.dumps({"foo": "bar"}))]) query = session.query(TestModel) filtered_query = apply_filter(TestModel, query, filter_request) diff --git a/mula/tests/unit/test_queue.py b/mula/tests/unit/test_queue.py index 0ec45d7db39..ff68da6c6c9 100644 --- a/mula/tests/unit/test_queue.py +++ b/mula/tests/unit/test_queue.py @@ -25,10 +25,7 @@ def setUp(self) -> None: # Priority Queue self.pq = mock_queue.MockPriorityQueue( - pq_id="test", - maxsize=10, - item_type=functions.TestModel, - pq_store=self.pq_store, + pq_id="test", maxsize=10, item_type=functions.TestModel, pq_store=self.pq_store ) self._check_queue_empty() diff --git a/mula/tests/utils/functions.py b/mula/tests/utils/functions.py index af08da29598..d31c5cd1edb 100644 --- a/mula/tests/utils/functions.py +++ b/mula/tests/utils/functions.py @@ -24,29 +24,16 @@ def __init__(self, **data: Any): def create_p_item_request(priority: int, data: TestModel | None = None) -> models.PrioritizedItemRequest: if data is None: - data = TestModel( - id=uuid.uuid4().hex, - name=uuid.uuid4().hex, - ) + data = TestModel(id=uuid.uuid4().hex, name=uuid.uuid4().hex) - return models.PrioritizedItemRequest( - priority=priority, - data=data.model_dump(), - ) + return models.PrioritizedItemRequest(priority=priority, data=data.model_dump()) def create_p_item(scheduler_id: str, priority: int, data: TestModel | None = None) -> models.PrioritizedItem: if data is None: - data = TestModel( - id=uuid.uuid4().hex, - name=uuid.uuid4().hex, - ) + data = TestModel(id=uuid.uuid4().hex, name=uuid.uuid4().hex) - return models.PrioritizedItem( - scheduler_id=scheduler_id, - priority=priority, - data=data.model_dump(), - ) + return models.PrioritizedItem(scheduler_id=scheduler_id, priority=priority, data=data.model_dump()) def create_task(p_item: models.PrioritizedItem) -> models.Task: diff --git a/octopoes/.ci/mock_bits/url_classification_mock/url_classification_mock.py b/octopoes/.ci/mock_bits/url_classification_mock/url_classification_mock.py index d32b4134c59..574ffaa4cd6 100644 --- a/octopoes/.ci/mock_bits/url_classification_mock/url_classification_mock.py +++ b/octopoes/.ci/mock_bits/url_classification_mock/url_classification_mock.py @@ -18,12 +18,7 @@ def run(url: URL, additional_oois: list, config: dict) -> Iterator[OOI]: path = url.raw.path if url.raw.path is not None else "/" - default_args = { - "network": url.network, - "scheme": WebScheme(url.raw.scheme), - "port": port, - "path": path, - } + default_args = {"network": url.network, "scheme": WebScheme(url.raw.scheme), "port": port, "path": path} try: addr = ip_address(url.raw.host) except ValueError: diff --git a/octopoes/bits/ask_disallowed_domains/ask_disallowed_domains.py b/octopoes/bits/ask_disallowed_domains/ask_disallowed_domains.py index b8aa105b6c8..971f1d68993 100644 --- a/octopoes/bits/ask_disallowed_domains/ask_disallowed_domains.py +++ b/octopoes/bits/ask_disallowed_domains/ask_disallowed_domains.py @@ -8,11 +8,7 @@ from octopoes.models.ooi.question import Question -def run( - input_ooi: Network, - additional_oois: list, - config: dict[str, Any], -) -> Iterator[OOI]: +def run(input_ooi: Network, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: network = input_ooi with (Path(__file__).parent / "question_schema.json").open() as f: diff --git a/octopoes/bits/ask_port_specification/ask_port_specification.py b/octopoes/bits/ask_port_specification/ask_port_specification.py index b8aa105b6c8..971f1d68993 100644 --- a/octopoes/bits/ask_port_specification/ask_port_specification.py +++ b/octopoes/bits/ask_port_specification/ask_port_specification.py @@ -8,11 +8,7 @@ from octopoes.models.ooi.question import Question -def run( - input_ooi: Network, - additional_oois: list, - config: dict[str, Any], -) -> Iterator[OOI]: +def run(input_ooi: Network, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: network = input_ooi with (Path(__file__).parent / "question_schema.json").open() as f: diff --git a/octopoes/bits/check_csp_header/bit.py b/octopoes/bits/check_csp_header/bit.py index 72bc8be7bc4..359017a70f6 100644 --- a/octopoes/bits/check_csp_header/bit.py +++ b/octopoes/bits/check_csp_header/bit.py @@ -2,8 +2,5 @@ from octopoes.models.types import HTTPHeader BIT = BitDefinition( - id="check-csp-header", - consumes=HTTPHeader, - parameters=[], - module="bits.check_csp_header.check_csp_header", + id="check-csp-header", consumes=HTTPHeader, parameters=[], module="bits.check_csp_header.check_csp_header" ) diff --git a/octopoes/bits/check_csp_header/check_csp_header.py b/octopoes/bits/check_csp_header/check_csp_header.py index c621a652e74..58491451e83 100644 --- a/octopoes/bits/check_csp_header/check_csp_header.py +++ b/octopoes/bits/check_csp_header/check_csp_header.py @@ -97,11 +97,7 @@ def run(input_ooi: HTTPHeader, additional_oois: list, config: dict[str, Any]) -> for index, finding in enumerate(findings): description += f"\n {index + 1}. {finding}" - yield from _create_kat_finding( - header.reference, - kat_id="KAT-CSP-VULNERABILITIES", - description=description, - ) + yield from _create_kat_finding(header.reference, kat_id="KAT-CSP-VULNERABILITIES", description=description) def _ip_valid(source: str) -> bool: @@ -120,27 +116,14 @@ def _ip_valid(source: str) -> bool: def _create_kat_finding(header: Reference, kat_id: str, description: str) -> Iterator[OOI]: finding_type = KATFindingType(id=kat_id) yield finding_type - yield Finding( - finding_type=finding_type.reference, - ooi=header, - description=description, - ) + yield Finding(finding_type=finding_type.reference, ooi=header, description=description) def _source_valid(policy: list[str]) -> bool: for value in policy: if not ( re.search(r"\S+\.\S{2,3}([\s]+|$|;|:[0-9]+)", value) - or value - in [ - "'none'", - "'self'", - "data:", - "unsafe-inline", - "unsafe-eval", - "unsafe-hashes", - "report-sample", - ] + or value in ["'none'", "'self'", "data:", "unsafe-inline", "unsafe-eval", "unsafe-hashes", "report-sample"] ): return False diff --git a/octopoes/bits/check_cve_2021_41773/check_cve_2021_41773.py b/octopoes/bits/check_cve_2021_41773/check_cve_2021_41773.py index c55e786b7fc..d07b81268c0 100644 --- a/octopoes/bits/check_cve_2021_41773/check_cve_2021_41773.py +++ b/octopoes/bits/check_cve_2021_41773/check_cve_2021_41773.py @@ -14,8 +14,4 @@ def run(input_ooi: HTTPHeader, additional_oois: list, config: dict[str, Any]) -> if "Apache/2.4.49" in header.value or "Apache/2.4.50" in header.value: finding_type = CVEFindingType(id="CVE-2021-41773") yield finding_type - yield Finding( - finding_type=finding_type.reference, - ooi=header.reference, - description="Bad apache version", - ) + yield Finding(finding_type=finding_type.reference, ooi=header.reference, description="Bad apache version") diff --git a/octopoes/bits/check_hsts_header/check_hsts_header.py b/octopoes/bits/check_hsts_header/check_hsts_header.py index d7e2dcf0c12..824e6948ace 100644 --- a/octopoes/bits/check_hsts_header/check_hsts_header.py +++ b/octopoes/bits/check_hsts_header/check_hsts_header.py @@ -35,18 +35,10 @@ def run(input_ooi: HTTPHeader, additional_oois: list, config: dict[str, Any]) -> for index, finding in enumerate(findings): description += f"\n {index + 1}. {finding}" - yield from _create_kat_finding( - header.reference, - kat_id="KAT-HSTS-VULNERABILITIES", - description=description, - ) + yield from _create_kat_finding(header.reference, kat_id="KAT-HSTS-VULNERABILITIES", description=description) def _create_kat_finding(header: Reference, kat_id: str, description: str) -> Iterator[OOI]: finding_type = KATFindingType(id=kat_id) yield finding_type - yield Finding( - finding_type=finding_type.reference, - ooi=header, - description=description, - ) + yield Finding(finding_type=finding_type.reference, ooi=header, description=description) diff --git a/octopoes/bits/cipher_classification/cipher_classification.py b/octopoes/bits/cipher_classification/cipher_classification.py index 8a4594613cf..5f701df86cb 100644 --- a/octopoes/bits/cipher_classification/cipher_classification.py +++ b/octopoes/bits/cipher_classification/cipher_classification.py @@ -85,9 +85,7 @@ def run(input_ooi: TLSCipher, additional_oois, config) -> Iterator[OOI]: return if highest_severity in SEVERITY_TO_ID: - ft = KATFindingType( - id=SEVERITY_TO_ID[highest_severity], - ) + ft = KATFindingType(id=SEVERITY_TO_ID[highest_severity]) yield ft yield Finding( finding_type=ft.reference, diff --git a/octopoes/bits/disallowed_csp_hostnames/disallowed_csp_hostnames.py b/octopoes/bits/disallowed_csp_hostnames/disallowed_csp_hostnames.py index 9df38e24c13..a4649cb85ba 100644 --- a/octopoes/bits/disallowed_csp_hostnames/disallowed_csp_hostnames.py +++ b/octopoes/bits/disallowed_csp_hostnames/disallowed_csp_hostnames.py @@ -37,9 +37,6 @@ def run(input_ooi: HTTPHeaderHostname, additional_oois: list, config: dict[str, if hostname.lower() in disallowed_domains: ft = KATFindingType(id="KAT-DISALLOWED-DOMAIN-IN-CSP") - f = Finding( - ooi=input_ooi.reference, - finding_type=ft.reference, - ) + f = Finding(ooi=input_ooi.reference, finding_type=ft.reference) yield ft yield f diff --git a/octopoes/bits/dns_alias_resolving/dns_alias_resolving.py b/octopoes/bits/dns_alias_resolving/dns_alias_resolving.py index 8316eb78f8a..982a822e866 100644 --- a/octopoes/bits/dns_alias_resolving/dns_alias_resolving.py +++ b/octopoes/bits/dns_alias_resolving/dns_alias_resolving.py @@ -15,10 +15,7 @@ def run( for cname_record in cname_records: for resolved_hostname in resolved_hostnames: - yield ResolvedHostname( - hostname=cname_record.hostname, - address=resolved_hostname.address, - ) + yield ResolvedHostname(hostname=cname_record.hostname, address=resolved_hostname.address) # Also the non-fqdn variant yield ResolvedHostname( hostname=Hostname( diff --git a/octopoes/bits/dns_resolving/dns_resolving.py b/octopoes/bits/dns_resolving/dns_resolving.py index 6083284de24..d7653678e6b 100644 --- a/octopoes/bits/dns_resolving/dns_resolving.py +++ b/octopoes/bits/dns_resolving/dns_resolving.py @@ -8,7 +8,4 @@ def run(hostname: Hostname, additional_oois: list[DNSARecord | DNSAAAARecord], config: dict[str, Any]) -> Iterator[OOI]: for record in additional_oois: - yield ResolvedHostname( - hostname=hostname.reference, - address=record.address, - ) + yield ResolvedHostname(hostname=hostname.reference, address=record.address) diff --git a/octopoes/bits/https_redirect/bit.py b/octopoes/bits/https_redirect/bit.py index 0c2661b031f..29ca96dd5b2 100644 --- a/octopoes/bits/https_redirect/bit.py +++ b/octopoes/bits/https_redirect/bit.py @@ -4,8 +4,6 @@ BIT = BitDefinition( id="https-redirect", consumes=HostnameHTTPURL, - parameters=[ - BitParameterDefinition(ooi_type=HTTPHeader, relation_path="resource.web_url"), - ], + parameters=[BitParameterDefinition(ooi_type=HTTPHeader, relation_path="resource.web_url")], module="bits.https_redirect.https_redirect", ) diff --git a/octopoes/bits/missing_caa/missing_caa.py b/octopoes/bits/missing_caa/missing_caa.py index b08b7e391df..f2e63908385 100644 --- a/octopoes/bits/missing_caa/missing_caa.py +++ b/octopoes/bits/missing_caa/missing_caa.py @@ -24,7 +24,5 @@ def run(input_ooi: Hostname, additional_oois: list[DNSCAARecord | NXDOMAIN], con ft = KATFindingType(id="KAT-NO-CAA") yield ft yield Finding( - ooi=input_ooi.reference, - finding_type=ft.reference, - description="This hostname does not have a CAA record", + ooi=input_ooi.reference, finding_type=ft.reference, description="This hostname does not have a CAA record" ) diff --git a/octopoes/bits/missing_certificate/bit.py b/octopoes/bits/missing_certificate/bit.py index 65dff3448e8..330ac1b65d9 100644 --- a/octopoes/bits/missing_certificate/bit.py +++ b/octopoes/bits/missing_certificate/bit.py @@ -2,8 +2,5 @@ from octopoes.models.ooi.web import Website BIT = BitDefinition( - id="missing-certificate", - consumes=Website, - parameters=[], - module="bits.missing_certificate.missing_certificate", + id="missing-certificate", consumes=Website, parameters=[], module="bits.missing_certificate.missing_certificate" ) diff --git a/octopoes/bits/missing_dmarc/missing_dmarc.py b/octopoes/bits/missing_dmarc/missing_dmarc.py index 865c59b99e9..33686e85568 100644 --- a/octopoes/bits/missing_dmarc/missing_dmarc.py +++ b/octopoes/bits/missing_dmarc/missing_dmarc.py @@ -26,7 +26,5 @@ def run(input_ooi: Hostname, additional_oois: list[DMARCTXTRecord | NXDOMAIN], c ft = KATFindingType(id="KAT-NO-DMARC") yield ft yield Finding( - ooi=input_ooi.reference, - finding_type=ft.reference, - description="This hostname does not have a DMARC record", + ooi=input_ooi.reference, finding_type=ft.reference, description="This hostname does not have a DMARC record" ) diff --git a/octopoes/bits/missing_headers/bit.py b/octopoes/bits/missing_headers/bit.py index 7b72dd3aeeb..97548ed57d8 100644 --- a/octopoes/bits/missing_headers/bit.py +++ b/octopoes/bits/missing_headers/bit.py @@ -4,8 +4,6 @@ BIT = BitDefinition( id="missing-headers", consumes=HTTPResource, - parameters=[ - BitParameterDefinition(ooi_type=HTTPHeader, relation_path="resource"), - ], + parameters=[BitParameterDefinition(ooi_type=HTTPHeader, relation_path="resource")], module="bits.missing_headers.missing_headers", ) diff --git a/octopoes/bits/missing_spf/missing_spf.py b/octopoes/bits/missing_spf/missing_spf.py index d40e368edd4..fe7d78d0a03 100644 --- a/octopoes/bits/missing_spf/missing_spf.py +++ b/octopoes/bits/missing_spf/missing_spf.py @@ -25,7 +25,5 @@ def run(input_ooi: Hostname, additional_oois: list[DNSSPFRecord | NXDOMAIN], con ft = KATFindingType(id="KAT-NO-SPF") yield ft yield Finding( - ooi=input_ooi.reference, - finding_type=ft.reference, - description="This hostname does not have an SPF record", + ooi=input_ooi.reference, finding_type=ft.reference, description="This hostname does not have an SPF record" ) diff --git a/octopoes/bits/nxdomain_flag/bit.py b/octopoes/bits/nxdomain_flag/bit.py index 13464c56483..7c593bb563c 100644 --- a/octopoes/bits/nxdomain_flag/bit.py +++ b/octopoes/bits/nxdomain_flag/bit.py @@ -5,8 +5,6 @@ BIT = BitDefinition( id="nxdomain-flag", consumes=Hostname, - parameters=[ - BitParameterDefinition(ooi_type=NXDOMAIN, relation_path="hostname"), - ], + parameters=[BitParameterDefinition(ooi_type=NXDOMAIN, relation_path="hostname")], module="bits.nxdomain_flag.nxdomain_flag", ) diff --git a/octopoes/bits/nxdomain_flag/nxdomain_flag.py b/octopoes/bits/nxdomain_flag/nxdomain_flag.py index b42685d9210..34401e6ea96 100644 --- a/octopoes/bits/nxdomain_flag/nxdomain_flag.py +++ b/octopoes/bits/nxdomain_flag/nxdomain_flag.py @@ -12,7 +12,5 @@ def run(input_ooi: Hostname, additional_oois: list[NXDOMAIN], config: dict[str, nxdomain = KATFindingType(id="KAT-NXDOMAIN") yield nxdomain yield Finding( - finding_type=nxdomain.reference, - ooi=input_ooi.reference, - description="The domain does not exist.", + finding_type=nxdomain.reference, ooi=input_ooi.reference, description="The domain does not exist." ) diff --git a/octopoes/bits/oois_in_headers/bit.py b/octopoes/bits/oois_in_headers/bit.py index dbdd9b7981e..eac7470aa10 100644 --- a/octopoes/bits/oois_in_headers/bit.py +++ b/octopoes/bits/oois_in_headers/bit.py @@ -2,8 +2,5 @@ from octopoes.models.types import HTTPHeader BIT = BitDefinition( - id="oois-in-headers", - consumes=HTTPHeader, - parameters=[], - module="bits.oois_in_headers.oois_in_headers", + id="oois-in-headers", consumes=HTTPHeader, parameters=[], module="bits.oois_in_headers.oois_in_headers" ) diff --git a/octopoes/bits/port_classification_ip/bit.py b/octopoes/bits/port_classification_ip/bit.py index 6f62eed5b97..7417cc3c4d4 100644 --- a/octopoes/bits/port_classification_ip/bit.py +++ b/octopoes/bits/port_classification_ip/bit.py @@ -4,9 +4,7 @@ BIT = BitDefinition( id="port-classification-ip", consumes=IPAddress, - parameters=[ - BitParameterDefinition(ooi_type=IPPort, relation_path="address"), - ], + parameters=[BitParameterDefinition(ooi_type=IPPort, relation_path="address")], module="bits.port_classification_ip.port_classification_ip", config_ooi_relation_path="IPAddress.network", ) diff --git a/octopoes/bits/port_classification_ip/port_classification_ip.py b/octopoes/bits/port_classification_ip/port_classification_ip.py index c1652a5e7eb..7ac7fbbb4cc 100644 --- a/octopoes/bits/port_classification_ip/port_classification_ip.py +++ b/octopoes/bits/port_classification_ip/port_classification_ip.py @@ -19,7 +19,7 @@ ] COMMON_UDP_PORTS = [ - 53, # DNS + 53 # DNS ] SA_TCP_PORTS = [ @@ -93,9 +93,7 @@ def run(input_ooi: IPPort, additional_oois: list, config: dict[str, Any]) -> Ite description=f"Port {port}/{protocol.value} is not a common port and should possibly not be open.", ) if aggregate_findings and open_ports: - ft = KATFindingType( - id="KAT-UNCOMMON-OPEN-PORT", - ) + ft = KATFindingType(id="KAT-UNCOMMON-OPEN-PORT") yield ft yield Finding( finding_type=ft.reference, diff --git a/octopoes/bits/port_common/bit.py b/octopoes/bits/port_common/bit.py index e319031e1c0..256122c9d34 100644 --- a/octopoes/bits/port_common/bit.py +++ b/octopoes/bits/port_common/bit.py @@ -2,9 +2,5 @@ from octopoes.models.ooi.network import IPPort BIT = BitDefinition( - id="port-common", - consumes=IPPort, - parameters=[], - module="bits.port_common.port_common", - default_enabled=False, + id="port-common", consumes=IPPort, parameters=[], module="bits.port_common.port_common", default_enabled=False ) diff --git a/octopoes/bits/port_common/port_common.py b/octopoes/bits/port_common/port_common.py index 141182dec90..eac9c7d0f7a 100644 --- a/octopoes/bits/port_common/port_common.py +++ b/octopoes/bits/port_common/port_common.py @@ -18,15 +18,11 @@ ] COMMON_UDP_PORTS = [ - 53, # DNS + 53 # DNS ] -def run( - input_ooi: IPPort, - additional_oois: list, - config: dict, -) -> Iterator[OOI]: +def run(input_ooi: IPPort, additional_oois: list, config: dict) -> Iterator[OOI]: port = input_ooi.port protocol = input_ooi.protocol if (protocol == Protocol.TCP and port in COMMON_TCP_PORTS) or ( diff --git a/octopoes/bits/resource_discovery/resource_discovery.py b/octopoes/bits/resource_discovery/resource_discovery.py index 86a496d74c3..6a1da7b4f50 100644 --- a/octopoes/bits/resource_discovery/resource_discovery.py +++ b/octopoes/bits/resource_discovery/resource_discovery.py @@ -20,7 +20,4 @@ def run(hostname: Hostname, additional_oois: list[HostnameHTTPURL | Website], co int(website.ip_service.tokenized.ip_port.port) == hostname_http_url.port and website.ip_service.tokenized.service.name == hostname_http_url.scheme.value ): - yield HTTPResource( - website=website.reference, - web_url=hostname_http_url.reference, - ) + yield HTTPResource(website=website.reference, web_url=hostname_http_url.reference) diff --git a/octopoes/bits/retire_js/bit.py b/octopoes/bits/retire_js/bit.py index 762c4637a16..e7c0adeea20 100644 --- a/octopoes/bits/retire_js/bit.py +++ b/octopoes/bits/retire_js/bit.py @@ -4,8 +4,6 @@ BIT = BitDefinition( id="retire-js", consumes=Software, - parameters=[ - BitParameterDefinition(ooi_type=SoftwareInstance, relation_path="software"), - ], + parameters=[BitParameterDefinition(ooi_type=SoftwareInstance, relation_path="software")], module="bits.retire_js.retire_js", ) diff --git a/octopoes/bits/spf_discovery/bit.py b/octopoes/bits/spf_discovery/bit.py index beaa1510555..04737497de1 100644 --- a/octopoes/bits/spf_discovery/bit.py +++ b/octopoes/bits/spf_discovery/bit.py @@ -1,9 +1,4 @@ from bits.definitions import BitDefinition from octopoes.models.ooi.dns.records import DNSTXTRecord -BIT = BitDefinition( - id="spf-discovery", - consumes=DNSTXTRecord, - parameters=[], - module="bits.spf_discovery.spf_discovery", -) +BIT = BitDefinition(id="spf-discovery", consumes=DNSTXTRecord, parameters=[], module="bits.spf_discovery.spf_discovery") diff --git a/octopoes/bits/ssl_certificate_hostname/ssl_certificate_hostname.py b/octopoes/bits/ssl_certificate_hostname/ssl_certificate_hostname.py index 8fb34ade12f..e7035b6219f 100644 --- a/octopoes/bits/ssl_certificate_hostname/ssl_certificate_hostname.py +++ b/octopoes/bits/ssl_certificate_hostname/ssl_certificate_hostname.py @@ -22,9 +22,7 @@ def hostname_in_qualifiers(hostname: str, qualifiers: list[str]) -> bool: def run( - input_ooi: X509Certificate, - additional_oois: list[Website | SubjectAlternativeNameHostname], - config: dict, + input_ooi: X509Certificate, additional_oois: list[Website | SubjectAlternativeNameHostname], config: dict ) -> Iterator[OOI]: websites = [website for website in additional_oois if isinstance(website, Website)] subject_alternative_name_hostnames = [ diff --git a/octopoes/bits/url_classification/url_classification.py b/octopoes/bits/url_classification/url_classification.py index 74ebc7a0aff..76228e1db9b 100644 --- a/octopoes/bits/url_classification/url_classification.py +++ b/octopoes/bits/url_classification/url_classification.py @@ -19,12 +19,7 @@ def run(url: URL, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI path = url.raw.path if url.raw.path is not None else "/" - default_args = { - "network": url.network, - "scheme": WebScheme(url.raw.scheme), - "port": port, - "path": path, - } + default_args = {"network": url.network, "scheme": WebScheme(url.raw.scheme), "port": port, "path": path} try: addr = ip_address(url.raw.host) except ValueError: diff --git a/octopoes/bits/website_discovery/website_discovery.py b/octopoes/bits/website_discovery/website_discovery.py index 068e3672226..826c1acf559 100644 --- a/octopoes/bits/website_discovery/website_discovery.py +++ b/octopoes/bits/website_discovery/website_discovery.py @@ -21,7 +21,4 @@ def is_service_http(ip_service: IPService) -> bool: # website is cartesian product of hostname and http services for http_service in http_services: for hostname in hostnames: - yield Website( - hostname=hostname, - ip_service=http_service.reference, - ) + yield Website(hostname=hostname, ip_service=http_service.reference) diff --git a/octopoes/octopoes/api/api.py b/octopoes/octopoes/api/api.py index 020f1f64a82..56181c4bbed 100644 --- a/octopoes/octopoes/api/api.py +++ b/octopoes/octopoes/api/api.py @@ -112,11 +112,7 @@ def uncaught_exception_handler(_: Request, exc: Exception) -> None: @app.get("/health") def root_health() -> ServiceHealth: - return ServiceHealth( - service="octopoes", - healthy=True, - version=__version__, - ) + return ServiceHealth(service="octopoes", healthy=True, version=__version__) @app.on_event("shutdown") diff --git a/octopoes/octopoes/api/router.py b/octopoes/octopoes/api/router.py index eb1afbd4e41..347c518189d 100644 --- a/octopoes/octopoes/api/router.py +++ b/octopoes/octopoes/api/router.py @@ -86,30 +86,16 @@ def octopoes_service( # Endpoints @router.get("/health") -def health( - xtdb_session_: XTDBSession = Depends(xtdb_session), -) -> ServiceHealth: +def health(xtdb_session_: XTDBSession = Depends(xtdb_session)) -> ServiceHealth: try: xtdb_status = xtdb_session_.client.status() - xtdb_health = ServiceHealth( - service="xtdb", - healthy=True, - version=xtdb_status.version, - additional=xtdb_status, - ) + xtdb_health = ServiceHealth(service="xtdb", healthy=True, version=xtdb_status.version, additional=xtdb_status) except HTTPError as ex: xtdb_health = ServiceHealth( - service="xtdb", - healthy=False, - additional="Cannot connect to XTDB at. Service possibly down", + service="xtdb", healthy=False, additional="Cannot connect to XTDB at. Service possibly down" ) logger.exception(ex) - return ServiceHealth( - service="octopoes", - healthy=xtdb_health.healthy, - version=__version__, - results=[xtdb_health], - ) + return ServiceHealth(service="octopoes", healthy=xtdb_health.healthy, version=__version__, results=[xtdb_health]) # OOI-related endpoints @@ -270,12 +256,7 @@ def get_tree( reference: Reference = Depends(extract_reference), depth: int = 1, ) -> ReferenceTree: - return octopoes.get_ooi_tree( - reference, - valid_time, - types, - depth, - ) + return octopoes.get_ooi_tree(reference, valid_time, types, depth) @router.get("/origins", tags=["Origins"]) @@ -288,11 +269,7 @@ def list_origins( origin_type: OriginType | None = Query(None), ) -> list[Origin]: return octopoes.origin_repository.list_origins( - valid_time, - task_id=task_id, - source=source, - result=result, - origin_type=origin_type, + valid_time, task_id=task_id, source=source, result=result, origin_type=origin_type ) @@ -306,10 +283,7 @@ def list_origin_parameters( @router.post("/observations", tags=["Origins"]) -def save_observation( - observation: ValidatedObservation, - octopoes: OctopoesService = Depends(octopoes_service), -) -> None: +def save_observation(observation: ValidatedObservation, octopoes: OctopoesService = Depends(octopoes_service)) -> None: origin = Origin( origin_type=OriginType.OBSERVATION, method=observation.method, @@ -322,10 +296,7 @@ def save_observation( @router.post("/declarations", tags=["Origins"]) -def save_declaration( - declaration: ValidatedDeclaration, - octopoes: OctopoesService = Depends(octopoes_service), -) -> None: +def save_declaration(declaration: ValidatedDeclaration, octopoes: OctopoesService = Depends(octopoes_service)) -> None: origin = Origin( origin_type=OriginType.DECLARATION, method=declaration.method if declaration.method else "manual", @@ -338,10 +309,7 @@ def save_declaration( @router.post("/affirmations", tags=["Origins"]) -def save_affirmation( - affirmation: ValidatedAffirmation, - octopoes: OctopoesService = Depends(octopoes_service), -) -> None: +def save_affirmation(affirmation: ValidatedAffirmation, octopoes: OctopoesService = Depends(octopoes_service)) -> None: origin = Origin( origin_type=OriginType.AFFIRMATION, method=affirmation.method if affirmation.method else "hydration", @@ -397,8 +365,7 @@ def save_many( @router.get("/scan_profiles/recalculate", tags=["Scan Profiles"]) def recalculate_scan_profiles( - octopoes: OctopoesService = Depends(octopoes_service), - valid_time: datetime = Depends(extract_valid_time), + octopoes: OctopoesService = Depends(octopoes_service), valid_time: datetime = Depends(extract_valid_time) ) -> None: octopoes.recalculate_scan_profiles(valid_time) octopoes.commit() @@ -432,20 +399,12 @@ def list_findings( valid_time: datetime = Depends(extract_valid_time), severities: set[RiskLevelSeverity] = Query(DEFAULT_SEVERITY_FILTER), ) -> Paginated[Finding]: - return octopoes.ooi_repository.list_findings( - severities, - valid_time, - exclude_muted, - only_muted, - offset, - limit, - ) + return octopoes.ooi_repository.list_findings(severities, valid_time, exclude_muted, only_muted, offset, limit) @router.get("/findings/count_by_severity", tags=["Findings"]) def get_finding_type_count( - octopoes: OctopoesService = Depends(octopoes_service), - valid_time: datetime = Depends(extract_valid_time), + octopoes: OctopoesService = Depends(octopoes_service), valid_time: datetime = Depends(extract_valid_time) ) -> Counter: return octopoes.ooi_repository.count_findings_by_severity(valid_time) @@ -494,25 +453,18 @@ def importer(data: bytes, xtdb_session_: XTDBSession, reset: bool = False) -> di xtdb_session_.commit() except XTDBException as e: raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Error recreating nodes", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error recreating nodes" ) from e for op in ops: try: operations: list[Operation] = [ - ( - OperationType(x[0]), - x[1], - datetime.strptime(x[2], "%Y-%m-%dT%H:%M:%SZ"), - ) - for x in op + (OperationType(x[0]), x[1], datetime.strptime(x[2], "%Y-%m-%dT%H:%M:%SZ")) for x in op ] xtdb_session_.client.submit_transaction(operations) except Exception as e: logger.debug("Error importing objects", exc_info=True) raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error importing object {op}", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Error importing object {op}" ) from e return {"detail": len(ops)} diff --git a/octopoes/octopoes/config/settings.py b/octopoes/octopoes/config/settings.py index c8cd2921ff2..3e86d281df6 100644 --- a/octopoes/octopoes/config/settings.py +++ b/octopoes/octopoes/config/settings.py @@ -19,9 +19,7 @@ class BackwardsCompatibleEnvSettings(EnvSettingsSource): - backwards_compatibility_mapping = { - "LOG_CFG": "OCTOPOES_LOG_CFG", - } + backwards_compatibility_mapping = {"LOG_CFG": "OCTOPOES_LOG_CFG"} def __call__(self) -> dict[str, Any]: d: dict[str, Any] = {} diff --git a/octopoes/octopoes/connector/octopoes.py b/octopoes/octopoes/connector/octopoes.py index ec8ff1393be..997aae7e8af 100644 --- a/octopoes/octopoes/connector/octopoes.py +++ b/octopoes/octopoes/connector/octopoes.py @@ -84,8 +84,7 @@ def list_objects( def get(self, reference: Reference, valid_time: datetime) -> OOI: res = self.session.get( - f"/{self.client}/object", - params={"reference": str(reference), "valid_time": str(valid_time)}, + f"/{self.client}/object", params={"reference": str(reference), "valid_time": str(valid_time)} ) return TypeAdapter(OOIType).validate_json(res.content) @@ -114,11 +113,7 @@ def get_history( return TypeAdapter(list[TransactionRecord]).validate_json(res.content) def get_tree( - self, - reference: Reference, - valid_time: datetime, - types: Set = frozenset(), - depth: int = 1, + self, reference: Reference, valid_time: datetime, types: Set = frozenset(), depth: int = 1 ) -> ReferenceTree: params: dict[str, str | int | list[str]] = { "reference": str(reference), @@ -145,10 +140,7 @@ def list_origins( "origin_type": str(origin_type) if origin_type else None, } params = {k: v for k, v in params.items() if v is not None} # filter out None values - res = self.session.get( - f"/{self.client}/origins", - params=params, - ) + res = self.session.get(f"/{self.client}/origins", params=params) return TypeAdapter(list[Origin]).validate_json(res.content) @@ -240,9 +232,7 @@ def list_findings( return TypeAdapter(Paginated[Finding]).validate_json(res.content) def load_objects_bulk(self, references: set[Reference], valid_time): - params = { - "valid_time": valid_time, - } + params = {"valid_time": valid_time} res = self.session.post( f"/{self.client}/objects/load_bulk", params=params, json=[str(ref) for ref in references] ) @@ -274,19 +264,12 @@ def query( ] def query_many( - self, - path: str, - valid_time: datetime, - sources: Sequence[OOI | Reference | str], + self, path: str, valid_time: datetime, sources: Sequence[OOI | Reference | str] ) -> list[tuple[str, OOIType]]: if not sources: return [] - params = { - "path": path, - "sources": [str(ooi) for ooi in sources], - "valid_time": str(valid_time), - } + params = {"path": path, "sources": [str(ooi) for ooi in sources], "valid_time": str(valid_time)} result = self.session.get(f"/{self.client}/query-many", params=params).json() diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index c0494f36d9d..05762a9305d 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -131,11 +131,7 @@ def list_ooi( return paginated def get_ooi_tree( - self, - reference: Reference, - valid_time: datetime, - search_types: set[type[OOI]] | None = None, - depth: int = 1, + self, reference: Reference, valid_time: datetime, search_types: set[type[OOI]] | None = None, depth: int = 1 ): tree = self.ooi_repository.get_tree(reference, valid_time, search_types, depth) self._populate_scan_profiles(tree.store.values(), valid_time) @@ -356,9 +352,7 @@ def _on_create_ooi(self, event: OOIDBEvent) -> None: self.scan_profile_repository.get(ooi.reference, event.valid_time) except ObjectNotFoundException: self.scan_profile_repository.save( - None, - EmptyScanProfile(reference=ooi.reference), - valid_time=event.valid_time, + None, EmptyScanProfile(reference=ooi.reference), valid_time=event.valid_time ) # analyze bit definitions @@ -366,11 +360,7 @@ def _on_create_ooi(self, event: OOIDBEvent) -> None: for bit_id, bit_definition in bit_definitions.items(): # attach bit instances if isinstance(ooi, bit_definition.consumes): - bit_instance = Origin( - origin_type=OriginType.INFERENCE, - method=bit_id, - source=ooi.reference, - ) + bit_instance = Origin(origin_type=OriginType.INFERENCE, method=bit_id, source=ooi.reference) self.origin_repository.save(bit_instance, event.valid_time) # attach bit parameters @@ -388,14 +378,9 @@ def _on_create_ooi(self, event: OOIDBEvent) -> None: if bit_ancestor: origin = Origin( - origin_type=OriginType.INFERENCE, - method=bit_id, - source=bit_ancestor[0].reference, - ) - origin_parameter = OriginParameter( - origin_id=origin.id, - reference=ooi.reference, + origin_type=OriginType.INFERENCE, method=bit_id, source=bit_ancestor[0].reference ) + origin_parameter = OriginParameter(origin_id=origin.id, reference=ooi.reference) self.origin_parameter_repository.save(origin_parameter, event.valid_time) def _on_update_ooi(self, event: OOIDBEvent) -> None: @@ -591,11 +576,7 @@ def recalculate_bits(self) -> int: logger.exception("Wut?") # insert, if not exists - bit_instance = Origin( - origin_type=OriginType.INFERENCE, - method=bit_id, - source=ooi.reference, - ) + bit_instance = Origin(origin_type=OriginType.INFERENCE, method=bit_id, source=ooi.reference) try: self.origin_repository.get(bit_instance.id, valid_time) except ObjectNotFoundException: diff --git a/octopoes/octopoes/models/ooi/certificate.py b/octopoes/octopoes/models/ooi/certificate.py index 8a642616f6f..b8e35a98695 100644 --- a/octopoes/octopoes/models/ooi/certificate.py +++ b/octopoes/octopoes/models/ooi/certificate.py @@ -37,9 +37,7 @@ class X509Certificate(OOI): def expired(self): return datetime.now() > datetime.fromisoformat(self.valid_until) - _reverse_relation_names = { - "signed_by": "signed_certificates", - } + _reverse_relation_names = {"signed_by": "signed_certificates"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: diff --git a/octopoes/octopoes/models/ooi/dns/records.py b/octopoes/octopoes/models/ooi/dns/records.py index dc510f46000..8d63cf70ed4 100644 --- a/octopoes/octopoes/models/ooi/dns/records.py +++ b/octopoes/octopoes/models/ooi/dns/records.py @@ -15,9 +15,7 @@ class DNSRecord(OOI): ttl: int | None = None # todo: validation _natural_key_attrs = ["hostname", "value"] - _reverse_relation_names = { - "hostname": "dns_records", - } + _reverse_relation_names = {"hostname": "dns_records"} @classmethod def _get_record_type(cls) -> str: @@ -45,10 +43,7 @@ class DNSAAAARecord(DNSRecord): address: Reference = ReferenceField(IPAddressV6) - _reverse_relation_names = { - "hostname": "dns_aaaa_records", - "address": "dns_aaaa_records", - } + _reverse_relation_names = {"hostname": "dns_aaaa_records", "address": "dns_aaaa_records"} class DNSMXRecord(DNSRecord): @@ -58,10 +53,7 @@ class DNSMXRecord(DNSRecord): mail_hostname: Reference | None = ReferenceField(Hostname, default=None) preference: int | None = None - _reverse_relation_names = { - "hostname": "dns_mx_records", - "mail_hostname": "mail_server_of", - } + _reverse_relation_names = {"hostname": "dns_mx_records", "mail_hostname": "mail_server_of"} class DNSTXTRecord(DNSRecord): @@ -83,10 +75,7 @@ class DNSNSRecord(DNSRecord): name_server_hostname: Reference = ReferenceField(Hostname, max_issue_scan_level=1, max_inherit_scan_level=0) - _reverse_relation_names = { - "hostname": "dns_ns_records", - "name_server_hostname": "ns_record_targets", - } + _reverse_relation_names = {"hostname": "dns_ns_records", "name_server_hostname": "ns_record_targets"} class DNSCNAMERecord(DNSRecord): @@ -95,10 +84,7 @@ class DNSCNAMERecord(DNSRecord): target_hostname: Reference = ReferenceField(Hostname) - _reverse_relation_names = { - "hostname": "dns_cname_records", - "target_hostname": "cname_target_of", - } + _reverse_relation_names = {"hostname": "dns_cname_records", "target_hostname": "cname_target_of"} class DNSSOARecord(DNSRecord): @@ -112,10 +98,7 @@ class DNSSOARecord(DNSRecord): expire: int | None = None minimum: int | None = None - _reverse_relation_names = { - "hostname": "dns_soa_records", - "soa_hostname": "subject_of_dns_soa_records", - } + _reverse_relation_names = {"hostname": "dns_soa_records", "soa_hostname": "subject_of_dns_soa_records"} _natural_key_attrs = ["hostname", "soa_hostname"] @@ -131,9 +114,7 @@ class NXDOMAIN(OOI): hostname: Reference = ReferenceField(Hostname) _natural_key_attrs = ["hostname"] - _reverse_relation_names = { - "hostname": "nxdomain_hostname", - } + _reverse_relation_names = {"hostname": "nxdomain_hostname"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: @@ -147,10 +128,7 @@ class DNSPTRRecord(DNSRecord): _natural_key_attrs = ["hostname", "address"] - _reverse_relation_names = { - "hostname": "dns_ptr_records", - "address": "ptr_record_ip", - } + _reverse_relation_names = {"hostname": "dns_ptr_records", "address": "ptr_record_ip"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: diff --git a/octopoes/octopoes/models/ooi/dns/zone.py b/octopoes/octopoes/models/ooi/dns/zone.py index 1b4829467d3..196724ed31a 100644 --- a/octopoes/octopoes/models/ooi/dns/zone.py +++ b/octopoes/octopoes/models/ooi/dns/zone.py @@ -41,11 +41,7 @@ class Hostname(OOI): _natural_key_attrs = ["network", "name"] - _reverse_relation_names = { - "network": "hostnames", - "dns_zone": "hostnames", - "registered_domain": "subdomains", - } + _reverse_relation_names = {"network": "hostnames", "dns_zone": "hostnames", "registered_domain": "subdomains"} @field_validator("name") @classmethod diff --git a/octopoes/octopoes/models/ooi/email_security.py b/octopoes/octopoes/models/ooi/email_security.py index 3ffce82dfbf..e21615288fa 100644 --- a/octopoes/octopoes/models/ooi/email_security.py +++ b/octopoes/octopoes/models/ooi/email_security.py @@ -17,9 +17,7 @@ class DNSSPFRecord(OOI): dns_txt_record: Reference = ReferenceField(DNSTXTRecord, max_inherit_scan_level=1) _natural_key_attrs = ["dns_txt_record", "value"] - _reverse_relation_names = { - "dns_txt_record": "dns_spf_record", - } + _reverse_relation_names = {"dns_txt_record": "dns_spf_record"} @property def natural_key(self) -> str: @@ -44,9 +42,7 @@ class DNSSPFMechanismIP(DNSSPFMechanism): _natural_key_attrs = ["spf_record", "mechanism", "ip"] _information_value = ["mechanism"] - _reverse_relation_names = { - "spf_record": "spf_ip_mechanisms", - } + _reverse_relation_names = {"spf_record": "spf_ip_mechanisms"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: @@ -63,9 +59,7 @@ class DNSSPFMechanismHostname(DNSSPFMechanism): _natural_key_attrs = ["spf_record", "mechanism", "hostname"] _information_value = ["mechanism"] - _reverse_relation_names = { - "spf_record": "spf_hostname_mechanisms", - } + _reverse_relation_names = {"spf_record": "spf_hostname_mechanisms"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: @@ -82,9 +76,7 @@ class DNSSPFMechanismNetBlock(DNSSPFMechanism): _natural_key_attrs = ["spf_record", "mechanism", "netblock"] _information_value = ["mechanism"] - _reverse_relation_names = { - "spf_record": "spf_netblock_mechanisms", - } + _reverse_relation_names = {"spf_record": "spf_netblock_mechanisms"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: @@ -102,9 +94,7 @@ class DMARCTXTRecord(OOI): hostname: Reference = ReferenceField(Hostname) _natural_key_attrs = ["value", "hostname"] - _reverse_relation_names = { - "hostname": "dmarc_txt_record", - } + _reverse_relation_names = {"hostname": "dmarc_txt_record"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: @@ -116,9 +106,7 @@ class DKIMExists(OOI): hostname: Reference = ReferenceField(Hostname) _natural_key_attrs = ["hostname"] - _reverse_relation_names = { - "hostname": "dkim_exists", - } + _reverse_relation_names = {"hostname": "dkim_exists"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: @@ -131,9 +119,7 @@ class DKIMSelector(OOI): hostname: Reference = ReferenceField(Hostname) _natural_key_attrs = ["selector", "hostname"] - _reverse_relation_names = { - "hostname": "dkim_selector", - } + _reverse_relation_names = {"hostname": "dkim_selector"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: @@ -146,9 +132,7 @@ class DKIMKey(OOI): dkim_selector: Reference = ReferenceField(DKIMSelector) _natural_key_attrs = ["key", "dkim_selector"] - _reverse_relation_names = { - "dkim_selector": "dkim_key", - } + _reverse_relation_names = {"dkim_selector": "dkim_key"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: diff --git a/octopoes/octopoes/models/ooi/network.py b/octopoes/octopoes/models/ooi/network.py index c36a8da02ae..83094b15dc0 100644 --- a/octopoes/octopoes/models/ooi/network.py +++ b/octopoes/octopoes/models/ooi/network.py @@ -43,10 +43,7 @@ class IPAddressV4(IPAddress): "IPV4NetBlock", optional=True, max_issue_scan_level=0, max_inherit_scan_level=4, default=None ) - _reverse_relation_names = { - "network": "ip_v4_addresses", - "netblock": "ip_v4_addresses", - } + _reverse_relation_names = {"network": "ip_v4_addresses", "netblock": "ip_v4_addresses"} class IPAddressV6(IPAddress): @@ -57,10 +54,7 @@ class IPAddressV6(IPAddress): "IPV6NetBlock", optional=True, max_issue_scan_level=0, max_inherit_scan_level=4, default=None ) - _reverse_relation_names = { - "network": "ip_v6_addresses", - "netblock": "ip_v6_addresses", - } + _reverse_relation_names = {"network": "ip_v6_addresses", "netblock": "ip_v6_addresses"} class Protocol(Enum): @@ -114,10 +108,7 @@ class NetBlock(OOI): _natural_key_attrs = ["network", "start_ip", "mask"] - _reverse_relation_names = { - "announced_by": "announced_netblocks", - "parent": "child_netblocks", - } + _reverse_relation_names = {"announced_by": "announced_netblocks", "parent": "child_netblocks"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: @@ -132,10 +123,7 @@ class IPV6NetBlock(NetBlock): start_ip: Reference = ReferenceField(IPAddressV6, max_issue_scan_level=4) mask: Annotated[int, Field(ge=0, lt=128)] - _reverse_relation_names = { - "parent": "child_netblocks", - "announced_by": "announced_ipv6_netblocks", - } + _reverse_relation_names = {"parent": "child_netblocks", "announced_by": "announced_ipv6_netblocks"} class IPV4NetBlock(NetBlock): @@ -146,10 +134,7 @@ class IPV4NetBlock(NetBlock): start_ip: Reference = ReferenceField(IPAddressV4, max_issue_scan_level=4) mask: Annotated[int, Field(ge=0, lt=32)] - _reverse_relation_names = { - "parent": "child_netblocks", - "announced_by": "announced_ipv4_netblocks", - } + _reverse_relation_names = {"parent": "child_netblocks", "announced_by": "announced_ipv4_netblocks"} IPAddressV4.model_rebuild() diff --git a/octopoes/octopoes/models/ooi/web.py b/octopoes/octopoes/models/ooi/web.py index 4e8f65101ba..eac3acd0ea7 100644 --- a/octopoes/octopoes/models/ooi/web.py +++ b/octopoes/octopoes/models/ooi/web.py @@ -93,10 +93,7 @@ class HTTPResource(OOI): _natural_key_attrs = ["website", "web_url"] - _reverse_relation_names = { - "website": "resources", - "web_url": "resources", - } + _reverse_relation_names = {"website": "resources", "web_url": "resources"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: @@ -244,9 +241,7 @@ class RESTAPI(OOI): api_url: Reference = ReferenceField(WebURL) _natural_key_attrs = ["api_url"] - _reverse_relation_names = { - "api_url": "api_url_of", - } + _reverse_relation_names = {"api_url": "api_url_of"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: @@ -276,10 +271,7 @@ class APIDesignRuleResult(OOI): message: str _natural_key_attrs = ["rest_api", "rule"] - _reverse_relation_names = { - "rest_api": "api_design_rule_results", - "rule": "results", - } + _reverse_relation_names = {"rest_api": "api_design_rule_results", "rule": "results"} @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: diff --git a/octopoes/octopoes/models/path.py b/octopoes/octopoes/models/path.py index 545e4c32185..399a9945804 100644 --- a/octopoes/octopoes/models/path.py +++ b/octopoes/octopoes/models/path.py @@ -19,13 +19,7 @@ class Direction(Enum): class Segment: - def __init__( - self, - source_type: type[OOI], - direction: Direction, - property_name: str, - target_type: type[OOI] | None, - ): + def __init__(self, source_type: type[OOI], direction: Direction, property_name: str, target_type: type[OOI] | None): self.source_type = source_type self.direction = direction self.property_name = property_name diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index f0a15c190ba..eb34d3e26ae 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -109,11 +109,7 @@ def delete(self, reference: Reference, valid_time: datetime) -> None: raise NotImplementedError def get_tree( - self, - reference: Reference, - valid_time: datetime, - search_types: set[type[OOI]] | None = None, - depth: int = 1, + self, reference: Reference, valid_time: datetime, search_types: set[type[OOI]] | None = None, depth: int = 1 ) -> ReferenceTree: raise NotImplementedError @@ -123,15 +119,7 @@ def list_oois_without_scan_profile(self, valid_time: datetime) -> set[Reference] def count_findings_by_severity(self, valid_time: datetime) -> Counter: raise NotImplementedError - def list_findings( - self, - severities, - valid_time, - exclude_muted, - only_muted, - offset, - limit, - ) -> Paginated[Finding]: + def list_findings(self, severities, valid_time, exclude_muted, only_muted, offset, limit) -> Paginated[Finding]: raise NotImplementedError def get_bit_configs(self, source: OOI, bit_definition: BitDefinition, valid_time: datetime) -> list[Config]: @@ -324,10 +312,7 @@ def list_oois( res = self.session.client.query(data_query, valid_time) oois = [self.deserialize(x[0]) for x in res] - return Paginated( - count=count, - items=oois, - ) + return Paginated(count=count, items=oois) def list_random( self, valid_time: datetime, amount: int = 1, scan_levels: set[ScanLevel] = DEFAULT_SCAN_LEVEL_FILTER @@ -347,10 +332,7 @@ def list_random( }} :in-args [[{scan_levels}]] }} - """.format( - amount=amount, - scan_levels=" ".join([str(scan_level.value) for scan_level in scan_levels]), - ) + """.format(amount=amount, scan_levels=" ".join([str(scan_level.value) for scan_level in scan_levels])) res = self.session.client.query(query, valid_time) if not res: @@ -359,11 +341,7 @@ def list_random( return list(self.load_bulk(references, valid_time).values()) def get_tree( - self, - reference: Reference, - valid_time: datetime, - search_types: set[type[OOI]] | None = None, - depth: int = 1, + self, reference: Reference, valid_time: datetime, search_types: set[type[OOI]] | None = None, depth: int = 1 ) -> ReferenceTree: if search_types is None: search_types = {OOI} @@ -387,10 +365,7 @@ def _get_related_objects(self, references: set[Reference], valid_time: datetime """ ooi_classes = {ooi.class_ for ooi in references} ooi_ids = [str(reference) for reference in references] - field_node = RelatedFieldNode( - data_model=datamodel, - object_types=ooi_classes, - ) + field_node = RelatedFieldNode(data_model=datamodel, object_types=ooi_classes) field_node.build_tree(1) query = generate_pull_query(FieldSet.ONLY_ID, {self.pk_prefix: ooi_ids}, field_node=field_node) res = self.session.client.query(query, valid_time=valid_time) @@ -587,10 +562,7 @@ def delete(self, reference: Reference, valid_time: datetime) -> None: self.session.add((XTDBOperationType.DELETE, str(reference), valid_time)) event = OOIDBEvent( - operation_type=OperationType.DELETE, - valid_time=valid_time, - old_data=ooi, - client=self.event_manager.client, + operation_type=OperationType.DELETE, valid_time=valid_time, old_data=ooi, client=self.event_manager.client ) self.session.listen_post_commit(lambda: self.event_manager.publish(event)) @@ -723,10 +695,7 @@ def list_findings( }} """ - return Paginated( - count=count, - items=[x[0] for x in self.query(finding_query, valid_time)], - ) + return Paginated(count=count, items=[x[0] for x in self.query(finding_query, valid_time)]) def query(self, query: str | Query, valid_time: datetime) -> list[OOI | tuple]: results = self.session.client.query(query, valid_time=valid_time) diff --git a/octopoes/octopoes/repositories/origin_parameter_repository.py b/octopoes/octopoes/repositories/origin_parameter_repository.py index 76c601cb09b..54347af3949 100644 --- a/octopoes/octopoes/repositories/origin_parameter_repository.py +++ b/octopoes/octopoes/repositories/origin_parameter_repository.py @@ -70,23 +70,13 @@ def get(self, origin_parameter_id: str, valid_time: datetime) -> OriginParameter raise e def list_by_origin(self, origin_id: set[str], valid_time: datetime) -> list[OriginParameter]: - query = generate_pull_query( - FieldSet.ALL_FIELDS, - { - "origin_id": origin_id, - "type": OriginParameter.__name__, - }, - ) + query = generate_pull_query(FieldSet.ALL_FIELDS, {"origin_id": origin_id, "type": OriginParameter.__name__}) results = self.session.client.query(query, valid_time=valid_time) return [self.deserialize(r[0]) for r in results] def list_by_reference(self, reference: Reference, valid_time: datetime): query = generate_pull_query( - FieldSet.ALL_FIELDS, - { - "reference": str(reference), - "type": OriginParameter.__name__, - }, + FieldSet.ALL_FIELDS, {"reference": str(reference), "type": OriginParameter.__name__} ) results = self.session.client.query(query, valid_time=valid_time) return [self.deserialize(r[0]) for r in results] diff --git a/octopoes/octopoes/repositories/origin_repository.py b/octopoes/octopoes/repositories/origin_repository.py index c21dc4854cc..dd143772a1a 100644 --- a/octopoes/octopoes/repositories/origin_repository.py +++ b/octopoes/octopoes/repositories/origin_repository.py @@ -89,10 +89,7 @@ def list_origins( if origin_type: where_parameters["origin_type"] = origin_type.value - query = generate_pull_query( - FieldSet.ALL_FIELDS, - where_parameters, - ) + query = generate_pull_query(FieldSet.ALL_FIELDS, where_parameters) results = self.session.client.query(query, valid_time=valid_time) return [self.deserialize(r[0]) for r in results] diff --git a/octopoes/octopoes/repositories/scan_profile_repository.py b/octopoes/octopoes/repositories/scan_profile_repository.py index 939db4338cf..a20d3c3cf1d 100644 --- a/octopoes/octopoes/repositories/scan_profile_repository.py +++ b/octopoes/octopoes/repositories/scan_profile_repository.py @@ -71,10 +71,7 @@ def list_scan_profiles(self, scan_profile_type: str | None, valid_time: datetime where = {"type": self.object_type} if scan_profile_type is not None: where["scan_profile_type"] = scan_profile_type - query = generate_pull_query( - FieldSet.ALL_FIELDS, - where, - ) + query = generate_pull_query(FieldSet.ALL_FIELDS, where) results = self.session.client.query(query, valid_time=valid_time) return [self.deserialize(r[0]) for r in results] diff --git a/octopoes/octopoes/tasks/app.py b/octopoes/octopoes/tasks/app.py index 39335720d3a..721a24e7a8a 100644 --- a/octopoes/octopoes/tasks/app.py +++ b/octopoes/octopoes/tasks/app.py @@ -13,5 +13,5 @@ "task": "octopoes.tasks.tasks.schedule_scan_profile_recalculations", "schedule": settings.scan_level_recalculation_interval, "args": tuple(), - }, + } } diff --git a/octopoes/octopoes/xtdb/query.py b/octopoes/octopoes/xtdb/query.py index 3f263046d72..0cba91014e1 100644 --- a/octopoes/octopoes/xtdb/query.py +++ b/octopoes/octopoes/xtdb/query.py @@ -217,13 +217,7 @@ def _where_field_is(self, ref: Ref, field_name: str, value: Ref | str | set[str] raise InvalidField(f"value '{value}' for abstract class fields should be a string or an OOI Type") if isinstance(value, Aliased) or issubclass(value, OOI): - self._add_or_statement_for_abstract_types( - ref, - field_name, - self._get_object_alias( - value, - ), - ) + self._add_or_statement_for_abstract_types(ref, field_name, self._get_object_alias(value)) return if isinstance(value, str): @@ -266,12 +260,7 @@ def _add_where_statement(self, ref: Ref, field_name: str, to_alias: str) -> None self._where_clauses.append(self._assert_type(ref, ooi_type)) self._where_clauses.append( - self._relationship( - self._get_object_alias(ref), - ooi_type.get_object_type(), - field_name, - to_alias, - ) + self._relationship(self._get_object_alias(ref), ooi_type.get_object_type(), field_name, to_alias) ) def _add_or_statement_for_abstract_types(self, ref: Ref, field_name: str, to_alias: str) -> None: @@ -280,10 +269,7 @@ def _add_or_statement_for_abstract_types(self, ref: Ref, field_name: str, to_ali self._where_clauses.append(self._assert_type(ref, ooi_type)) self._where_clauses.append( self._or_statement_for_abstract_types( - self._get_object_alias(ref), - ooi_type.strict_subclasses(), - field_name, - to_alias, + self._get_object_alias(ref), ooi_type.strict_subclasses(), field_name, to_alias ) ) diff --git a/octopoes/octopoes/xtdb/related_field_generator.py b/octopoes/octopoes/xtdb/related_field_generator.py index ac48a8f0e88..1b44112338c 100644 --- a/octopoes/octopoes/xtdb/related_field_generator.py +++ b/octopoes/octopoes/xtdb/related_field_generator.py @@ -2,12 +2,7 @@ class RelatedFieldNode: - def __init__( - self, - data_model: Datamodel, - object_types: set[str], - path: tuple[ForeignKey, ...] = (), - ): + def __init__(self, data_model: Datamodel, object_types: set[str], path: tuple[ForeignKey, ...] = ()): self.data_model = data_model self.object_types = object_types @@ -35,9 +30,7 @@ def construct_outgoing_relations(self): # Don't traverse the same relation back if not self.path or foreign_key != self.path[-1]: self.relations_out[(object_type, foreign_key.attr_name)] = RelatedFieldNode( - self.data_model, - foreign_key.related_entities, - self.path + (foreign_key,), + self.data_model, foreign_key.related_entities, self.path + (foreign_key,) ) def construct_incoming_relations(self): @@ -46,10 +39,7 @@ def construct_incoming_relations(self): types = types - {"Network"} # Loop all object types - for ( - foreign_object_type, - foreign_object_relations, - ) in self.data_model.entities.items(): + for foreign_object_type, foreign_object_relations in self.data_model.entities.items(): # Loop all attributes for foreign_key in foreign_object_relations: # Other object points to one of the types in this QueryNode (i.e. sets are NOT disjoint) @@ -57,16 +47,8 @@ def construct_incoming_relations(self): if not foreign_key.related_entities.isdisjoint(types) and ( not self.path or foreign_key != self.path[-1] ): - self.relations_in[ - ( - foreign_key.source_entity, - foreign_key.attr_name, - foreign_key.reverse_name, - ) - ] = RelatedFieldNode( - self.data_model, - {foreign_object_type}, - self.path + (foreign_key,), + self.relations_in[(foreign_key.source_entity, foreign_key.attr_name, foreign_key.reverse_name)] = ( + RelatedFieldNode(self.data_model, {foreign_object_type}, self.path + (foreign_key,)) ) def build_tree(self, depth: int): diff --git a/octopoes/tests/conftest.py b/octopoes/tests/conftest.py index a6b6c2558a2..3c66c11b5d0 100644 --- a/octopoes/tests/conftest.py +++ b/octopoes/tests/conftest.py @@ -142,10 +142,7 @@ def dns_zone(network, ooi_repository, hostname, scan_profile_repository, valid_t @pytest.fixture def hostname(network, ooi_repository, scan_profile_repository, valid_time): return add_ooi( - Hostname(name="example.com", network=network.reference), - ooi_repository, - scan_profile_repository, - valid_time, + Hostname(name="example.com", network=network.reference), ooi_repository, scan_profile_repository, valid_time ) diff --git a/octopoes/tests/integration/test_api_connector.py b/octopoes/tests/integration/test_api_connector.py index e73eef80880..b49cc3f4268 100644 --- a/octopoes/tests/integration/test_api_connector.py +++ b/octopoes/tests/integration/test_api_connector.py @@ -21,22 +21,13 @@ def test_bulk_operations(octopoes_api_connector: OctopoesAPIConnector, valid_time: datetime): network = Network(name="test") - octopoes_api_connector.save_declaration( - Declaration( - ooi=network, - valid_time=valid_time, - ) - ) + octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=valid_time)) hostnames = [Hostname(network=network.reference, name=f"test{i}") for i in range(10)] task_id = uuid.uuid4() octopoes_api_connector.save_observation( Observation( - method="normalizer_id", - source=network.reference, - task_id=task_id, - valid_time=valid_time, - result=hostnames, + method="normalizer_id", source=network.reference, task_id=task_id, valid_time=valid_time, result=hostnames ) ) @@ -71,20 +62,10 @@ def test_bulk_operations(octopoes_api_connector: OctopoesAPIConnector, valid_tim def test_history(octopoes_api_connector: OctopoesAPIConnector): network = Network(name="test") first_seen = datetime(year=2020, month=10, day=10, tzinfo=timezone.utc) # XTDB only returns a precision of seconds - octopoes_api_connector.save_declaration( - Declaration( - ooi=network, - valid_time=first_seen, - ) - ) + octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=first_seen)) octopoes_api_connector.delete(network.reference, datetime(year=2020, month=10, day=11, tzinfo=timezone.utc)) last_seen = datetime(year=2020, month=10, day=12, tzinfo=timezone.utc) - octopoes_api_connector.save_declaration( - Declaration( - ooi=network, - valid_time=last_seen, - ) - ) + octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=last_seen)) history = octopoes_api_connector.get_history(network.reference, with_docs=True) assert len(history) == 3 @@ -109,12 +90,7 @@ def test_history(octopoes_api_connector: OctopoesAPIConnector): def test_query(octopoes_api_connector: OctopoesAPIConnector, valid_time: datetime): network = Network(name="test") - octopoes_api_connector.save_declaration( - Declaration( - ooi=network, - valid_time=valid_time, - ) - ) + octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=valid_time)) hostnames: list[OOI] = [Hostname(network=network.reference, name=f"test{i}") for i in range(10)] @@ -160,13 +136,7 @@ def test_query(octopoes_api_connector: OctopoesAPIConnector, valid_time: datetim ) octopoes_api_connector.save_many_scan_profiles( - [ - DeclaredScanProfile( - reference=ooi.reference, - level=ScanLevel.L2, - ) - for ooi in all_new_oois + [network] - ], + [DeclaredScanProfile(reference=ooi.reference, level=ScanLevel.L2) for ooi in all_new_oois + [network]], valid_time, ) @@ -214,9 +184,7 @@ def test_query(octopoes_api_connector: OctopoesAPIConnector, valid_time: datetim assert len(octopoes_api_connector.query(query, valid_time, hostnames[3])) == 0 result = octopoes_api_connector.query_many( - query, - valid_time, - [hostnames[0], hostnames[1], hostnames[2], hostnames[3]], + query, valid_time, [hostnames[0], hostnames[1], hostnames[2], hostnames[3]] ) assert len(result) == 3 assert result[0][0] == hostnames[0].reference diff --git a/octopoes/tests/integration/test_io.py b/octopoes/tests/integration/test_io.py index 1e998a10fcb..0491b0c3b20 100644 --- a/octopoes/tests/integration/test_io.py +++ b/octopoes/tests/integration/test_io.py @@ -16,12 +16,7 @@ def test_io(octopoes_api_connector: OctopoesAPIConnector, valid_time: datetime): network = Network(name="internet") - octopoes_api_connector.save_declaration( - Declaration( - ooi=network, - valid_time=valid_time, - ) - ) + octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=valid_time)) time.sleep(2) assert octopoes_api_connector.list_objects(types={Network}, valid_time=valid_time).count == 1 diff --git a/octopoes/tests/integration/test_ooi_deletion.py b/octopoes/tests/integration/test_ooi_deletion.py index ad52fbe1b98..01bab02bcdc 100644 --- a/octopoes/tests/integration/test_ooi_deletion.py +++ b/octopoes/tests/integration/test_ooi_deletion.py @@ -47,11 +47,7 @@ def test_hostname_nxd_ooi(octopoes_api_connector: OctopoesAPIConnector, valid_ti nxd = NXDOMAIN(hostname=hostname.reference) octopoes_api_connector.save_observation( Observation( - method="normalizer_id", - source=hostname.reference, - task_id=uuid.uuid4(), - valid_time=valid_time, - result=[nxd], + method="normalizer_id", source=hostname.reference, task_id=uuid.uuid4(), valid_time=valid_time, result=[nxd] ) ) octopoes_api_connector.recalculate_bits() diff --git a/octopoes/tests/integration/test_unicode.py b/octopoes/tests/integration/test_unicode.py index b57bbc83895..4ddd9cb4e46 100644 --- a/octopoes/tests/integration/test_unicode.py +++ b/octopoes/tests/integration/test_unicode.py @@ -20,12 +20,7 @@ def test_unicode_network(octopoes_api_connector: OctopoesAPIConnector, valid_time: datetime): network = Network(name=NAMES[0]) - octopoes_api_connector.save_declaration( - Declaration( - ooi=network, - valid_time=valid_time, - ) - ) + octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=valid_time)) time.sleep(1) @@ -37,12 +32,7 @@ def test_unicode_network(octopoes_api_connector: OctopoesAPIConnector, valid_tim def test_unicode_hostname(octopoes_api_connector: OctopoesAPIConnector, valid_time: datetime): network = Network(name=NAMES[0]) - octopoes_api_connector.save_declaration( - Declaration( - ooi=network, - valid_time=valid_time, - ) - ) + octopoes_api_connector.save_declaration(Declaration(ooi=network, valid_time=valid_time)) with pytest.raises(ValueError): Hostname(network=network.reference, name="%@.com") @@ -52,11 +42,7 @@ def test_unicode_hostname(octopoes_api_connector: OctopoesAPIConnector, valid_ti octopoes_api_connector.save_observation( Observation( - method=NAMES[2], - source=network.reference, - task_id=task_id, - valid_time=valid_time, - result=[hostname], + method=NAMES[2], source=network.reference, task_id=task_id, valid_time=valid_time, result=[hostname] ) ) diff --git a/octopoes/tests/integration/test_xtdb_client.py b/octopoes/tests/integration/test_xtdb_client.py index 0c747cacfbc..4c2c7452526 100644 --- a/octopoes/tests/integration/test_xtdb_client.py +++ b/octopoes/tests/integration/test_xtdb_client.py @@ -282,18 +282,14 @@ def test_query_for_system_report( "hostnames": [ str(x.name) for x in octopoes_api_connector.query( - "IPAddress. str: return f"{self.ooi}|{self.label_id}" - _reverse_relation_names = { - "ooi": "labels", - } + _reverse_relation_names = {"ooi": "labels"} ALL_OOI_TYPES = { diff --git a/octopoes/tests/test_api.py b/octopoes/tests/test_api.py index 8e78b19f071..5d9c6e8a5f0 100644 --- a/octopoes/tests/test_api.py +++ b/octopoes/tests/test_api.py @@ -37,13 +37,7 @@ def test_health(httpx_mock, patch_pika): "version": __version__, "additional": None, "results": [ - { - "healthy": True, - "service": "xtdb", - "version": "1.24.1", - "additional": xtdb_status, - "results": [], - }, + {"healthy": True, "service": "xtdb", "version": "1.24.1", "additional": xtdb_status, "results": []} ], } assert response.status_code == 200 @@ -66,7 +60,7 @@ def test_health_no_xtdb_connection(httpx_mock, patch_pika): "version": None, "additional": "Cannot connect to XTDB at. Service possibly down", "results": [], - }, + } ], } assert response.status_code == 200 @@ -131,11 +125,7 @@ def test_count_findings_by_severity(httpx_mock, patch_pika, caplog, valid_time): }, 1, ], - [ - "KATFindingType|KAT-NO-FINDING-TYPE", - None, - 2, - ], + ["KATFindingType|KAT-NO-FINDING-TYPE", None, 2], ] httpx_mock.add_response( diff --git a/octopoes/tests/test_bit_missing_headers.py b/octopoes/tests/test_bit_missing_headers.py index e09bc1ee3df..f2ec452ea61 100644 --- a/octopoes/tests/test_bit_missing_headers.py +++ b/octopoes/tests/test_bit_missing_headers.py @@ -20,9 +20,7 @@ def test_https_hsts(http_resource_https): def test_https_no_hsts(http_resource_https): - headers = [ - HTTPHeader(resource=http_resource_https.reference, key="Content-Type", value="text/html"), - ] + headers = [HTTPHeader(resource=http_resource_https.reference, key="Content-Type", value="text/html")] results = list(run(http_resource_https, headers, {})) hsts_findings = [r for r in results if r.object_type == "Finding" and r.finding_type.natural_key == "KAT-NO-HSTS"] @@ -31,9 +29,7 @@ def test_https_no_hsts(http_resource_https): def test_http_no_hsts(http_resource_http): - headers = [ - HTTPHeader(resource=http_resource_http.reference, key="Content-Type", value="text/html"), - ] + headers = [HTTPHeader(resource=http_resource_http.reference, key="Content-Type", value="text/html")] results = list(run(http_resource_http, headers, {})) hsts_findings = [r for r in results if r.object_type == "Finding" and r.finding_type.natural_key == "KAT-NO-HSTS"] diff --git a/octopoes/tests/test_bit_spf_discovery.py b/octopoes/tests/test_bit_spf_discovery.py index 0840e485803..0219c8cc339 100644 --- a/octopoes/tests/test_bit_spf_discovery.py +++ b/octopoes/tests/test_bit_spf_discovery.py @@ -37,8 +37,7 @@ def test_spf_discovery_simple_success(): def test_spf_discovery_invalid_(): dnstxt_record = DNSTXTRecord( - hostname=Reference.from_str("Hostname|internet|example.com"), - value="v=spf1 assdfsdf w rgw", + hostname=Reference.from_str("Hostname|internet|example.com"), value="v=spf1 assdfsdf w rgw" ) results = list(run(dnstxt_record, [], {})) diff --git a/octopoes/tests/test_octopoes_service.py b/octopoes/tests/test_octopoes_service.py index 5eac373b197..cad81d9c6a9 100644 --- a/octopoes/tests/test_octopoes_service.py +++ b/octopoes/tests/test_octopoes_service.py @@ -27,22 +27,13 @@ def test_process_ooi_create_event(octopoes_service, valid_time): ooi = Hostname(network=Network(name="internet").reference, name="example.com") octopoes_service.process_event( OOIDBEvent( - operation_type=OperationType.CREATE, - valid_time=valid_time, - client="_dev", - old_data=None, - new_data=ooi, + operation_type=OperationType.CREATE, valid_time=valid_time, client="_dev", old_data=None, new_data=ooi ) ) # octopoes should create a new origin, because there is a matching bit definition octopoes_service.origin_repository.save.assert_called_once_with( - Origin( - origin_type=OriginType.INFERENCE, - method="fake-hostname-bit", - source=ooi.reference, - ), - valid_time, + Origin(origin_type=OriginType.INFERENCE, method="fake-hostname-bit", source=ooi.reference), valid_time ) @@ -52,22 +43,13 @@ def test_process_event_abstract_bit_consumes(octopoes_service, valid_time): ooi = IPAddressV4(network=Network(name="internet").reference, address=ip_address("1.1.1.1")) octopoes_service.process_event( OOIDBEvent( - operation_type=OperationType.CREATE, - valid_time=valid_time, - client="_dev", - old_data=None, - new_data=ooi, + operation_type=OperationType.CREATE, valid_time=valid_time, client="_dev", old_data=None, new_data=ooi ) ) # octopoes should create a new origin, because there is a matching bit definition (w/ abstract class) octopoes_service.origin_repository.save.assert_called_once_with( - Origin( - origin_type=OriginType.INFERENCE, - method="fake-ipaddress-bit", - source=ooi.reference, - ), - valid_time, + Origin(origin_type=OriginType.INFERENCE, method="fake-ipaddress-bit", source=ooi.reference), valid_time ) @@ -85,11 +67,7 @@ def test_on_update_origin(octopoes_service, valid_time): source=Reference.from_str("Hostname|internet|example.com"), ) event = OriginDBEvent( - operation_type=OperationType.UPDATE, - valid_time=valid_time, - client="_dev", - old_data=old_data, - new_data=new_data, + operation_type=OperationType.UPDATE, valid_time=valid_time, client="_dev", old_data=old_data, new_data=new_data ) # and the deferenced ooi is no longer referred to by any origins diff --git a/octopoes/tests/test_ooi_repository.py b/octopoes/tests/test_ooi_repository.py index e62ccdafd51..b9bec6d4d5e 100644 --- a/octopoes/tests/test_ooi_repository.py +++ b/octopoes/tests/test_ooi_repository.py @@ -40,9 +40,7 @@ class TestOOIClass(OOI): internet = Network(name="internet") internet2 = Network(name="internet2") - test = TestOOIClass( - multiple_refs=[internet.reference, internet2.reference], - ) + test = TestOOIClass(multiple_refs=[internet.reference, internet2.reference]) serial = self.repository.serialize(test) self.assertEqual(["Network|internet", "Network|internet2"], serial["TestOOIClass/multiple_refs"]) self.assertEqual("TestOOIClass|test_id", serial["xt/id"]) diff --git a/octopoes/tests/test_path.py b/octopoes/tests/test_path.py index 1b0d2ebafa3..ab7910a2cfb 100644 --- a/octopoes/tests/test_path.py +++ b/octopoes/tests/test_path.py @@ -28,19 +28,13 @@ class PathTest(TestCase): def test_path_outoing_relation(self): path = Path.parse("MockResolvedHostname.hostname") self.assertCountEqual( - [ - Segment(MockResolvedHostname, Direction.OUTGOING, "hostname", MockHostname), - ], - path.segments, + [Segment(MockResolvedHostname, Direction.OUTGOING, "hostname", MockHostname)], path.segments ) def test_path_incoming_relation(self): path = Path.parse("MockHostname. None: ... def test_serialize_declared(self): - scan_profile = DeclaredScanProfile( - reference=Reference.from_str("MockIPAddressV4|internet|1.1.1.1"), - level=1, - ) + scan_profile = DeclaredScanProfile(reference=Reference.from_str("MockIPAddressV4|internet|1.1.1.1"), level=1) serialized = XTDBScanProfileRepository.serialize(scan_profile) @@ -27,10 +24,7 @@ def test_serialize_inherited(self): network = MockNetwork(name="internet") ip = MockIPAddressV4(address="1.1.1.1", network=network.reference) - scan_profile = InheritedScanProfile( - reference=ip.reference, - level=2, - ) + scan_profile = InheritedScanProfile(reference=ip.reference, level=2) serialized = XTDBScanProfileRepository.serialize(scan_profile) self.assertEqual("ScanProfile|MockIPAddressV4|internet|1.1.1.1", serialized["xt/id"]) @@ -38,9 +32,7 @@ def test_serialize_inherited(self): self.assertEqual("inherited", serialized["scan_profile_type"]) self.assertEqual(2, serialized["level"]) - def test_deserialize_declared( - self, - ): + def test_deserialize_declared(self): serialized = { "reference": "MockIPAddressV4|internet|1.1.1.1", "level": 1, @@ -54,9 +46,7 @@ def test_deserialize_declared( self.assertEqual("declared", scan_profile.scan_profile_type) self.assertEqual(1, scan_profile.level) - def test_deserialize_inherited_legacy( - self, - ): + def test_deserialize_inherited_legacy(self): serialized = { "reference": "MockIPAddressV4|internet|1.1.1.2", "level": 2, @@ -64,12 +54,7 @@ def test_deserialize_inherited_legacy( "xt/id": "ScanProfile|MockIPAddressV4|internet|1.1.1.1", "type": "ScanProfile", "inheritances": [ - { - "parent": "MockNetwork|internet2", - "source": "MockNetwork|internet2", - "level": 2, - "depth": 1, - } + {"parent": "MockNetwork|internet2", "source": "MockNetwork|internet2", "level": 2, "depth": 1} ], } scan_profile = XTDBScanProfileRepository.deserialize(serialized) diff --git a/octopoes/tests/test_type_analysis.py b/octopoes/tests/test_type_analysis.py index 4fe5b127783..b4ed78e138d 100644 --- a/octopoes/tests/test_type_analysis.py +++ b/octopoes/tests/test_type_analysis.py @@ -88,10 +88,7 @@ def test_ooi_to_concrete(self): ) def test_concrete_to_concrete(self): - self.assertSetEqual( - {MockIPAddressV4}, - to_concrete({MockIPAddressV4}), - ) + self.assertSetEqual({MockIPAddressV4}, to_concrete({MockIPAddressV4})) def test_type_by_name(self): self.assertEqual(MockIPAddressV4, type_by_name("MockIPAddressV4")) diff --git a/octopoes/tools/run_bit.py b/octopoes/tools/run_bit.py index 8aa9439cf81..e93f441df41 100755 --- a/octopoes/tools/run_bit.py +++ b/octopoes/tools/run_bit.py @@ -44,11 +44,7 @@ def run_bit(start_pdb, organization_code, bit_id, ooi): bit_definition = get_bit_definitions()[bit_id] - bit_instance = Origin( - origin_type=OriginType.INFERENCE, - method=bit_id, - source=ooi.reference, - ) + bit_instance = Origin(origin_type=OriginType.INFERENCE, method=bit_id, source=ooi.reference) try: try: diff --git a/pyproject.toml b/pyproject.toml index 8f4ab8bc977..08af569c6dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,9 @@ line-length = 120 # Support Python 3.10 and higher target-version = "py310" +[tool.ruff.format] +skip-magic-trailing-comma = true + [tool.ruff.lint] # Enable classic flake8, pyflakes, eradicate, and tidyimport # To be extended with DJ, PT, RUF, C90, D, PL, RET diff --git a/rocky/account/admin.py b/rocky/account/admin.py index 564513931ac..984c3f1d511 100644 --- a/rocky/account/admin.py +++ b/rocky/account/admin.py @@ -11,25 +11,13 @@ @admin.register(User) class KATUserAdmin(UserAdmin): model = User - list_display = ( - "email", - "is_staff", - "is_active", - ) + list_display = ("email", "is_staff", "is_active") fieldsets = ( (None, {"fields": ("email", "password", "full_name")}), (_("Permissions"), {"fields": ("is_active", "is_staff", "is_superuser", "groups", "user_permissions")}), (_("Important dates"), {"fields": ("last_login", "date_joined")}), ) - add_fieldsets = ( - ( - None, - { - "classes": ("wide",), - "fields": ("email", "password1", "password2", "is_staff"), - }, - ), - ) + add_fieldsets = ((None, {"classes": ("wide",), "fields": ("email", "password1", "password2", "is_staff")}),) search_fields = ("email",) ordering = ("email",) diff --git a/rocky/account/forms/account_setup.py b/rocky/account/forms/account_setup.py index faa20c2861e..c6e7149920b 100644 --- a/rocky/account/forms/account_setup.py +++ b/rocky/account/forms/account_setup.py @@ -43,11 +43,7 @@ class UserRegistrationForm(forms.Form): max_length=254, help_text=_("Enter an email address."), widget=forms.EmailInput( - attrs={ - "autocomplete": "off", - "placeholder": "name@example.com", - "aria-describedby": "explanation-email", - } + attrs={"autocomplete": "off", "placeholder": "name@example.com", "aria-describedby": "explanation-email"} ), ) password = forms.CharField( @@ -93,17 +89,8 @@ class AccountTypeSelectForm(forms.Form): account_type = forms.CharField( label=_("Account Type"), help_text=_("Every member of OpenKAT must be part of an account type."), - error_messages={ - "group": { - "required": _("Please select an account type to proceed."), - }, - }, - widget=forms.Select( - choices=ACCOUNT_TYPE_CHOICES, - attrs={ - "aria-describedby": "explanation-account-type", - }, - ), + error_messages={"group": {"required": _("Please select an account type to proceed.")}}, + widget=forms.Select(choices=ACCOUNT_TYPE_CHOICES, attrs={"aria-describedby": "explanation-account-type"}), ) @@ -115,11 +102,7 @@ class TrustedClearanceLevelRadioPawsForm(forms.Form): initial=-1, help_text=_("Select a clearance level you trust this member with."), widget=forms.RadioSelect(attrs={"radio_paws": True}), - error_messages={ - "trusted_clearance_level": { - "required": _("Please select a clearance level to proceed."), - }, - }, + error_messages={"trusted_clearance_level": {"required": _("Please select a clearance level to proceed.")}}, ) @@ -168,7 +151,7 @@ class Meta: "placeholder": _("The name of the organization."), "autocomplete": "off", "aria-describedby": _("explanation-organization-name"), - }, + } ), "code": forms.TextInput( attrs={ @@ -177,7 +160,7 @@ class Meta: ), "autocomplete": "off", "aria-describedby": _("explanation-organization-code"), - }, + } ), } error_messages = { @@ -212,15 +195,11 @@ class IndemnificationAddForm(BaseRockyForm): class AssignClearanceLevelForm(BaseRockyForm): - assigned_level = forms.BooleanField( - label=_("Trusted to change Clearance Levels."), - ) + assigned_level = forms.BooleanField(label=_("Trusted to change Clearance Levels.")) class AcknowledgeClearanceLevelForm(BaseRockyForm): - acknowledged_level = forms.BooleanField( - label=_("Acknowledged to change Clearance Levels."), - ) + acknowledged_level = forms.BooleanField(label=_("Acknowledged to change Clearance Levels.")) class OrganizationMemberEditForm(BaseRockyModelForm, TrustedClearanceLevelRadioPawsForm): @@ -281,17 +260,10 @@ class SetPasswordForm(auth_forms.SetPasswordForm): password """ - error_messages = { - "password_mismatch": _("The two password fields didn’t match."), - } + error_messages = {"password_mismatch": _("The two password fields didn’t match.")} new_password1 = forms.CharField( label=_("New password"), - widget=forms.PasswordInput( - attrs={ - "autocomplete": "new-password", - "placeholder": _("Enter a new password"), - } - ), + widget=forms.PasswordInput(attrs={"autocomplete": "new-password", "placeholder": _("Enter a new password")}), strip=False, help_text=get_password_validators_help_texts, validators=[validate_password], @@ -299,12 +271,7 @@ class SetPasswordForm(auth_forms.SetPasswordForm): new_password2 = forms.CharField( label=_("New password confirmation"), strip=False, - widget=forms.PasswordInput( - attrs={ - "autocomplete": "new-password", - "placeholder": _("Repeat the new password"), - } - ), + widget=forms.PasswordInput(attrs={"autocomplete": "new-password", "placeholder": _("Repeat the new password")}), help_text=_("Confirm the new password"), validators=[validate_password], ) diff --git a/rocky/account/forms/organization.py b/rocky/account/forms/organization.py index c582fa7755e..64a6b426f60 100644 --- a/rocky/account/forms/organization.py +++ b/rocky/account/forms/organization.py @@ -9,9 +9,7 @@ class OrganizationListForm(forms.Form): Creates a dropdown list of Organizations of a particular member. """ - error_messages = { - "required": _("Organization is required."), - } + error_messages = {"required": _("Organization is required.")} def __init__(self, user, exclude_organization=None, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/rocky/account/forms/password_reset.py b/rocky/account/forms/password_reset.py index 53185b61402..3507e596fb4 100644 --- a/rocky/account/forms/password_reset.py +++ b/rocky/account/forms/password_reset.py @@ -8,9 +8,5 @@ class PasswordResetForm(auth_forms.PasswordResetForm): label=_("Email address"), max_length=254, help_text=_("A reset link will be sent to this email"), - widget=forms.TextInput( - attrs={ - "placeholder": _("The email address connected to your OpenKAT-account"), - } - ), + widget=forms.TextInput(attrs={"placeholder": _("The email address connected to your OpenKAT-account")}), ) diff --git a/rocky/account/migrations/0001_initial.py b/rocky/account/migrations/0001_initial.py index fadbc8ef964..1dda75fcf9d 100644 --- a/rocky/account/migrations/0001_initial.py +++ b/rocky/account/migrations/0001_initial.py @@ -11,9 +11,7 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ("auth", "0012_alter_user_first_name_max_length"), - ] + dependencies = [("auth", "0012_alter_user_first_name_max_length")] operations = [ migrations.CreateModel( @@ -29,14 +27,8 @@ class Migration(migrations.Migration): verbose_name="superuser status", ), ), - ( - "first_name", - models.CharField(blank=True, max_length=150, verbose_name="first name"), - ), - ( - "last_name", - models.CharField(blank=True, max_length=150, verbose_name="last name"), - ), + ("first_name", models.CharField(blank=True, max_length=150, verbose_name="first name")), + ("last_name", models.CharField(blank=True, max_length=150, verbose_name="last name")), ("id", models.AutoField(primary_key=True, serialize=False, verbose_name="ID")), ("full_name", models.CharField(max_length=150, verbose_name="full name", blank=True)), ("email", account.models.LowercaseEmailField(max_length=254, unique=True, verbose_name="email")), @@ -80,8 +72,6 @@ class Migration(migrations.Migration): ), ), ], - options={ - "abstract": False, - }, - ), + options={"abstract": False}, + ) ] diff --git a/rocky/account/migrations/0001_squashed_0004_authtoken_authtoken_unique_name.py b/rocky/account/migrations/0001_squashed_0004_authtoken_authtoken_unique_name.py index 694b6190894..d16121b1913 100644 --- a/rocky/account/migrations/0001_squashed_0004_authtoken_authtoken_unique_name.py +++ b/rocky/account/migrations/0001_squashed_0004_authtoken_authtoken_unique_name.py @@ -19,9 +19,7 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ("auth", "0012_alter_user_first_name_max_length"), - ] + dependencies = [("auth", "0012_alter_user_first_name_max_length")] operations = [ migrations.CreateModel( @@ -80,9 +78,7 @@ class Migration(migrations.Migration): ), ), ], - options={ - "abstract": False, - }, + options={"abstract": False}, ), migrations.CreateModel( name="AuthToken", diff --git a/rocky/account/migrations/0002_remove_first_last_name.py b/rocky/account/migrations/0002_remove_first_last_name.py index 26bd5b3bf99..94576f562f8 100644 --- a/rocky/account/migrations/0002_remove_first_last_name.py +++ b/rocky/account/migrations/0002_remove_first_last_name.py @@ -4,17 +4,9 @@ class Migration(migrations.Migration): - dependencies = [ - ("account", "0001_initial"), - ] + dependencies = [("account", "0001_initial")] operations = [ - migrations.RemoveField( - model_name="katuser", - name="first_name", - ), - migrations.RemoveField( - model_name="katuser", - name="last_name", - ), + migrations.RemoveField(model_name="katuser", name="first_name"), + migrations.RemoveField(model_name="katuser", name="last_name"), ] diff --git a/rocky/account/migrations/0003_alter_katuser_full_name.py b/rocky/account/migrations/0003_alter_katuser_full_name.py index 90ee2ad613e..7e8b0531bbb 100644 --- a/rocky/account/migrations/0003_alter_katuser_full_name.py +++ b/rocky/account/migrations/0003_alter_katuser_full_name.py @@ -4,14 +4,10 @@ class Migration(migrations.Migration): - dependencies = [ - ("account", "0002_remove_first_last_name"), - ] + dependencies = [("account", "0002_remove_first_last_name")] operations = [ migrations.AlterField( - model_name="katuser", - name="full_name", - field=models.CharField(max_length=150, verbose_name="full name"), - ), + model_name="katuser", name="full_name", field=models.CharField(max_length=150, verbose_name="full name") + ) ] diff --git a/rocky/account/migrations/0004_authtoken_authtoken_unique_name.py b/rocky/account/migrations/0004_authtoken_authtoken_unique_name.py index 073ca49a359..a20b8c58a6d 100644 --- a/rocky/account/migrations/0004_authtoken_authtoken_unique_name.py +++ b/rocky/account/migrations/0004_authtoken_authtoken_unique_name.py @@ -7,9 +7,7 @@ class Migration(migrations.Migration): - dependencies = [ - ("account", "0003_alter_katuser_full_name"), - ] + dependencies = [("account", "0003_alter_katuser_full_name")] operations = [ migrations.CreateModel( diff --git a/rocky/account/mixins.py b/rocky/account/mixins.py index 951c6707628..cd0751e5bce 100644 --- a/rocky/account/mixins.py +++ b/rocky/account/mixins.py @@ -130,8 +130,7 @@ def verify_raise_clearance_level(self, level: int) -> bool: def raise_clearance_level(self, ooi_reference: Reference, level: int) -> bool: self.verify_raise_clearance_level(level) self.octopoes_api_connector.save_scan_profile( - DeclaredScanProfile(reference=ooi_reference, level=ScanLevel(level)), - datetime.now(timezone.utc), + DeclaredScanProfile(reference=ooi_reference, level=ScanLevel(level)), datetime.now(timezone.utc) ) return True @@ -154,11 +153,7 @@ def can_raise_clearance_level(self, ooi: OOI, level: int) -> bool: messages.error( self.request, _("Could not raise clearance level of %s to L%s. Indemnification not present at organization %s.") - % ( - ooi.reference.human_readable, - level, - self.organization.name, - ), + % (ooi.reference.human_readable, level, self.organization.name), ) except TrustedClearanceLevelTooLowException: @@ -169,11 +164,7 @@ def can_raise_clearance_level(self, ooi: OOI, level: int) -> bool: "You were trusted a clearance level of L%s. " "Contact your administrator to receive a higher clearance." ) - % ( - ooi.reference.human_readable, - level, - self.organization_member.acknowledged_clearance_level, - ), + % (ooi.reference.human_readable, level, self.organization_member.acknowledged_clearance_level), ) except AcknowledgedClearanceLevelTooLowException: messages.error( @@ -183,11 +174,7 @@ def can_raise_clearance_level(self, ooi: OOI, level: int) -> bool: "You acknowledged a clearance level of L%s. " "Please accept the clearance level first on your profile page to proceed." ) - % ( - ooi.reference.human_readable, - level, - self.organization_member.acknowledged_clearance_level, - ), + % (ooi.reference.human_readable, level, self.organization_member.acknowledged_clearance_level), ) return False diff --git a/rocky/account/models.py b/rocky/account/models.py index 7d082739f7d..7dd37dd1818 100644 --- a/rocky/account/models.py +++ b/rocky/account/models.py @@ -66,9 +66,7 @@ class KATUser(AbstractBaseUser, PermissionsMixin): full_name = models.CharField(_("full name"), max_length=150) email = LowercaseEmailField(_("email"), max_length=254, unique=True) is_staff = models.BooleanField( - _("staff status"), - default=False, - help_text=_("Designates whether the user can log into this admin site."), + _("staff status"), default=False, help_text=_("Designates whether the user can log into this admin site.") ) is_active = models.BooleanField( _("active"), @@ -125,9 +123,7 @@ class AuthToken(AbstractAuthToken): name = models.CharField(_("name"), max_length=150) class Meta: - constraints = [ - models.UniqueConstraint("user", Lower("name"), name="unique name"), - ] + constraints = [models.UniqueConstraint("user", Lower("name"), name="unique name")] def __str__(self): return f"{self.name} ({self.user})" diff --git a/rocky/account/urls.py b/rocky/account/urls.py index 726a52b8e8d..8f57b8879a3 100644 --- a/rocky/account/urls.py +++ b/rocky/account/urls.py @@ -10,14 +10,6 @@ "two_factor/setup/", views.SetupRockyView.as_view(), name="setup" ), # Bypass the two_factor setup show that users have to be verified path("recover-email/", views.RecoverEmailView.as_view(), name="recover_email"), - path( - "password_reset/", - views.PasswordResetView.as_view(), - name="password_reset", - ), - path( - "reset///", - views.PasswordResetConfirmView.as_view(), - name="password_reset_confirm", - ), + path("password_reset/", views.PasswordResetView.as_view(), name="password_reset"), + path("reset///", views.PasswordResetConfirmView.as_view(), name="password_reset_confirm"), ] diff --git a/rocky/account/views/account.py b/rocky/account/views/account.py index 49c5c298565..c16b78188ed 100644 --- a/rocky/account/views/account.py +++ b/rocky/account/views/account.py @@ -36,7 +36,5 @@ class AccountView(OrganizationView, TemplateView, OOIClearanceMixin): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["breadcrumbs"] = [ - {"url": "", "text": _("Account details")}, - ] + context["breadcrumbs"] = [{"url": "", "text": _("Account details")}] return context diff --git a/rocky/account/views/login.py b/rocky/account/views/login.py index f3bb1cd9f02..96bad61ddc8 100644 --- a/rocky/account/views/login.py +++ b/rocky/account/views/login.py @@ -20,11 +20,7 @@ @class_view_decorator(sensitive_post_parameters()) @class_view_decorator(never_cache) class LoginRockyView(LoginView): - form_list = ( - ("auth", LoginForm), - ("token", TwoFactorVerifyTokenForm), - ("backup", TwoFactorBackupTokenForm), - ) + form_list = (("auth", LoginForm), ("token", TwoFactorVerifyTokenForm), ("backup", TwoFactorBackupTokenForm)) def get_form(self, step=None, **kwargs): """ @@ -37,10 +33,7 @@ def get_form(self, step=None, **kwargs): form = TwoFactorBackupTokenForm(user=self.get_user(), initial_device=self.get_device(), **kwargs) if self.show_timeout_error: form.cleaned_data = getattr(form, "cleaned_data", {}) - form.add_error( - None, - ValidationError(_("Your session has timed out. Please login again.")), - ) + form.add_error(None, ValidationError(_("Your session has timed out. Please login again."))) return form def get_context_data(self, form, **kwargs): @@ -49,14 +42,8 @@ def get_context_data(self, form, **kwargs): context["two_factor_enabled"] = default_device(self.request.user) context["form_name"] = "login" context["breadcrumbs"] = [ - { - "url": reverse("landing_page"), - "text": _("OpenKAT"), - }, - { - "url": reverse("login"), - "text": _("Login"), - }, + {"url": reverse("landing_page"), "text": _("OpenKAT")}, + {"url": reverse("login"), "text": _("Login")}, ] return context @@ -71,22 +58,13 @@ def get_success_url(self): @class_view_decorator(never_cache) class SetupRockyView(SetupView): # This is set to skip the extra welcome form which is for OpenKAT a redundant step. - form_list = ( - ("method", MethodForm), - ("generator", TwoFactorSetupTokenForm), - ) + form_list = (("method", MethodForm), ("generator", TwoFactorSetupTokenForm)) def get_context_data(self, form, **kwargs): context = super().get_context_data(form, **kwargs) context["breadcrumbs"] = [ - { - "url": reverse("login"), - "text": _("Login"), - }, - { - "url": reverse("setup"), - "text": _("Two factor authentication"), - }, + {"url": reverse("login"), "text": _("Login")}, + {"url": reverse("setup"), "text": _("Two factor authentication")}, ] return context diff --git a/rocky/account/views/password_reset.py b/rocky/account/views/password_reset.py index 70dccfe81bf..85c56431abf 100644 --- a/rocky/account/views/password_reset.py +++ b/rocky/account/views/password_reset.py @@ -23,14 +23,8 @@ class PasswordResetView(auth_views.PasswordResetView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["breadcrumbs"] = [ - { - "url": reverse("login"), - "text": _("Login"), - }, - { - "url": reverse("password_reset"), - "text": _("Reset password"), - }, + {"url": reverse("login"), "text": _("Login")}, + {"url": reverse("password_reset"), "text": _("Reset password")}, ] return context @@ -43,10 +37,7 @@ def form_valid(self, form): return super().form_valid(form) def is_smtp_valid(self): - smtp_credentials = [ - settings.EMAIL_HOST, - settings.EMAIL_PORT, - ] + smtp_credentials = [settings.EMAIL_HOST, settings.EMAIL_PORT] return not ("" in smtp_credentials or None in smtp_credentials) def add_error_notification(self): diff --git a/rocky/account/views/recover_email.py b/rocky/account/views/recover_email.py index b55c45b36f5..0a92de9be85 100644 --- a/rocky/account/views/recover_email.py +++ b/rocky/account/views/recover_email.py @@ -15,13 +15,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["help_desk_email"] = settings.HELP_DESK_EMAIL context["breadcrumbs"] = [ - { - "url": reverse("login"), - "text": _("Login"), - }, - { - "url": reverse("recover_email"), - "text": _("Recover email address"), - }, + {"url": reverse("login"), "text": _("Login")}, + {"url": reverse("recover_email"), "text": _("Recover email address")}, ] return context diff --git a/rocky/crisis_room/urls.py b/rocky/crisis_room/urls.py index 64b62e41cd2..3fad14c20ac 100644 --- a/rocky/crisis_room/urls.py +++ b/rocky/crisis_room/urls.py @@ -2,6 +2,4 @@ from . import views -urlpatterns = [ - path("", views.CrisisRoomView.as_view(), name="crisis_room"), -] +urlpatterns = [path("", views.CrisisRoomView.as_view(), name="crisis_room")] diff --git a/rocky/crisis_room/views.py b/rocky/crisis_room/views.py index 6e16e8ba0ef..9974d0bc105 100644 --- a/rocky/crisis_room/views.py +++ b/rocky/crisis_room/views.py @@ -42,9 +42,7 @@ def total_critical(self) -> int: class CrisisRoomView(BreadcrumbsMixin, ConnectorFormMixin, ObservedAtMixin, TemplateView): template_name = "crisis_room/crisis_room.html" connector_form_class = ObservedAtForm - breadcrumbs = [ - {"url": "", "text": "Crisis Room"}, - ] + breadcrumbs = [{"url": "", "text": "Crisis Room"}] def sort_by_total( self, finding_counts: list[OrganizationFindingCountPerSeverity] @@ -81,16 +79,12 @@ def get_context_data(self, **kwargs): # query each organization's finding type count org_finding_counts_per_severity = [ OrganizationFindingCountPerSeverity( - name=org.name, - code=org.code, - finding_count_per_severity=self.get_finding_type_severity_count(org), + name=org.name, code=org.code, finding_count_per_severity=self.get_finding_type_severity_count(org) ) for org in user.organizations ] - context["breadcrumb_list"] = [ - {"url": reverse("crisis_room"), "text": "CRISIS ROOM"}, - ] + context["breadcrumb_list"] = [{"url": reverse("crisis_room"), "text": "CRISIS ROOM"}] context["organizations"] = user.organizations diff --git a/rocky/fmea/migrations/0001_initial.py b/rocky/fmea/migrations/0001_initial.py index fab3a5b5018..9b664ab075e 100644 --- a/rocky/fmea/migrations/0001_initial.py +++ b/rocky/fmea/migrations/0001_initial.py @@ -11,15 +11,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name="FailureMode", fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("failure_mode", models.CharField(max_length=256)), ( "severity_level", @@ -39,14 +31,8 @@ class Migration(migrations.Migration): models.PositiveSmallIntegerField( choices=[ ("", "--- Select an option ----"), - ( - 1, - "1. Very Rare. Incident (almost) never occurs, almost unthinkable.", - ), - ( - 2, - "2. Rare. Incidents occur less than once a year (3-5).", - ), + (1, "1. Very Rare. Incident (almost) never occurs, almost unthinkable."), + (2, "2. Rare. Incidents occur less than once a year (3-5)."), (3, "3. Occurs. Incidents occur several times a year."), (4, "4. Regularly. Incidents occur weekly."), (5, "5. Frequent. Incidents occur daily."), @@ -58,22 +44,10 @@ class Migration(migrations.Migration): models.PositiveSmallIntegerField( choices=[ ("", "--- Select an option ----"), - ( - 1, - "1. Always Detectable. Incident (almost) never occurs, almost unthinkable.", - ), - ( - 2, - "2. Usually Detectable. Incidents occur less than once a year (3-5).", - ), - ( - 3, - "3. Detectable. Failure mode is detectable with effort.", - ), - ( - 4, - "4. Poorly Detectable. Detecting the failure mode is difficult.", - ), + (1, "1. Always Detectable. Incident (almost) never occurs, almost unthinkable."), + (2, "2. Usually Detectable. Incidents occur less than once a year (3-5)."), + (3, "3. Detectable. Failure mode is detectable with effort."), + (4, "4. Poorly Detectable. Detecting the failure mode is difficult."), ( 5, "5. Almost Undetectable. " @@ -86,22 +60,12 @@ class Migration(migrations.Migration): ("effect", models.CharField(blank=True, max_length=256)), ("description", models.CharField(blank=True, max_length=256)), ], - options={ - "verbose_name_plural": "Failure modes", - }, + options={"verbose_name_plural": "Failure modes"}, ), migrations.CreateModel( name="FailureModeDepartment", fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ( "affected_department", models.PositiveSmallIntegerField( @@ -118,15 +82,9 @@ class Migration(migrations.Migration): ), ( "failure_mode", - models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="fmea.failuremode", - ), + models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to="fmea.failuremode"), ), ], - options={ - "verbose_name_plural": "Failure Mode Departments", - }, + options={"verbose_name_plural": "Failure Mode Departments"}, ), ] diff --git a/rocky/fmea/migrations/0002_auto_20220120_0839.py b/rocky/fmea/migrations/0002_auto_20220120_0839.py index a4c237701f2..8dc6d3f7233 100644 --- a/rocky/fmea/migrations/0002_auto_20220120_0839.py +++ b/rocky/fmea/migrations/0002_auto_20220120_0839.py @@ -5,23 +5,13 @@ class Migration(migrations.Migration): - dependencies = [ - ("fmea", "0001_initial"), - ] + dependencies = [("fmea", "0001_initial")] operations = [ migrations.CreateModel( name="FailureModeAffectedObject", fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ( "affected_department", models.CharField( @@ -39,22 +29,12 @@ class Migration(migrations.Migration): ), ("affected_ooi_type", models.CharField(max_length=100)), ], - options={ - "verbose_name_plural": "Failure Mode Affected Objects", - }, + options={"verbose_name_plural": "Failure Mode Affected Objects"}, ), migrations.CreateModel( name="FailureModeTreeObject", fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("tree_object", models.CharField(max_length=256)), ( "affected_department", @@ -79,33 +59,16 @@ class Migration(migrations.Migration): field=models.PositiveSmallIntegerField( choices=[ ("", "--- Select an option ----"), - ( - 1, - "Level 1: Always Detectable. Incident (almost) never occurs, almost unthinkable.", - ), - ( - 2, - "Level 2: Usually Detectable. Incidents occur less than once a year (3-5).", - ), - ( - 3, - "Level 3: Detectable. Failure mode is detectable with effort.", - ), - ( - 4, - "Level 4: Poorly Detectable. Detecting the failure mode is difficult.", - ), - ( - 5, - "Level 5: Almost Undetectable. Failure mode detection is very difficult or nearly impossible.", - ), + (1, "Level 1: Always Detectable. Incident (almost) never occurs, almost unthinkable."), + (2, "Level 2: Usually Detectable. Incidents occur less than once a year (3-5)."), + (3, "Level 3: Detectable. Failure mode is detectable with effort."), + (4, "Level 4: Poorly Detectable. Detecting the failure mode is difficult."), + (5, "Level 5: Almost Undetectable. Failure mode detection is very difficult or nearly impossible."), ] ), ), migrations.AlterField( - model_name="failuremode", - name="failure_mode", - field=models.CharField(max_length=256, unique=True), + model_name="failuremode", name="failure_mode", field=models.CharField(max_length=256, unique=True) ), migrations.AlterField( model_name="failuremode", @@ -113,10 +76,7 @@ class Migration(migrations.Migration): field=models.PositiveSmallIntegerField( choices=[ ("", "--- Select an option ----"), - ( - 1, - "Level 1: Very Rare. Incident (almost) never occurs, almost unthinkable.", - ), + (1, "Level 1: Very Rare. Incident (almost) never occurs, almost unthinkable."), (2, "Level 2: Rare. Incidents occur less than once a year (3-5)."), (3, "Level 3: Occurs. Incidents occur several times a year."), (4, "Level 4: Regularly. Incidents occur weekly."), @@ -138,16 +98,10 @@ class Migration(migrations.Migration): ] ), ), - migrations.DeleteModel( - name="FailureModeDepartment", - ), + migrations.DeleteModel(name="FailureModeDepartment"), migrations.AddField( model_name="failuremodeaffectedobject", name="failure_mode", - field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="fmea.failuremode", - ), + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to="fmea.failuremode"), ), ] diff --git a/rocky/fmea/migrations/0003_auto_20220203_1534.py b/rocky/fmea/migrations/0003_auto_20220203_1534.py index 3f6a49a5b96..ece4f437458 100644 --- a/rocky/fmea/migrations/0003_auto_20220203_1534.py +++ b/rocky/fmea/migrations/0003_auto_20220203_1534.py @@ -4,23 +4,13 @@ class Migration(migrations.Migration): - dependencies = [ - ("fmea", "0002_auto_20220120_0839"), - ] + dependencies = [("fmea", "0002_auto_20220120_0839")] operations = [ migrations.CreateModel( name="FailureModeEffect", fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("effect", models.TextField(max_length=256, unique=True)), ( "severity_level", @@ -37,27 +27,15 @@ class Migration(migrations.Migration): ), ], ), - migrations.RemoveField( - model_name="failuremode", - name="severity_level", - ), + migrations.RemoveField(model_name="failuremode", name="severity_level"), migrations.AddField( - model_name="failuremode", - name="critical_score", - field=models.PositiveSmallIntegerField(default=0), + model_name="failuremode", name="critical_score", field=models.PositiveSmallIntegerField(default=0) ), migrations.AddField( - model_name="failuremode", - name="risk_priority_number", - field=models.PositiveSmallIntegerField(default=0), - ), - migrations.RemoveField( - model_name="failuremode", - name="effect", + model_name="failuremode", name="risk_priority_number", field=models.PositiveSmallIntegerField(default=0) ), + migrations.RemoveField(model_name="failuremode", name="effect"), migrations.AddField( - model_name="failuremode", - name="effect", - field=models.ManyToManyField(to="fmea.FailureModeEffect"), + model_name="failuremode", name="effect", field=models.ManyToManyField(to="fmea.FailureModeEffect") ), ] diff --git a/rocky/fmea/migrations/0004_remove_failuremode_effect_and_more.py b/rocky/fmea/migrations/0004_remove_failuremode_effect_and_more.py index a20f0efa96d..c92854b2f46 100644 --- a/rocky/fmea/migrations/0004_remove_failuremode_effect_and_more.py +++ b/rocky/fmea/migrations/0004_remove_failuremode_effect_and_more.py @@ -4,29 +4,13 @@ class Migration(migrations.Migration): - dependencies = [ - ("fmea", "0003_auto_20220203_1534"), - ] + dependencies = [("fmea", "0003_auto_20220203_1534")] operations = [ - migrations.RemoveField( - model_name="failuremode", - name="effect", - ), - migrations.RemoveField( - model_name="failuremodeaffectedobject", - name="failure_mode", - ), - migrations.DeleteModel( - name="FailureModeTreeObject", - ), - migrations.DeleteModel( - name="FailureMode", - ), - migrations.DeleteModel( - name="FailureModeAffectedObject", - ), - migrations.DeleteModel( - name="FailureModeEffect", - ), + migrations.RemoveField(model_name="failuremode", name="effect"), + migrations.RemoveField(model_name="failuremodeaffectedobject", name="failure_mode"), + migrations.DeleteModel(name="FailureModeTreeObject"), + migrations.DeleteModel(name="FailureMode"), + migrations.DeleteModel(name="FailureModeAffectedObject"), + migrations.DeleteModel(name="FailureModeEffect"), ] diff --git a/rocky/katalogus/client.py b/rocky/katalogus/client.py index 84425683c91..12da80c2a58 100644 --- a/rocky/katalogus/client.py +++ b/rocky/katalogus/client.py @@ -178,8 +178,7 @@ def get_enabled_normalizers(self) -> list[Plugin]: def _patch_boefje_state(self, boefje_id: str, enabled: bool, repository_id: str) -> None: response = self.session.patch( - f"{self.organization_uri}/repositories/{repository_id}/plugins/{boefje_id}", - json={"enabled": enabled}, + f"{self.organization_uri}/repositories/{repository_id}/plugins/{boefje_id}", json={"enabled": enabled} ) response.raise_for_status() diff --git a/rocky/katalogus/forms/katalogus_filter.py b/rocky/katalogus/forms/katalogus_filter.py index 5e344dd3343..edb4062976f 100644 --- a/rocky/katalogus/forms/katalogus_filter.py +++ b/rocky/katalogus/forms/katalogus_filter.py @@ -2,11 +2,7 @@ from django.utils.translation import gettext_lazy as _ from tools.forms.base import BaseRockyForm -FILTER_OPTIONS = ( - ("all", _("Show all")), - ("enabled", _("Enabled")), - ("disabled", _("Disabled")), -) +FILTER_OPTIONS = (("all", _("Show all")), ("enabled", _("Enabled")), ("disabled", _("Disabled"))) SORTING_OPTIONS = ( ("a-z", "A-Z"), diff --git a/rocky/katalogus/forms/plugin_settings.py b/rocky/katalogus/forms/plugin_settings.py index d09ac808066..303f0df4816 100644 --- a/rocky/katalogus/forms/plugin_settings.py +++ b/rocky/katalogus/forms/plugin_settings.py @@ -9,9 +9,7 @@ class PluginSchemaForm(forms.Form): """This Form takes a plugin schema and turn all settings of schema into form fields.""" - error_messages = { - "required": _("This field is required."), - } + error_messages = {"required": _("This field is required.")} def __init__(self, plugin_schema: dict, values: dict, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/rocky/katalogus/health.py b/rocky/katalogus/health.py index 8024d6e2936..4d1547d2712 100644 --- a/rocky/katalogus/health.py +++ b/rocky/katalogus/health.py @@ -15,8 +15,6 @@ def get_katalogus_health() -> ServiceHealth: except HTTPError: logger.exception("Error while retrieving KATalogus health state") katalogus_health = ServiceHealth( - service="katalogus", - healthy=False, - additional="Could not connect to KATalogus. Service is possibly down", + service="katalogus", healthy=False, additional="Could not connect to KATalogus. Service is possibly down" ) return katalogus_health diff --git a/rocky/katalogus/urls.py b/rocky/katalogus/urls.py index 8c0834ab52b..a8eca77a2d4 100644 --- a/rocky/katalogus/urls.py +++ b/rocky/katalogus/urls.py @@ -11,65 +11,31 @@ urlpatterns = [ path("", KATalogusView.as_view(), name="katalogus"), path("view//", KATalogusView.as_view(), name="katalogus"), - path( - "settings/", - KATalogusSettingsView.as_view(), - name="katalogus_settings", - ), - path( - "settings/migrate/", - KATalogusSettingsView.as_view(), - name="katalogus_clone_settings", - ), + path("settings/", KATalogusSettingsView.as_view(), name="katalogus_settings"), + path("settings/migrate/", KATalogusSettingsView.as_view(), name="katalogus_clone_settings"), path( "settings/migrate/confirmation//", ConfirmCloneSettingsView.as_view(), name="confirm_clone_settings", ), - path( - "plugins/boefjes//", - BoefjeListView.as_view(), - name="boefjes_list", - ), - path( - "plugins/normalizers//", - NormalizerListView.as_view(), - name="normalizers_list", - ), - path( - "plugins/about-plugins/", - AboutPluginsView.as_view(), - name="about_plugins", - ), - path( - "plugins/boefje//", - BoefjeDetailView.as_view(), - name="boefje_detail", - ), - path( - "plugins/normalizer//", - NormalizerDetailView.as_view(), - name="normalizer_detail", - ), + path("plugins/boefjes//", BoefjeListView.as_view(), name="boefjes_list"), + path("plugins/normalizers//", NormalizerListView.as_view(), name="normalizers_list"), + path("plugins/about-plugins/", AboutPluginsView.as_view(), name="about_plugins"), + path("plugins/boefje//", BoefjeDetailView.as_view(), name="boefje_detail"), + path("plugins/normalizer//", NormalizerDetailView.as_view(), name="normalizer_detail"), path( "plugins////", PluginEnableDisableView.as_view(), name="plugin_enable_disable", ), - path( - "plugins//cover.jpg", - PluginCoverImgView.as_view(), - name="plugin_cover", - ), + path("plugins//cover.jpg", PluginCoverImgView.as_view(), name="plugin_cover"), path( "plugins///change-clearance-level//", ChangeClearanceLevel.as_view(), name="change_clearance_level", ), path( - "plugins///settings/add/", - PluginSettingsAddView.as_view(), - name="plugin_settings_add", + "plugins///settings/add/", PluginSettingsAddView.as_view(), name="plugin_settings_add" ), path( "plugins///settings/delete/", diff --git a/rocky/katalogus/views/change_clearance_level.py b/rocky/katalogus/views/change_clearance_level.py index f3dcbbf2eaa..bd44b069350 100644 --- a/rocky/katalogus/views/change_clearance_level.py +++ b/rocky/katalogus/views/change_clearance_level.py @@ -26,10 +26,7 @@ def get(self, request, *args, **kwargs): return redirect( reverse( "boefje_detail", - kwargs={ - "organization_code": self.organization.code, - "plugin_id": kwargs["plugin_id"], - }, + kwargs={"organization_code": self.organization.code, "plugin_id": kwargs["plugin_id"]}, ) ) return super().get(request, *args, **kwargs) @@ -58,11 +55,7 @@ def get_context_data(self, **kwargs): }, { "url": reverse( - "boefje_detail", - kwargs={ - "organization_code": self.organization.code, - "plugin_id": self.plugin.id, - }, + "boefje_detail", kwargs={"organization_code": self.organization.code, "plugin_id": self.plugin.id} ), "text": self.plugin.name, }, diff --git a/rocky/katalogus/views/katalogus.py b/rocky/katalogus/views/katalogus.py index e63748b49a2..18b56d5bcff 100644 --- a/rocky/katalogus/views/katalogus.py +++ b/rocky/katalogus/views/katalogus.py @@ -57,10 +57,7 @@ def get_context_data(self, **kwargs): context["view_type"] = self.kwargs.get("view_type", "grid") context["url_name"] = self.request.resolver_match.url_name context["breadcrumbs"] = [ - { - "url": reverse("katalogus", kwargs={"organization_code": self.organization.code}), - "text": _("KAT-alogus"), - }, + {"url": reverse("katalogus", kwargs={"organization_code": self.organization.code}), "text": _("KAT-alogus")} ] return context @@ -102,9 +99,6 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["view_type"] = self.kwargs.get("view_type", "grid") context["breadcrumbs"] = [ - { - "url": reverse("katalogus", kwargs={"organization_code": self.organization.code}), - "text": _("KAT-alogus"), - }, + {"url": reverse("katalogus", kwargs={"organization_code": self.organization.code}), "text": _("KAT-alogus")} ] return context diff --git a/rocky/katalogus/views/katalogus_settings.py b/rocky/katalogus/views/katalogus_settings.py index 14edf7a5719..f9e827801fd 100644 --- a/rocky/katalogus/views/katalogus_settings.py +++ b/rocky/katalogus/views/katalogus_settings.py @@ -15,10 +15,7 @@ class ConfirmCloneSettingsView( - OrganizationPermissionRequiredMixin, - OrganizationView, - UserPassesTestMixin, - TemplateView, + OrganizationPermissionRequiredMixin, OrganizationView, UserPassesTestMixin, TemplateView ): template_name = "confirmation_clone_settings.html" permission_required = "tools.can_set_katalogus_settings" @@ -41,12 +38,7 @@ def post(self, request, *args, **kwargs): messages.SUCCESS, _("Settings from {} to {} successfully cloned.").format(self.organization.name, to_organization.name), ) - return HttpResponseRedirect( - reverse( - "katalogus_settings", - kwargs={"organization_code": self.organization.code}, - ) - ) + return HttpResponseRedirect(reverse("katalogus_settings", kwargs={"organization_code": self.organization.code})) class KATalogusSettingsView(OrganizationPermissionRequiredMixin, OrganizationView, FormView): @@ -64,10 +56,7 @@ def get_context_data(self, **kwargs): "text": _("KAT-alogus"), }, { - "url": reverse( - "katalogus_settings", - kwargs={"organization_code": self.organization.code}, - ), + "url": reverse("katalogus_settings", kwargs={"organization_code": self.organization.code}), "text": _("Settings"), }, ] @@ -85,9 +74,7 @@ def get_settings(self): plugin_setting = katalogus_client.get_plugin_settings(boefje.id) except HTTPError: messages.add_message( - self.request, - messages.ERROR, - _("Failed getting settings for boefje {}").format(self.plugin.id), + self.request, messages.ERROR, _("Failed getting settings for boefje {}").format(self.plugin.id) ) continue @@ -96,12 +83,7 @@ def get_settings(self): for key, value in plugin_setting.items(): all_plugins_settings.append( - { - "plugin_id": boefje.id, - "plugin_name": boefje.name, - "name": key, - "value": value, - } + {"plugin_id": boefje.id, "plugin_name": boefje.name, "name": key, "value": value} ) return all_plugins_settings @@ -117,8 +99,5 @@ def form_valid(self, form): def get_success_url(self, **kwargs): return reverse_lazy( "confirm_clone_settings", - kwargs={ - "organization_code": self.organization.code, - "to_organization": kwargs["to_organization"], - }, + kwargs={"organization_code": self.organization.code, "to_organization": kwargs["to_organization"]}, ) diff --git a/rocky/katalogus/views/mixins.py b/rocky/katalogus/views/mixins.py index fe4146bdf5f..189e362373e 100644 --- a/rocky/katalogus/views/mixins.py +++ b/rocky/katalogus/views/mixins.py @@ -89,11 +89,7 @@ def run_boefje(self, katalogus_boefje: KATalogusBoefje, ooi: OOI | None) -> None task = PrioritizedItem(priority=1, data=boefje_task) schedule_task(self.request, self.organization.code, task) - def run_boefje_for_oois( - self, - boefje: KATalogusBoefje, - oois: list[OOI], - ) -> None: + def run_boefje_for_oois(self, boefje: KATalogusBoefje, oois: list[OOI]) -> None: if not oois and not boefje.consumes: self.run_boefje(boefje, None) diff --git a/rocky/katalogus/views/plugin_detail.py b/rocky/katalogus/views/plugin_detail.py index b1e524f2bd7..f7c20eca265 100644 --- a/rocky/katalogus/views/plugin_detail.py +++ b/rocky/katalogus/views/plugin_detail.py @@ -114,10 +114,7 @@ def get_context_data(self, **kwargs): { "url": reverse( "normalizer_detail", - kwargs={ - "organization_code": self.organization.code, - "plugin_id": self.plugin.id, - }, + kwargs={"organization_code": self.organization.code, "plugin_id": self.plugin.id}, ), "text": self.plugin.name, }, @@ -153,11 +150,7 @@ def get_context_data(self, **kwargs): }, { "url": reverse( - "boefje_detail", - kwargs={ - "organization_code": self.organization.code, - "plugin_id": self.plugin.id, - }, + "boefje_detail", kwargs={"organization_code": self.organization.code, "plugin_id": self.plugin.id} ), "text": self.plugin.name, }, @@ -189,10 +182,7 @@ def post(self, request, *args, **kwargs): oois_without_clearance_level = oois["oois_without_clearance"] if oois_with_clearance_level: - self.run_boefje_for_oois( - boefje=boefje, - oois=oois_with_clearance_level, - ) + self.run_boefje_for_oois(boefje=boefje, oois=oois_with_clearance_level) if oois_without_clearance_level: if not self.organization_member.has_perm("tools.can_set_clearance_level"): @@ -242,7 +232,4 @@ def get_oois(self, selected_oois: list[str]) -> dict[str, Any]: oois_with_clearance.append(ooi_object) else: oois_without_clearance.append(ooi_object.primary_key) - return { - "oois_with_clearance": oois_with_clearance, - "oois_without_clearance": oois_without_clearance, - } + return {"oois_with_clearance": oois_with_clearance, "oois_without_clearance": oois_without_clearance} diff --git a/rocky/katalogus/views/plugin_enable_disable.py b/rocky/katalogus/views/plugin_enable_disable.py index 8717c6d540b..e98352918c1 100644 --- a/rocky/katalogus/views/plugin_enable_disable.py +++ b/rocky/katalogus/views/plugin_enable_disable.py @@ -40,11 +40,7 @@ def post(self, request, *args, **kwargs): ) return redirect( reverse( - "boefje_detail", - kwargs={ - "organization_code": self.organization.code, - "plugin_id": self.plugin.id, - }, + "boefje_detail", kwargs={"organization_code": self.organization.code, "plugin_id": self.plugin.id} ) ) @@ -68,9 +64,7 @@ def post(self, request, *args, **kwargs): if self.plugin.can_scan(self.organization_member): self.katalogus_client.enable_boefje(self.plugin) messages.add_message( - self.request, - messages.SUCCESS, - _("{} '{}' enabled.").format(self.plugin.type.title(), self.plugin.name), + self.request, messages.SUCCESS, _("{} '{}' enabled.").format(self.plugin.type.title(), self.plugin.name) ) else: member_clearance_level_text = ( @@ -91,8 +85,7 @@ def post(self, request, *args, **kwargs): self.request, messages.ERROR, _("To enable {} you need at least a clearance level of L{}. " + member_clearance_level_text).format( - self.plugin.name.title(), - self.plugin.scan_level.value, + self.plugin.name.title(), self.plugin.scan_level.value ), ) diff --git a/rocky/katalogus/views/plugin_settings_add.py b/rocky/katalogus/views/plugin_settings_add.py index 5abbf5bc358..1df62bac8a7 100644 --- a/rocky/katalogus/views/plugin_settings_add.py +++ b/rocky/katalogus/views/plugin_settings_add.py @@ -43,9 +43,7 @@ def dispatch(self, request, *args, **kwargs): def form_valid(self, form): if form.cleaned_data == {}: messages.add_message( - self.request, - messages.WARNING, - _("No changes to the settings added: no form data present"), + self.request, messages.WARNING, _("No changes to the settings added: no form data present") ) return redirect(self.get_success_url()) @@ -76,11 +74,7 @@ def get_context_data(self, **kwargs): }, { "url": reverse( - "boefje_detail", - kwargs={ - "organization_code": self.organization.code, - "plugin_id": self.plugin.id, - }, + "boefje_detail", kwargs={"organization_code": self.organization.code, "plugin_id": self.plugin.id} ), "text": self.plugin.name, }, @@ -103,11 +97,7 @@ def get_context_data(self, **kwargs): def get_success_url(self): return reverse( - "boefje_detail", - kwargs={ - "organization_code": self.organization.code, - "plugin_id": self.plugin.id, - }, + "boefje_detail", kwargs={"organization_code": self.organization.code, "plugin_id": self.plugin.id} ) def add_error_notification(self, message): diff --git a/rocky/katalogus/views/plugin_settings_delete.py b/rocky/katalogus/views/plugin_settings_delete.py index 6762c013f1e..21563167fd4 100644 --- a/rocky/katalogus/views/plugin_settings_delete.py +++ b/rocky/katalogus/views/plugin_settings_delete.py @@ -26,11 +26,7 @@ def get_context_data(self, **kwargs): }, { "url": reverse( - "boefje_detail", - kwargs={ - "organization_code": self.organization.code, - "plugin_id": self.plugin.id, - }, + "boefje_detail", kwargs={"organization_code": self.organization.code, "plugin_id": self.plugin.id} ), "text": self.plugin.name, }, @@ -54,27 +50,19 @@ def get_context_data(self, **kwargs): def get_success_url(self): return reverse( - "boefje_detail", - kwargs={ - "organization_code": self.organization.code, - "plugin_id": self.plugin.id, - }, + "boefje_detail", kwargs={"organization_code": self.organization.code, "plugin_id": self.plugin.id} ) def delete(self, request, *args, **kwargs): try: self.katalogus_client.delete_plugin_settings(self.plugin.id) messages.add_message( - request, - messages.SUCCESS, - _("Settings for plugin {} successfully deleted.").format(self.plugin.name), + request, messages.SUCCESS, _("Settings for plugin {} successfully deleted.").format(self.plugin.name) ) except HTTPError as e: if isinstance(e, HTTPStatusError) and e.response.status_code == codes.NOT_FOUND: messages.add_message( - request, - messages.WARNING, - _("Plugin {} has no settings.").format(self.plugin.name), + request, messages.WARNING, _("Plugin {} has no settings.").format(self.plugin.name) ) else: messages.add_message( diff --git a/rocky/onboarding/forms.py b/rocky/onboarding/forms.py index 0f163976f90..2f08636187f 100644 --- a/rocky/onboarding/forms.py +++ b/rocky/onboarding/forms.py @@ -24,16 +24,9 @@ class OnboardingSetClearanceLevelForm(forms.Form): help_text=_( "Boefjes that has a scan level below or equal to the clearance level, is permitted to scan an object." ), - error_messages={ - "level": { - "required": _("Please select a clearance level to proceed."), - }, - }, + error_messages={"level": {"required": _("Please select a clearance level to proceed.")}}, widget=ClearanceLevelSelect( - choices=SCAN_LEVEL_CHOICES, - attrs={ - "aria-describedby": _("explanation-clearance-level"), - }, + choices=SCAN_LEVEL_CHOICES, attrs={"aria-describedby": _("explanation-clearance-level")} ), ) diff --git a/rocky/onboarding/urls.py b/rocky/onboarding/urls.py index 0c8763347da..dbee64d25d4 100644 --- a/rocky/onboarding/urls.py +++ b/rocky/onboarding/urls.py @@ -5,9 +5,7 @@ urlpatterns = [ path("/", views.OnboardingStart.as_view(), name="onboarding_index"), path( - "/step/introduction/", - views.OnboardingIntroductionView.as_view(), - name="step_introduction", + "/step/introduction/", views.OnboardingIntroductionView.as_view(), name="step_introduction" ), path( "/step/choose-report-info/", @@ -54,21 +52,13 @@ views.OnboardingSetupScanOOIAddView.as_view(), name="step_setup_scan_ooi_add", ), - path( - "/step/report/", - views.OnboardingReportView.as_view(), - name="step_report", - ), + path("/step/report/", views.OnboardingReportView.as_view(), name="step_report"), path( "step/introduction/registration/", views.OnboardingIntroductionRegistrationView.as_view(), name="step_introduction_registration", ), - path( - "step/organization-setup/", - views.OnboardingOrganizationSetupView.as_view(), - name="step_organization_setup", - ), + path("step/organization-setup/", views.OnboardingOrganizationSetupView.as_view(), name="step_organization_setup"), path( "/step/organization-setup/update/", views.OnboardingOrganizationUpdateView.as_view(), @@ -79,20 +69,14 @@ views.OnboardingIndemnificationSetupView.as_view(), name="step_indemnification_setup", ), - path( - "step/choose-user-type/", - views.OnboardingChooseUserTypeView.as_view(), - name="step_choose_user_type", - ), + path("step/choose-user-type/", views.OnboardingChooseUserTypeView.as_view(), name="step_choose_user_type"), path( "/step/choose-user-type/", views.OnboardingChooseUserTypeView.as_view(), name="step_choose_user_type", ), path( - "/step/complete-onboarding/", - views.CompleteOnboarding.as_view(), - name="complete_onboarding", + "/step/complete-onboarding/", views.CompleteOnboarding.as_view(), name="complete_onboarding" ), path( "/step/account-setup/introduction/", diff --git a/rocky/onboarding/view_helpers.py b/rocky/onboarding/view_helpers.py index 246fa890ed7..78146742e49 100644 --- a/rocky/onboarding/view_helpers.py +++ b/rocky/onboarding/view_helpers.py @@ -47,14 +47,8 @@ class IntroductionRegistrationStepsMixin(StepsMixin): def build_steps(self): steps = [ - { - "text": _("1: Introduction"), - "url": reverse_lazy("step_introduction_registration"), - }, - { - "text": _("2: Organization setup"), - "url": reverse_lazy("step_organization_setup"), - }, + {"text": _("1: Introduction"), "url": reverse_lazy("step_introduction_registration")}, + {"text": _("2: Organization setup"), "url": reverse_lazy("step_organization_setup")}, {"text": _("3: Indemnification"), "url": ""}, {"text": _("4: Account setup"), "url": ""}, ] @@ -66,14 +60,8 @@ class IntroductionAdminStepsMixin(StepsMixin): def build_steps(self): steps = [ - { - "text": _("1: Introduction"), - "url": reverse_lazy("step_introduction_registration"), - }, - { - "text": _("2: Organization setup"), - "url": reverse_lazy("step_organization_setup"), - }, + {"text": _("1: Introduction"), "url": reverse_lazy("step_introduction_registration")}, + {"text": _("2: Organization setup"), "url": reverse_lazy("step_organization_setup")}, { "text": _("3: Indemnification"), "url": reverse_lazy("step_indemnification_setup", kwargs={"organization_code": self.organization.code}), @@ -94,11 +82,9 @@ def build_breadcrumbs(self): { "url": reverse_lazy("step_introduction", kwargs={"organization_code": self.organization.code}), "text": _("OpenKAT introduction"), - }, + } ] class RegistrationBreadcrumbsMixin(BreadcrumbsMixin): - breadcrumbs = [ - {"url": reverse_lazy("step_introduction_registration"), "text": _("OpenKAT Setup")}, - ] + breadcrumbs = [{"url": reverse_lazy("step_introduction_registration"), "text": _("OpenKAT Setup")}] diff --git a/rocky/onboarding/views.py b/rocky/onboarding/views.py index 2aa5f103532..cbf0209d62d 100644 --- a/rocky/onboarding/views.py +++ b/rocky/onboarding/views.py @@ -57,10 +57,7 @@ def get(self, request, *args, **kwargs): class OnboardingIntroductionView( - OrganizationPermissionRequiredMixin, - IntroductionStepsMixin, - OrganizationView, - TemplateView, + OrganizationPermissionRequiredMixin, IntroductionStepsMixin, OrganizationView, TemplateView ): """ 1. Start the onboarding wizard. What is OpenKAT and what it does. @@ -72,10 +69,7 @@ class OnboardingIntroductionView( class OnboardingChooseReportInfoView( - OrganizationPermissionRequiredMixin, - IntroductionStepsMixin, - OrganizationView, - TemplateView, + OrganizationPermissionRequiredMixin, IntroductionStepsMixin, OrganizationView, TemplateView ): """ 2. Introduction into reporting. All the necessities to generate a report. @@ -87,10 +81,7 @@ class OnboardingChooseReportInfoView( class OnboardingChooseReportTypeView( - OrganizationPermissionRequiredMixin, - IntroductionStepsMixin, - OrganizationView, - TemplateView, + OrganizationPermissionRequiredMixin, IntroductionStepsMixin, OrganizationView, TemplateView ): """ 3. Choose a report type. Gives the user a choice of many report types. Ex. DNS report @@ -102,10 +93,7 @@ class OnboardingChooseReportTypeView( class OnboardingSetupScanOOIInfoView( - OrganizationPermissionRequiredMixin, - IntroductionStepsMixin, - OrganizationView, - TemplateView, + OrganizationPermissionRequiredMixin, IntroductionStepsMixin, OrganizationView, TemplateView ): """ 4. Explanation that an object is needed to make scans. @@ -117,10 +105,7 @@ class OnboardingSetupScanOOIInfoView( class OnboardingSetupScanOOIAddView( - OrganizationPermissionRequiredMixin, - IntroductionStepsMixin, - SingleOOITreeMixin, - FormView, + OrganizationPermissionRequiredMixin, IntroductionStepsMixin, SingleOOITreeMixin, FormView ): """ 5. The user will create a URL object. Shows a form and validation to create object. @@ -157,7 +142,7 @@ def build_breadcrumbs(self) -> list[Breadcrumb]: "url": reverse("ooi_add_type_select", kwargs={"organization_code": self.organization.code}) + get_selection(self.request), "text": _("Creating an object"), - }, + } ] @@ -267,10 +252,7 @@ def get_context_data(self, **kwargs): class OnboardingSetupScanSelectPluginsView( - OrganizationPermissionRequiredMixin, - IntroductionStepsMixin, - OrganizationView, - TemplateView, + OrganizationPermissionRequiredMixin, IntroductionStepsMixin, OrganizationView, TemplateView ): """ 9. Shows the user all required and optional plugins to select from. Required plugins are mandatory to continue. @@ -344,10 +326,7 @@ def get_context_data(self, **kwargs): class OnboardingReportView( - OrganizationPermissionRequiredMixin, - IntroductionStepsMixin, - SingleOOITreeMixin, - TemplateView, + OrganizationPermissionRequiredMixin, IntroductionStepsMixin, SingleOOITreeMixin, TemplateView ): """ 10. The user already started the scan and is now waiting till scans are finished to generate the report. @@ -405,11 +384,7 @@ class OnboardingIntroductionRegistrationView(PermissionRequiredMixin, Introducti permission_required = "tools.add_organizationmember" -class OnboardingOrganizationSetupView( - PermissionRequiredMixin, - IntroductionRegistrationStepsMixin, - CreateView, -): +class OnboardingOrganizationSetupView(PermissionRequiredMixin, IntroductionRegistrationStepsMixin, CreateView): """ Step 2: Create a new organization """ @@ -447,10 +422,7 @@ def form_valid(self, form): return result def create_first_member(self, organization): - member = OrganizationMember.objects.create( - user=self.request.user, - organization=organization, - ) + member = OrganizationMember.objects.create(user=self.request.user, organization=organization) if member.user.is_superuser: member.trusted_clearance_level = 4 member.acknowledged_clearance_level = 4 @@ -462,10 +434,7 @@ def add_success_notification(self, org_name): class OnboardingOrganizationUpdateView( - OrganizationPermissionRequiredMixin, - IntroductionAdminStepsMixin, - OrganizationView, - UpdateView, + OrganizationPermissionRequiredMixin, IntroductionAdminStepsMixin, OrganizationView, UpdateView ): """ Step 2: Update an existing organization (only name not code) @@ -493,10 +462,7 @@ def add_success_notification(self, org_name): messages.add_message(self.request, messages.SUCCESS, success_message) -class OnboardingIndemnificationSetupView( - IntroductionAdminStepsMixin, - IndemnificationAddView, -): +class OnboardingIndemnificationSetupView(IntroductionAdminStepsMixin, IndemnificationAddView): """ Step 3: Agree to idemnification to scan oois """ @@ -548,10 +514,7 @@ class OnboardingChooseUserTypeView( permission_required = "tools.add_organizationmember" -class OnboardingAccountSetupAdminView( - RegistrationBreadcrumbsMixin, - OnboardingAccountCreationMixin, -): +class OnboardingAccountSetupAdminView(RegistrationBreadcrumbsMixin, OnboardingAccountCreationMixin): """ Step 1: Create an admin account with admin rights """ @@ -574,10 +537,7 @@ def add_success_notification(self, name): messages.add_message(self.request, messages.SUCCESS, success_message) -class OnboardingAccountSetupRedTeamerView( - RegistrationBreadcrumbsMixin, - OnboardingAccountCreationMixin, -): +class OnboardingAccountSetupRedTeamerView(RegistrationBreadcrumbsMixin, OnboardingAccountCreationMixin): """ Step 2: Create an redteamer account with redteam rights """ diff --git a/rocky/reports/forms.py b/rocky/reports/forms.py index ed34728cf30..556da081e85 100644 --- a/rocky/reports/forms.py +++ b/rocky/reports/forms.py @@ -7,9 +7,7 @@ class OOITypeMultiCheckboxForReportForm(BaseRockyForm): ooi_type = forms.MultipleChoiceField( - label=_("Filter by OOI types"), - required=False, - widget=forms.CheckboxSelectMultiple, + label=_("Filter by OOI types"), required=False, widget=forms.CheckboxSelectMultiple ) def __init__(self, ooi_types: list[str], *args, **kwargs): @@ -19,9 +17,7 @@ def __init__(self, ooi_types: list[str], *args, **kwargs): class ReportTypeMultiselectForm(BaseRockyForm): report_type = forms.MultipleChoiceField( - label=_("Report types"), - required=False, - widget=forms.CheckboxSelectMultiple, + label=_("Report types"), required=False, widget=forms.CheckboxSelectMultiple ) def __init__(self, report_types: set[Report], *args, **kwargs): diff --git a/rocky/reports/report_types/definitions.py b/rocky/reports/report_types/definitions.py index 62acac98b23..03151eef77c 100644 --- a/rocky/reports/report_types/definitions.py +++ b/rocky/reports/report_types/definitions.py @@ -59,8 +59,7 @@ def class_attributes(cls) -> dict[str, Any]: @staticmethod def group_by_source( - query_result: list[tuple[str, OOIType]], - check: Callable[[OOIType], bool] | None = None, + query_result: list[tuple[str, OOIType]], check: Callable[[OOIType], bool] | None = None ) -> dict[str, list[OOIType]]: """Transform a query-many result from [(ref1, obj1), (ref1, obj2), ...] into {ref1: [obj1, obj2], ...}""" @@ -77,8 +76,7 @@ def group_by_source( @staticmethod def group_finding_types_by_source( - query_result: list[tuple[str, OOIType]], - keep_ids: list[str] | None = None, + query_result: list[tuple[str, OOIType]], keep_ids: list[str] | None = None ) -> dict[str, list[OOIType]]: if keep_ids: return Report.group_by_source(query_result, lambda x: x.id in keep_ids) diff --git a/rocky/reports/report_types/findings_report/report.py b/rocky/reports/report_types/findings_report/report.py index 47d75a41b5f..01052dc4570 100644 --- a/rocky/reports/report_types/findings_report/report.py +++ b/rocky/reports/report_types/findings_report/report.py @@ -45,12 +45,7 @@ def generate_data(self, input_ooi: str, valid_time: datetime) -> dict[str, Any]: for finding in findings: try: - finding_type = next( - filter( - lambda x: x.id == finding.finding_type.tokenized.id, - all_finding_types, - ) - ) + finding_type = next(filter(lambda x: x.id == finding.finding_type.tokenized.id, all_finding_types)) except StopIteration: continue @@ -75,16 +70,11 @@ def generate_data(self, input_ooi: str, valid_time: datetime) -> dict[str, Any]: if finding_type.id in finding_types: finding_types[finding_type.id]["occurrences"].append(finding_dict) else: - finding_types[finding_type.id] = { - "finding_type": finding_type, - "occurrences": [finding_dict], - } + finding_types[finding_type.id] = {"finding_type": finding_type, "occurrences": [finding_dict]} total_by_severity_per_finding_type[severity] += 1 sorted_finding_types: list[Any] = sorted( - finding_types.values(), - key=lambda x: x["finding_type"].risk_score or 0, - reverse=True, + finding_types.values(), key=lambda x: x["finding_type"].risk_score or 0, reverse=True ) summary = { diff --git a/rocky/reports/report_types/helpers.py b/rocky/reports/report_types/helpers.py index b7bf2733879..558a487f2e1 100644 --- a/rocky/reports/report_types/helpers.py +++ b/rocky/reports/report_types/helpers.py @@ -85,9 +85,7 @@ def get_plugins_for_report_ids(reports: list[str]) -> dict[str, set[str]]: return {"required": required_boefjes, "optional": optional_boefjes} -def get_report_types_from_aggregate_report( - aggregate_report: type[AggregateReport], -) -> dict[str, set[type[Report]]]: +def get_report_types_from_aggregate_report(aggregate_report: type[AggregateReport]) -> dict[str, set[type[Report]]]: required_reports = set() optional_reports = set() diff --git a/rocky/reports/report_types/multi_organization_report/report.py b/rocky/reports/report_types/multi_organization_report/report.py index 3f236eba780..ac8ba1e0dfc 100644 --- a/rocky/reports/report_types/multi_organization_report/report.py +++ b/rocky/reports/report_types/multi_organization_report/report.py @@ -253,11 +253,7 @@ def post_process_data(self, data: dict[str, Any]) -> dict[str, Any]: } -def collect_report_data( - connector: OctopoesAPIConnector, - input_ooi_references: list[str], - observed_at: datetime, -): +def collect_report_data(connector: OctopoesAPIConnector, input_ooi_references: list[str], observed_at: datetime): report_data = {} for ooi in [x for x in input_ooi_references if Reference.from_str(x).class_type == ReportData]: report_data[ooi] = connector.get(Reference.from_str(ooi), observed_at).dict() diff --git a/rocky/reports/report_types/name_server_report/report.py b/rocky/reports/report_types/name_server_report/report.py index 1b5544da74f..4ac612417c0 100644 --- a/rocky/reports/report_types/name_server_report/report.py +++ b/rocky/reports/report_types/name_server_report/report.py @@ -54,14 +54,7 @@ class NameServerSystemReport(Report): id = "name-server-report" name = _("Name Server Report") description = _("Name server report checks name servers on basic security standards.") - plugins = { - "required": [ - "nmap", - "dns-records", - "dns-sec", - ], - "optional": [], - } + plugins = {"required": ["nmap", "dns-records", "dns-sec"], "optional": []} input_ooi_types = {Hostname, IPAddressV4, IPAddressV6} template_path = "name_server_report/report.html" diff --git a/rocky/reports/report_types/open_ports_report/report.py b/rocky/reports/report_types/open_ports_report/report.py index 628b9799911..80929ff9200 100644 --- a/rocky/reports/report_types/open_ports_report/report.py +++ b/rocky/reports/report_types/open_ports_report/report.py @@ -16,10 +16,7 @@ class OpenPortsReport(Report): id = "open-ports-report" name = _("Open Ports Report") description = _("Find open ports of IP addresses") - plugins = { - "required": ["nmap"], - "optional": ["shodan", "nmap-udp", "nmap-ports", "nmap-ip-range", "masscan"], - } + plugins = {"required": ["nmap"], "optional": ["shodan", "nmap-udp", "nmap-ports", "nmap-ip-range", "masscan"]} input_ooi_types = {Hostname, IPAddressV4, IPAddressV6} template_path = "open_ports_report/report.html" label_style = "5-light" @@ -28,17 +25,17 @@ def collect_data(self, input_oois: Iterable[str], valid_time: datetime) -> dict[ ips_by_input_ooi = self.to_ips(input_oois, valid_time) all_ips = list({ip for key, ips in ips_by_input_ooi.items() for ip in ips}) ports_by_source = self.group_by_source( - self.octopoes_api_connector.query_many("IPAddress. dict[ services[port.port] = [service.name for service in services_by_port.get(port.reference, [])] sorted_port_numbers = dict(sorted(port_numbers.items())) - by_ip[ip] = { - "ports": sorted_port_numbers, - "hostnames": hostnames, - "services": services, - } + by_ip[ip] = {"ports": sorted_port_numbers, "hostnames": hostnames, "services": services} result[input_ooi] = by_ip return result diff --git a/rocky/reports/report_types/rpki_report/report.py b/rocky/reports/report_types/rpki_report/report.py index b8200fa832c..9a54a6b14ce 100644 --- a/rocky/reports/report_types/rpki_report/report.py +++ b/rocky/reports/report_types/rpki_report/report.py @@ -34,11 +34,7 @@ def collect_data(self, input_oois: Iterable[str], valid_time: datetime) -> dict[ ips_by_input_ooi = self.to_ips(input_oois, valid_time) all_ips = list({ip for key, ips in ips_by_input_ooi.items() for ip in ips}) finding_types_by_source = self.group_finding_types_by_source( - self.octopoes_api_connector.query_many( - "IPAddress. dict[ source: [SERVICE_MAPPING.get(str(service.name), SystemType.OTHER) for service in services] for source, services in self.group_by_source( self.octopoes_api_connector.query_many( - "IPAddress. dict[ ] for source, sw_instances in self.group_by_source( self.octopoes_api_connector.query_many( - "IPAddress. dict[ result[input_ooi] = { "input_ooi": input_ooi, "services": ip_services, - "summary": { - "total_systems": len(ip_services), - "total_domains": len(domains), - }, + "summary": {"total_systems": len(ip_services), "total_domains": len(domains)}, } return result diff --git a/rocky/reports/report_types/tls_report/report.py b/rocky/reports/report_types/tls_report/report.py index 4b3c3c61b02..9fe93d76413 100644 --- a/rocky/reports/report_types/tls_report/report.py +++ b/rocky/reports/report_types/tls_report/report.py @@ -11,11 +11,7 @@ logger = getLogger(__name__) -CIPHER_FINDINGS = [ - "KAT-RECOMMENDATION-BAD-CIPHER", - "KAT-MEDIUM-BAD-CIPHER", - "KAT-CRITICAL-BAD-CIPHER", -] +CIPHER_FINDINGS = ["KAT-RECOMMENDATION-BAD-CIPHER", "KAT-MEDIUM-BAD-CIPHER", "KAT-CRITICAL-BAD-CIPHER"] TREE_DEPTH = 3 diff --git a/rocky/reports/report_types/vulnerability_report/report.py b/rocky/reports/report_types/vulnerability_report/report.py index 8b45bd37d84..80451d47e8f 100644 --- a/rocky/reports/report_types/vulnerability_report/report.py +++ b/rocky/reports/report_types/vulnerability_report/report.py @@ -90,21 +90,14 @@ def collect_data(self, input_oois: Iterable[str], valid_time: datetime) -> dict[ str(_("Evidence")): evidence, } - cvss: dict[str, str | float | None] = { - "class": "-", - "score": None, - "risk_level": "-", - } + cvss: dict[str, str | float | None] = {"class": "-", "score": None, "risk_level": "-"} if finding_type.risk_severity: cvss = { "class": str(finding_type.risk_severity.value).lower(), "score": finding_type.risk_score, "risk_level": " ".join( - [ - str(finding_type.risk_severity.value).title(), - str(finding_type.risk_score), - ] + [str(finding_type.risk_severity.value).title(), str(finding_type.risk_score)] ), } @@ -123,9 +116,7 @@ def collect_data(self, input_oois: Iterable[str], valid_time: datetime) -> dict[ recommendations.append(finding_type.recommendation) sorted_vulnerabilities = sorted( - vulnerabilities.items(), - key=lambda x: x[1].get("cvss", {}).get("score", 0) or 0, - reverse=True, + vulnerabilities.items(), key=lambda x: x[1].get("cvss", {}).get("score", 0) or 0, reverse=True ) data[ip] = { @@ -148,9 +139,7 @@ def get_findings(self, input_oois: Iterable[str], valid_time: datetime): port_finding_types = self.group_finding_types_by_source( self.octopoes_api_connector.query_many( - "IPAddress. dict[ query = "Hostname. dict[ query = "Hostname. dict[ # We need cast here because mypy doesn't understand that we only add finding_types # when risk level severity isn't None "finding_types": sorted( - finding_types.values(), - reverse=True, - key=lambda x: cast(RiskLevelSeverity, x.risk_severity), + finding_types.values(), reverse=True, key=lambda x: cast(RiskLevelSeverity, x.risk_severity) ), } diff --git a/rocky/reports/views/aggregate_report.py b/rocky/reports/views/aggregate_report.py index 789f2d34471..95df3fb3d69 100644 --- a/rocky/reports/views/aggregate_report.py +++ b/rocky/reports/views/aggregate_report.py @@ -34,26 +34,14 @@ def build_breadcrumbs(self): kwargs = self.get_kwargs() selection = get_selection(self.request) breadcrumbs += [ - { - "url": reverse("aggregate_report_landing", kwargs=kwargs) + selection, - "text": _("Aggregate report"), - }, - { - "url": reverse("aggregate_report_select_oois", kwargs=kwargs) + selection, - "text": _("Select Objects"), - }, + {"url": reverse("aggregate_report_landing", kwargs=kwargs) + selection, "text": _("Aggregate report")}, + {"url": reverse("aggregate_report_select_oois", kwargs=kwargs) + selection, "text": _("Select Objects")}, { "url": reverse("aggregate_report_select_report_types", kwargs=kwargs) + selection, "text": _("Select report types"), }, - { - "url": reverse("aggregate_report_setup_scan", kwargs=kwargs) + selection, - "text": _("Configuration"), - }, - { - "url": reverse("aggregate_report_view", kwargs=kwargs) + selection, - "text": _("View report"), - }, + {"url": reverse("aggregate_report_setup_scan", kwargs=kwargs) + selection, "text": _("Configuration")}, + {"url": reverse("aggregate_report_view", kwargs=kwargs) + selection, "text": _("View report")}, ] return breadcrumbs @@ -71,10 +59,7 @@ def get(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: class OOISelectionAggregateReportView( - AggregateReportStepsMixin, - BreadcrumbsAggregateReportView, - ReportOOIView, - BaseOOIListView, + AggregateReportStepsMixin, BreadcrumbsAggregateReportView, ReportOOIView, BaseOOIListView ): """ Select Objects for the 'Aggregate Report' flow. @@ -93,11 +78,7 @@ def get_context_data(self, **kwargs): class ReportTypesSelectionAggregateReportView( - AggregateReportStepsMixin, - BreadcrumbsAggregateReportView, - ReportOOIView, - ReportTypeView, - TemplateView, + AggregateReportStepsMixin, BreadcrumbsAggregateReportView, ReportOOIView, ReportTypeView, TemplateView ): """ Shows all possible report types from a list of Objects. @@ -183,10 +164,7 @@ def get(self, request, *args, **kwargs): "organization_code": self.organization.code, "organization_name": self.organization.name, "organization_tags": list(self.organization.tags.all()), - "data": { - "report_data": report_data, - "post_processed_data": post_processed_data, - }, + "data": {"report_data": report_data, "post_processed_data": post_processed_data}, } try: @@ -204,14 +182,9 @@ def get(self, request, *args, **kwargs): return super().get(request, *args, **kwargs) - def generate_reports_for_oois( - self, - ) -> tuple[AggregateOrganisationReport, Any, dict[Any, dict[Any, Any]]]: + def generate_reports_for_oois(self) -> tuple[AggregateOrganisationReport, Any, dict[Any, dict[Any, Any]]]: aggregate_report, post_processed_data, report_data, report_errors = aggregate_reports( - self.octopoes_api_connector, - self.get_oois(), - self.selected_report_types, - self.observed_at, + self.octopoes_api_connector, self.get_oois(), self.selected_report_types, self.observed_at ) # If OOI could not be found or the date is incorrect, it will be shown to the user as a message error @@ -252,6 +225,4 @@ class AggregateReportPDFView(AggregateReportView, WeasyTemplateResponseMixin): pdf_filename = "aggregate_report.pdf" pdf_attachment = False - pdf_options = { - "pdf_variant": "pdf/ua-1", - } + pdf_options = {"pdf_variant": "pdf/ua-1"} diff --git a/rocky/reports/views/base.py b/rocky/reports/views/base.py index c93064e28e4..0b84e63007d 100644 --- a/rocky/reports/views/base.py +++ b/rocky/reports/views/base.py @@ -22,10 +22,7 @@ from rocky.views.mixins import OOIList from rocky.views.ooi_view import OOIFilterView -REPORTS_PRE_SELECTION = { - "clearance_level": ["2", "3", "4"], - "clearance_type": "declared", -} +REPORTS_PRE_SELECTION = {"clearance_level": ["2", "3", "4"], "clearance_type": "declared"} def get_selection(request: HttpRequest, pre_selection: dict[str, str | Sequence[str]] | None = None) -> str: @@ -54,12 +51,7 @@ def build_breadcrumbs(self): kwargs = self.get_kwargs() selection = get_selection(self.request) - breadcrumbs = [ - { - "url": reverse("reports", kwargs=kwargs) + selection, - "text": _("Reports"), - }, - ] + breadcrumbs = [{"url": reverse("reports", kwargs=kwargs) + selection, "text": _("Reports")}] return breadcrumbs @@ -134,8 +126,7 @@ def get_oois(self) -> list[OOI]: def get_ooi_filter_forms(self, ooi_types: Iterable[type[OOI]]) -> dict[str, Form]: return { "ooi_type_form": OOITypeMultiCheckboxForReportForm( - sorted([ooi_class.get_ooi_type() for ooi_class in ooi_types]), - self.request.GET, + sorted([ooi_class.get_ooi_type() for ooi_class in ooi_types]), self.request.GET ) } @@ -263,17 +254,11 @@ def get_plugin_data(self): for plugin in report_plugins: if plugin not in plugin_report_types: plugin_report_types[plugin] = [ - { - "name": report_type.name, - "label_style": report_type.label_style, - } + {"name": report_type.name, "label_style": report_type.label_style} ] else: plugin_report_types[plugin].append( - { - "name": report_type.name, - "label_style": report_type.label_style, - } + {"name": report_type.name, "label_style": report_type.label_style} ) total_enabled_plugins[plugin_type] += number_of_enabled diff --git a/rocky/reports/views/generate_report.py b/rocky/reports/views/generate_report.py index b4e5b14763e..d0a16158f5e 100644 --- a/rocky/reports/views/generate_report.py +++ b/rocky/reports/views/generate_report.py @@ -33,26 +33,14 @@ def build_breadcrumbs(self): kwargs = self.get_kwargs() selection = get_selection(self.request) breadcrumbs += [ - { - "url": reverse("generate_report_landing", kwargs=kwargs) + selection, - "text": _("Generate report"), - }, - { - "url": reverse("generate_report_select_oois", kwargs=kwargs) + selection, - "text": _("Select Objects"), - }, + {"url": reverse("generate_report_landing", kwargs=kwargs) + selection, "text": _("Generate report")}, + {"url": reverse("generate_report_select_oois", kwargs=kwargs) + selection, "text": _("Select Objects")}, { "url": reverse("generate_report_select_report_types", kwargs=kwargs) + selection, "text": _("Select report types"), }, - { - "url": reverse("generate_report_setup_scan", kwargs=kwargs) + selection, - "text": _("Configuration"), - }, - { - "url": reverse("generate_report_view", kwargs=kwargs) + selection, - "text": _("View report"), - }, + {"url": reverse("generate_report_setup_scan", kwargs=kwargs) + selection, "text": _("Configuration")}, + {"url": reverse("generate_report_view", kwargs=kwargs) + selection, "text": _("View report")}, ] return breadcrumbs @@ -70,10 +58,7 @@ def get(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: class OOISelectionGenerateReportView( - GenerateReportStepsMixin, - BreadcrumbsGenerateReportView, - ReportOOIView, - BaseOOIListView, + GenerateReportStepsMixin, BreadcrumbsGenerateReportView, ReportOOIView, BaseOOIListView ): """ Select objects for the 'Generate Report' flow. @@ -92,11 +77,7 @@ def get_context_data(self, **kwargs): class ReportTypesSelectionGenerateReportView( - GenerateReportStepsMixin, - BreadcrumbsGenerateReportView, - ReportOOIView, - ReportTypeView, - TemplateView, + GenerateReportStepsMixin, BreadcrumbsGenerateReportView, ReportOOIView, ReportTypeView, TemplateView ): """ Shows all possible report types from a list of OOIs. @@ -120,10 +101,7 @@ def get_context_data(self, **kwargs): class SetupScanGenerateReportView( - GenerateReportStepsMixin, - BreadcrumbsGenerateReportView, - ReportPluginView, - TemplateView, + GenerateReportStepsMixin, BreadcrumbsGenerateReportView, ReportPluginView, TemplateView ): """ Show required and optional plugins to start scans to generate OOIs to include in report. @@ -231,6 +209,4 @@ class GenerateReportPDFView(GenerateReportView, WeasyTemplateResponseMixin): pdf_filename = "generate_report.pdf" pdf_attachment = False - pdf_options = { - "pdf_variant": "pdf/ua-1", - } + pdf_options = {"pdf_variant": "pdf/ua-1"} diff --git a/rocky/reports/views/multi_report.py b/rocky/reports/views/multi_report.py index 0425c0185c3..215ec09840f 100644 --- a/rocky/reports/views/multi_report.py +++ b/rocky/reports/views/multi_report.py @@ -29,26 +29,14 @@ def build_breadcrumbs(self): kwargs = self.get_kwargs() selection = get_selection(self.request) breadcrumbs += [ - { - "url": reverse("multi_report_landing", kwargs=kwargs) + selection, - "text": _("Multi report"), - }, - { - "url": reverse("multi_report_select_oois", kwargs=kwargs) + selection, - "text": _("Select OOIs"), - }, + {"url": reverse("multi_report_landing", kwargs=kwargs) + selection, "text": _("Multi report")}, + {"url": reverse("multi_report_select_oois", kwargs=kwargs) + selection, "text": _("Select OOIs")}, { "url": reverse("multi_report_select_report_types", kwargs=kwargs) + selection, "text": _("Select report types"), }, - { - "url": reverse("multi_report_setup_scan", kwargs=kwargs) + selection, - "text": _("Configuration"), - }, - { - "url": reverse("multi_report_view", kwargs=kwargs) + selection, - "text": _("View report"), - }, + {"url": reverse("multi_report_setup_scan", kwargs=kwargs) + selection, "text": _("Configuration")}, + {"url": reverse("multi_report_view", kwargs=kwargs) + selection, "text": _("View report")}, ] return breadcrumbs @@ -82,11 +70,7 @@ def get_context_data(self, **kwargs): class ReportTypesSelectionMultiReportView( - MultiReportStepsMixin, - BreadcrumbsMultiReportView, - ReportOOIView, - ReportTypeView, - TemplateView, + MultiReportStepsMixin, BreadcrumbsMultiReportView, ReportOOIView, ReportTypeView, TemplateView ): """ Shows all possible report types from a list of OOIs. @@ -146,9 +130,7 @@ def get_context_data(self, **kwargs): context["template"] = MultiOrganizationReport.template_path context["report_data"] = report_data context["report_download_url"] = url_with_querystring( - reverse("multi_report_pdf", kwargs={"organization_code": self.organization.code}), - True, - **self.request.GET, + reverse("multi_report_pdf", kwargs={"organization_code": self.organization.code}), True, **self.request.GET ) return context @@ -158,6 +140,4 @@ class MultiReportPDFView(MultiReportView, WeasyTemplateResponseMixin): pdf_filename = "multi_report.pdf" pdf_attachment = False - pdf_options = { - "pdf_variant": "pdf/ua-1", - } + pdf_options = {"pdf_variant": "pdf/ua-1"} diff --git a/rocky/rocky/bytes_client.py b/rocky/rocky/bytes_client.py index cb2c2964677..ed5c8110b5a 100644 --- a/rocky/rocky/bytes_client.py +++ b/rocky/rocky/bytes_client.py @@ -16,10 +16,7 @@ class BytesClient: def __init__(self, base_url: str, username: str, password: str, organization: str): - self.credentials = { - "username": username, - "password": password, - } + self.credentials = {"username": username, "password": password} self.session = httpx.Client(base_url=base_url) self.organization = organization @@ -67,7 +64,7 @@ def add_manual_proof( normalizer=Normalizer(id="normalizer/manual"), started_at=datetime.now(timezone.utc), ended_at=datetime.now(timezone.utc), - ), + ) ) def upload_raw(self, raw: bytes, manual_mime_types: set[str], input_ooi: str | None = None): @@ -157,9 +154,7 @@ def _authorization_header(self) -> dict[str, str]: def _get_token(self) -> str: response = self.session.post( - "/token", - data=self.credentials, - headers={"content-type": "application/x-www-form-urlencoded"}, + "/token", data=self.credentials, headers={"content-type": "application/x-www-form-urlencoded"} ) return response.json()["access_token"] diff --git a/rocky/rocky/keiko.py b/rocky/rocky/keiko.py index 944cc4ddd1d..486e8288941 100644 --- a/rocky/rocky/keiko.py +++ b/rocky/rocky/keiko.py @@ -39,14 +39,7 @@ def __init__(self, base_uri: str, timeout: int = 60): def generate_report(self, template: str, data: dict, glossary: str) -> str: try: - res = self.session.post( - "/reports", - json={ - "template": template, - "data": data, - "glossary": glossary, - }, - ) + res = self.session.post("/reports", json={"template": template, "data": data, "glossary": glossary}) res.raise_for_status() except HTTPError as e: raise GeneratingReportFailed from e @@ -139,11 +132,7 @@ def __init__( self.origin = origin def to_dict(self) -> dict: - return { - "observed_at": str(self.observed_at), - "ooi": self.ooi.reference, - "depth": self.depth, - } + return {"observed_at": str(self.observed_at), "ooi": self.ooi.reference, "depth": self.depth} def to_url(self) -> str: translation.activate(self.language) @@ -162,12 +151,7 @@ def __init__(self, keiko_client: KeikoClient): self.keiko_client = keiko_client def get_report( - self, - valid_time: datetime, - source_type: str, - source_value: str, - store: dict, - filters: ReportQuery, + self, valid_time: datetime, source_type: str, source_value: str, store: dict, filters: ReportQuery ) -> BinaryIO: report_data = build_findings_list_from_store(store) # reuse existing dict structure report_data["findings_grouped"] = _ooi_field_as_string(report_data["findings_grouped"], store) @@ -237,10 +221,7 @@ def _ooi_field_as_string(findings_grouped: dict, store: dict): list_of_findings.append({**finding, "ooi": ooi_field}) - new_findings_grouped[finding_type] = { - "list": list_of_findings, - "finding_type": finding_group["finding_type"], - } + new_findings_grouped[finding_type] = {"list": list_of_findings, "finding_type": finding_group["finding_type"]} return new_findings_grouped @@ -264,23 +245,14 @@ def build_findings_list_from_store(ooi_store: dict, finding_filter: list[str] | findings_grouped = {} for finding in findings: if finding["finding_type"]["id"] not in findings_grouped: - findings_grouped[finding["finding_type"]["id"]] = { - "finding_type": finding["finding_type"], - "list": [], - } + findings_grouped[finding["finding_type"]["id"]] = {"finding_type": finding["finding_type"], "list": []} findings_grouped[finding["finding_type"]["id"]]["list"].append(finding) - return { - "meta": build_meta(findings), - "findings_grouped": findings_grouped, - } + return {"meta": build_meta(findings), "findings_grouped": findings_grouped} -def build_finding_dict( - finding_ooi: Finding, - ooi_store: dict[str, OOI], -) -> dict: +def build_finding_dict(finding_ooi: Finding, ooi_store: dict[str, OOI]) -> dict: finding_dict = get_ooi_dict(finding_ooi) finding_type_ooi = ooi_store[finding_ooi.finding_type] diff --git a/rocky/rocky/middleware/auth_required.py b/rocky/rocky/middleware/auth_required.py index 61f6c5f57ac..02e26cdd097 100644 --- a/rocky/rocky/middleware/auth_required.py +++ b/rocky/rocky/middleware/auth_required.py @@ -30,11 +30,7 @@ def middleware(request): f"/{translation.get_language()}/reset/", ] # URLs only excluded from 2fa - excluded_2fa = [ - two_factor_setup_path, - reverse("two_factor:qr"), - reverse("logout"), - ] + excluded_2fa = [two_factor_setup_path, reverse("two_factor:qr"), reverse("logout")] if not request.user.is_authenticated and "authorization" in request.headers: authenticator = TokenAuthentication() diff --git a/rocky/rocky/paginator.py b/rocky/rocky/paginator.py index d78b18c5097..aa6b5fa8bcf 100644 --- a/rocky/rocky/paginator.py +++ b/rocky/rocky/paginator.py @@ -3,11 +3,7 @@ class RockyPaginator(Paginator): - def __init__( - self, - *args, - **kwargs, - ) -> None: + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) if self.orphans != 0: diff --git a/rocky/rocky/scheduler.py b/rocky/rocky/scheduler.py index f9ac40fb633..86e7b7329d7 100644 --- a/rocky/rocky/scheduler.py +++ b/rocky/rocky/scheduler.py @@ -123,11 +123,7 @@ class PaginatedTasksResponse(BaseModel): class LazyTaskList: - def __init__( - self, - scheduler_client: SchedulerClient, - **kwargs, - ): + def __init__(self, scheduler_client: SchedulerClient, **kwargs): self.scheduler_client = scheduler_client self.kwargs = kwargs self._count: int | None = None @@ -135,10 +131,7 @@ def __init__( @property def count(self) -> int: if self._count is None: - self._count = self.scheduler_client.list_tasks( - limit=0, - **self.kwargs, - ).count + self._count = self.scheduler_client.list_tasks(limit=0, **self.kwargs).count return self._count def __len__(self): @@ -154,11 +147,7 @@ def __getitem__(self, key) -> list[Task]: else: raise TypeError("Invalid slice argument type.") - res = self.scheduler_client.list_tasks( - limit=limit, - offset=offset, - **self.kwargs, - ) + res = self.scheduler_client.list_tasks(limit=limit, offset=offset, **self.kwargs) self._count = res.count return res.results @@ -191,10 +180,7 @@ class SchedulerClient: def __init__(self, base_uri: str): self._client = httpx.Client(base_url=base_uri) - def list_tasks( - self, - **kwargs, - ) -> PaginatedTasksResponse: + def list_tasks(self, **kwargs) -> PaginatedTasksResponse: kwargs = {k: v for k, v in kwargs.items() if v is not None} # filter Nones from kwargs res = self._client.get("/tasks", params=kwargs) return PaginatedTasksResponse.model_validate_json(res.content) diff --git a/rocky/rocky/settings.py b/rocky/rocky/settings.py index 05eb9799fab..bc40655c46a 100644 --- a/rocky/rocky/settings.py +++ b/rocky/rocky/settings.py @@ -55,15 +55,8 @@ LOGGING = { "version": 1, "disable_existing_loggers": False, - "handlers": { - "console": { - "class": "logging.StreamHandler", - }, - }, - "root": { - "handlers": ["console"], - "level": "WARNING", - }, + "handlers": {"console": {"class": "logging.StreamHandler"}}, + "root": {"handlers": ["console"], "level": "WARNING"}, } # Make sure this header can never be set by an attacker, see also the security @@ -75,13 +68,9 @@ # Optional list of default organizations to add remote users to, # format: space separated list of ORGANIZATION_CODE:GROUP_NAME, e.g. `test:admin test2:redteam` REMOTE_USER_DEFAULT_ORGANIZATIONS = env.list("REMOTE_USER_DEFAULT_ORGANIZATIONS", default=[]) - AUTHENTICATION_BACKENDS = [ - "rocky.auth.remote_user.RemoteUserBackend", - ] + AUTHENTICATION_BACKENDS = ["rocky.auth.remote_user.RemoteUserBackend"] if REMOTE_USER_FALLBACK: - AUTHENTICATION_BACKENDS += [ - "django.contrib.auth.backends.ModelBackend", - ] + AUTHENTICATION_BACKENDS += ["django.contrib.auth.backends.ModelBackend"] # SECURITY WARNING: enable two factor authentication in production! TWOFACTOR_ENABLED = env.bool("TWOFACTOR_ENABLED", not REMOTE_USER_HEADER) @@ -200,7 +189,7 @@ ], "builtins": ["tools.templatetags.ooi_extra"], }, - }, + } ] FORM_RENDERER = "django.forms.renderers.TemplatesSetting" @@ -241,9 +230,7 @@ AUTH_PASSWORD_VALIDATORS = [ { "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", - "OPTIONS": { - "min_length": env.int("PASSWORD_MIN_LENGTH", 12), - }, + "OPTIONS": {"min_length": env.int("PASSWORD_MIN_LENGTH", 12)}, }, { "NAME": "django_password_validators.password_character_requirements" @@ -283,18 +270,11 @@ LANG_INFO.update(EXTRA_LANG_INFO) locale.LANG_INFO = LANG_INFO -LANGUAGES = [ - ("en", "en"), - ("nl", "nl"), - ("pap", "pap"), - ("it", "it"), -] +LANGUAGES = [("en", "en"), ("nl", "nl"), ("pap", "pap"), ("it", "it")] if env.bool("PIRATE", False): LANGUAGE_CODE = "en@pirate" - LANGUAGES += [ - ("en@pirate", "en@pirate"), - ] + LANGUAGES += [("en@pirate", "en@pirate")] # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/4.2/howto/static-files/ @@ -311,11 +291,7 @@ COMPRESS_OFFLINE = True COMPRESS_STORAGE = "compressor.storage.BrotliCompressorFileStorage" -STORAGES = { - "staticfiles": { - "BACKEND": "rocky.storage.RockyStaticFilesStorage", - }, -} +STORAGES = {"staticfiles": {"BACKEND": "rocky.storage.RockyStaticFilesStorage"}} _IMMUTABLE_FILE_TEST_PATTERN = re.compile(r"^.+\.[0-9a-f]{12}\..+$") @@ -417,7 +393,7 @@ def immutable_file_test(path, url): "DEFAULT_PERMISSION_CLASSES": [ # For now this will provide a safe default, but non-admin users will # need to be able to use the API in the future.. - "rest_framework.permissions.IsAdminUser", + "rest_framework.permissions.IsAdminUser" ], "DEFAULT_RENDERER_CLASSES": DEFAULT_RENDERER_CLASSES, "EXCEPTION_HANDLER": "drf_standardized_errors.handler.exception_handler", @@ -452,12 +428,7 @@ def immutable_file_test(path, url): ("color-6-dark", _("Violet dark")), ] -TAG_BORDER_TYPES = [ - ("plain", _("Plain")), - ("solid", _("Solid")), - ("dashed", _("Dashed")), - ("dotted", _("Dotted")), -] +TAG_BORDER_TYPES = [("plain", _("Plain")), ("solid", _("Solid")), ("dashed", _("Dashed")), ("dotted", _("Dotted"))] WEASYPRINT_BASEURL = env("WEASYPRINT_BASEURL", default="http://127.0.0.1:8000/") diff --git a/rocky/rocky/settings_test.py b/rocky/rocky/settings_test.py index e57e699a6a1..c6410197d8d 100644 --- a/rocky/rocky/settings_test.py +++ b/rocky/rocky/settings_test.py @@ -1,8 +1,4 @@ from rocky.settings import * # noqa: F401, F403, TID251 -STORAGES = { - "staticfiles": { - "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", - }, -} +STORAGES = {"staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}} COMPRESS_OFFLINE = False diff --git a/rocky/rocky/urls.py b/rocky/rocky/urls.py index f5e17a8ba65..9453c744a7d 100644 --- a/rocky/rocky/urls.py +++ b/rocky/rocky/urls.py @@ -56,10 +56,7 @@ path("api/v1/", include(router.urls)), path("/health/", Health.as_view(), name="health"), path("", include(tf_urls)), - path( - "robots.txt", - TemplateView.as_view(template_name="robots.txt", content_type="text/plain"), - ), + path("robots.txt", TemplateView.as_view(template_name="robots.txt", content_type="text/plain")), ] urlpatterns += i18n_patterns( path("", include("account.urls"), name="account"), @@ -67,15 +64,9 @@ path("", LandingPageView.as_view(), name="landing_page"), path("onboarding/", include("onboarding.urls"), name="onboarding"), path("crisis-room/", include("crisis_room.urls"), name="crisis_room"), + path("privacy-statement/", PrivacyStatementView.as_view(), name="privacy_statement"), path( - "privacy-statement/", - PrivacyStatementView.as_view(), - name="privacy_statement", - ), - path( - "/settings/indemnifications/", - IndemnificationAddView.as_view(), - name="indemnification_add", + "/settings/indemnifications/", IndemnificationAddView.as_view(), name="indemnification_add" ), path("/findings/", FindingListView.as_view(), name="finding_list"), path("/findings/add/", FindingAddView.as_view(), name="finding_add"), @@ -90,16 +81,8 @@ path("/objects/tree/", OOITreeView.as_view(), name="ooi_tree"), path("/objects/findings/", OOIFindingListView.as_view(), name="ooi_findings"), path("organizations/", OrganizationListView.as_view(), name="organization_list"), - path( - "organizations/add/", - OrganizationAddView.as_view(), - name="organization_add", - ), - path( - "/settings/edit/", - OrganizationEditView.as_view(), - name="organization_edit", - ), + path("organizations/add/", OrganizationAddView.as_view(), name="organization_add"), + path("/settings/edit/", OrganizationEditView.as_view(), name="organization_edit"), path( "/members/add/", OrganizationMemberAddAccountTypeView.as_view(), @@ -115,84 +98,40 @@ DownloadMembersTemplateView.as_view(), name="download_organization_member_template", ), - path( - "/members/upload/", - MembersUploadView.as_view(), - name="organization_member_upload", - ), - path( - "/", - OrganizationCrisisRoomView.as_view(), - name="organization_crisis_room", - ), - path( - "/settings", - OrganizationSettingsView.as_view(), - name="organization_settings", - ), - path( - "/members", - OrganizationMemberListView.as_view(), - name="organization_member_list", - ), + path("/members/upload/", MembersUploadView.as_view(), name="organization_member_upload"), + path("/", OrganizationCrisisRoomView.as_view(), name="organization_crisis_room"), + path("/settings", OrganizationSettingsView.as_view(), name="organization_settings"), + path("/members", OrganizationMemberListView.as_view(), name="organization_member_list"), path( "/members/edit//", OrganizationMemberEditView.as_view(), name="organization_member_edit", ), - path( - "/health/v1/", - HealthChecks.as_view(), - name="health_beautified", - ), + path("/health/v1/", HealthChecks.as_view(), name="health_beautified"), path("/objects/", OOIListView.as_view(), name="ooi_list"), path("/objects/add/", OOIAddTypeSelectView.as_view(), name="ooi_add_type_select"), - path( - "/objects/add-related/", - OOIRelatedObjectAddView.as_view(), - name="ooi_add_related", - ), + path("/objects/add-related/", OOIRelatedObjectAddView.as_view(), name="ooi_add_related"), path("/objects/add//", OOIAddView.as_view(), name="ooi_add"), path("/objects/edit/", OOIEditView.as_view(), name="ooi_edit"), path("/objects/delete/", OOIDeleteView.as_view(), name="ooi_delete"), path("/objects/detail/", OOIDetailView.as_view(), name="ooi_detail"), path("/objects/export", OOIListExportView.as_view(), name="ooi_list_export"), path( - "/objects/indemnification/reset/", - ScanProfileResetView.as_view(), - name="scan_profile_reset", - ), - path( - "/objects/scan-profile/", - ScanProfileDetailView.as_view(), - name="scan_profile_detail", + "/objects/indemnification/reset/", ScanProfileResetView.as_view(), name="scan_profile_reset" ), + path("/objects/scan-profile/", ScanProfileDetailView.as_view(), name="scan_profile_detail"), path("/objects/scans/", ScanListView.as_view(), name="scan_list"), path("/objects/upload/csv/", UploadCSV.as_view(), name="upload_csv"), path("/objects/upload/raw/", UploadRaw.as_view(), name="upload_raw"), path("/objects/upload/raw/", UploadRaw.as_view(), name="upload_raw_typed"), path("/tasks/", BoefjesTaskListView.as_view(), name="task_list"), path("/tasks/boefjes", BoefjesTaskListView.as_view(), name="boefjes_task_list"), + path("/tasks/boefjes/", BoefjeTaskDetailView.as_view(), name="boefje_task_view"), + path("/tasks/normalizers", NormalizersTaskListView.as_view(), name="normalizers_task_list"), path( - "/tasks/boefjes/", - BoefjeTaskDetailView.as_view(), - name="boefje_task_view", - ), - path( - "/tasks/normalizers", - NormalizersTaskListView.as_view(), - name="normalizers_task_list", - ), - path( - "/tasks/normalizers/", - NormalizerTaskJSONView.as_view(), - name="normalizer_task_view", - ), - path( - "/tasks//download/", - DownloadTaskDetail.as_view(), - name="download_task_meta", + "/tasks/normalizers/", NormalizerTaskJSONView.as_view(), name="normalizer_task_view" ), + path("/tasks//download/", DownloadTaskDetail.as_view(), name="download_task_meta"), path("/bytes//raw", BytesRawView.as_view(), name="bytes_raw"), path("/kat-alogus/", include("katalogus.urls"), name="katalogus"), path("/reports/", include("reports.urls"), name="reports"), diff --git a/rocky/rocky/views/bytes_raw.py b/rocky/rocky/views/bytes_raw.py index 3ddbadaf90b..331010553c5 100644 --- a/rocky/rocky/views/bytes_raw.py +++ b/rocky/rocky/views/bytes_raw.py @@ -21,10 +21,7 @@ def get(self, request, **kwargs): raw_metas = self.bytes_client.get_raw_metas(boefje_meta_id, self.organization.code) raws = {raw_meta["id"]: self.bytes_client.get_raw(raw_meta["id"]) for raw_meta in raw_metas} - return FileResponse( - zip_data(raws, raw_metas), - filename=f"{boefje_meta_id}.zip", - ) + return FileResponse(zip_data(raws, raw_metas), filename=f"{boefje_meta_id}.zip") except Http404 as e: msg = _("Getting raw data failed.") logger.error(msg) diff --git a/rocky/rocky/views/finding_add.py b/rocky/rocky/views/finding_add.py index e165fd50bb3..40c313c3033 100644 --- a/rocky/rocky/views/finding_add.py +++ b/rocky/rocky/views/finding_add.py @@ -33,9 +33,7 @@ } -def get_finding_type_from_id( - finding_type_id: str, -) -> FindingType: +def get_finding_type_from_id(finding_type_id: str) -> FindingType: finding_type_id = finding_type_id.upper() prefix = finding_type_id.upper().split("-")[0] @@ -67,10 +65,7 @@ def get_context_data(self, **kwargs): return context def get_form_kwargs(self): - kwargs = { - "connector": self.octopoes_api_connector, - "ooi_list": self.get_ooi_options(), - } + kwargs = {"connector": self.octopoes_api_connector, "ooi_list": self.get_ooi_options()} kwargs.update(super().get_form_kwargs()) if "ooi_class" in kwargs: diff --git a/rocky/rocky/views/finding_list.py b/rocky/rocky/views/finding_list.py index 042fd67a250..32ff9c831a1 100644 --- a/rocky/rocky/views/finding_list.py +++ b/rocky/rocky/views/finding_list.py @@ -27,8 +27,7 @@ def sort_by_severity_desc(findings) -> list[dict[str, Any]]: def generate_findings_metadata( - findings: FindingList, - severity_filter: Iterable[RiskLevelSeverity] | None = None, + findings: FindingList, severity_filter: Iterable[RiskLevelSeverity] | None = None ) -> list[dict[str, Any]]: findings_meta = [] diff --git a/rocky/rocky/views/handler403.py b/rocky/rocky/views/handler403.py index 0007064e09d..b8b0001fa91 100644 --- a/rocky/rocky/views/handler403.py +++ b/rocky/rocky/views/handler403.py @@ -2,10 +2,6 @@ def handler403(request, exception): - context = { - "breadcrumbs": [ - {"url": "", "text": "Error code 403"}, - ], - } + context = {"breadcrumbs": [{"url": "", "text": "Error code 403"}]} return render(request, "403.html", context, status=403) diff --git a/rocky/rocky/views/handler404.py b/rocky/rocky/views/handler404.py index 7ea45cf88ff..516f0c473cd 100644 --- a/rocky/rocky/views/handler404.py +++ b/rocky/rocky/views/handler404.py @@ -2,10 +2,6 @@ def handler404(request, exception): - context = { - "breadcrumbs": [ - {"url": "", "text": "Error code 404"}, - ], - } + context = {"breadcrumbs": [{"url": "", "text": "Error code 404"}]} return render(request, "404.html", context, status=404) diff --git a/rocky/rocky/views/health.py b/rocky/rocky/views/health.py index 222df344813..59e575020f2 100644 --- a/rocky/rocky/views/health.py +++ b/rocky/rocky/views/health.py @@ -31,9 +31,7 @@ def get_bytes_health() -> ServiceHealth: except HTTPError: logger.exception("Error while retrieving Bytes health state") bytes_health = ServiceHealth( - service="bytes", - healthy=False, - additional="Could not connect to Bytes. Service is possibly down", + service="bytes", healthy=False, additional="Could not connect to Bytes. Service is possibly down" ) return bytes_health @@ -45,9 +43,7 @@ def get_octopoes_health(octopoes_api_connector: OctopoesAPIConnector) -> Service except HTTPError: logger.exception("Error while retrieving Octopoes health state") octopoes_health = ServiceHealth( - service="octopoes", - healthy=False, - additional="Could not connect to Octopoes. Service is possibly down", + service="octopoes", healthy=False, additional="Could not connect to Octopoes. Service is possibly down" ) return octopoes_health @@ -58,9 +54,7 @@ def get_scheduler_health() -> ServiceHealth: except HTTPError: logger.exception("Error while retrieving Scheduler health state") scheduler_health = ServiceHealth( - service="scheduler", - healthy=False, - additional="Could not connect to Scheduler. Service is possibly down", + service="scheduler", healthy=False, additional="Could not connect to Scheduler. Service is possibly down" ) return scheduler_health @@ -71,9 +65,7 @@ def get_keiko_health() -> ServiceHealth: except HTTPError: logger.exception("Error while retrieving Keiko health state") return ServiceHealth( - service="keiko", - healthy=False, - additional="Could not connect to Keiko. Service is possibly down", + service="keiko", healthy=False, additional="Could not connect to Keiko. Service is possibly down" ) @@ -91,11 +83,7 @@ def get_rocky_health(octopoes_api_connector: OctopoesAPIConnector) -> ServiceHea if not services_healthy: additional = "Rocky will not function properly. Not all services are healthy." rocky_health = ServiceHealth( - service="rocky", - healthy=services_healthy, - version=__version__, - results=services, - additional=additional, + service="rocky", healthy=services_healthy, version=__version__, results=services, additional=additional ) return rocky_health diff --git a/rocky/rocky/views/indemnification_add.py b/rocky/rocky/views/indemnification_add.py index ebd1313086c..4ed8247e379 100644 --- a/rocky/rocky/views/indemnification_add.py +++ b/rocky/rocky/views/indemnification_add.py @@ -14,10 +14,7 @@ class IndemnificationAddView(OrganizationPermissionRequiredMixin, OrganizationVi permission_required = "tools.add_indemnification" def post(self, request, *args, **kwargs): - Indemnification.objects.get_or_create( - user=self.request.user, - organization=self.organization, - ) + Indemnification.objects.get_or_create(user=self.request.user, organization=self.organization) self.add_success_notification() return super().post(request, *args, **kwargs) @@ -39,10 +36,7 @@ def get_context_data(self, **kwargs): "text": "Settings", }, { - "url": reverse( - "indemnification_add", - kwargs={"organization_code": self.organization.code}, - ), + "url": reverse("indemnification_add", kwargs={"organization_code": self.organization.code}), "text": _("Add indemnification"), }, ] diff --git a/rocky/rocky/views/mixins.py b/rocky/rocky/views/mixins.py index 17f80b0a353..b41e0a20114 100644 --- a/rocky/rocky/views/mixins.py +++ b/rocky/rocky/views/mixins.py @@ -61,10 +61,7 @@ def observed_at(self) -> datetime: datetime_format = "%Y-%m-%d" date_time = convert_date_to_datetime(datetime.strptime(observed_at, datetime_format)) if date_time.date() > datetime.now(timezone.utc).date(): - messages.warning( - self.request, - _("The selected date is in the future."), - ) + messages.warning(self.request, _("The selected date is in the future.")) return date_time except ValueError: try: @@ -90,9 +87,7 @@ def get_single_ooi(self, pk: str) -> OOI: return ooi def get_origins( - self, - reference: Reference, - organization: Organization, + self, reference: Reference, organization: Organization ) -> tuple[list[OriginData], list[OriginData], list[OriginData]]: try: origins = self.octopoes_api_connector.list_origins(self.observed_at, result=reference) @@ -287,10 +282,7 @@ def get_ooi(self, pk: str | None = None) -> OOI: return self.get_single_ooi(pk) def get_breadcrumb_list(self): - start = { - "url": reverse("ooi_list", kwargs={"organization_code": self.organization.code}), - "text": "Objects", - } + start = {"url": reverse("ooi_list", kwargs={"organization_code": self.organization.code}), "text": "Objects"} if isinstance(self.ooi, Finding): start = { "url": reverse("finding_list", kwargs={"organization_code": self.organization.code}), diff --git a/rocky/rocky/views/ooi_detail.py b/rocky/rocky/views/ooi_detail.py index 3052b3e1637..c4529efdcb5 100644 --- a/rocky/rocky/views/ooi_detail.py +++ b/rocky/rocky/views/ooi_detail.py @@ -33,12 +33,7 @@ class PageActions(Enum): CHANGE_CLEARANCE_LEVEL = "change_clearance_level" -class OOIDetailView( - BoefjeMixin, - OOIRelatedObjectAddView, - OOIFindingManager, - BaseOOIDetailView, -): +class OOIDetailView(BoefjeMixin, OOIRelatedObjectAddView, OOIFindingManager, BaseOOIDetailView): template_name = "oois/ooi_detail.html" connector_form_class = ObservedAtForm task_history_limit = 10 @@ -172,8 +167,7 @@ def get_context_data(self, **kwargs): declarations, observations, inferences = self.get_origins(self.ooi.reference, self.organization) inference_params = self.octopoes_api_connector.list_origin_parameters( - {inference.origin.id for inference in inferences}, - self.observed_at, + {inference.origin.id for inference in inferences}, self.observed_at ) inference_params_per_inference = defaultdict(list) for inference_param in inference_params: diff --git a/rocky/rocky/views/ooi_detail_related_object.py b/rocky/rocky/views/ooi_detail_related_object.py index e34eeabc36b..ac8a042be17 100644 --- a/rocky/rocky/views/ooi_detail_related_object.py +++ b/rocky/rocky/views/ooi_detail_related_object.py @@ -81,10 +81,7 @@ def get(self, request, *args, **kwargs): def split_ooi_type_choice(self, ooi_type_choice) -> dict[str, str]: ooi_type = ooi_type_choice.split("|", 1) - return { - "ooi_type": ooi_type[0], - "ooi_relation": ooi_type[1] if len(ooi_type) > 1 else None, - } + return {"ooi_type": ooi_type[0], "ooi_relation": ooi_type[1] if len(ooi_type) > 1 else None} def ooi_add_url(self, ooi: OOI, ooi_type: str, ooi_relation: str = "ooi_id") -> str: """ @@ -138,12 +135,7 @@ def get_ooi_types_input_values(self, ooi: OOI) -> list[dict[str, str]]: if relation == "ooi": text = ooi_type - input_values.append( - { - "text": text, - "value": value, - } - ) + input_values.append({"text": text, "value": value}) return input_values diff --git a/rocky/rocky/views/ooi_edit.py b/rocky/rocky/views/ooi_edit.py index a9ebd586a20..743b27eaab0 100644 --- a/rocky/rocky/views/ooi_edit.py +++ b/rocky/rocky/views/ooi_edit.py @@ -36,10 +36,7 @@ def get_context_data(self, **kwargs): # Construct breadcrumbs breadcrumb_list = self.get_breadcrumb_list() breadcrumb_list.append( - { - "url": get_ooi_url("ooi_edit", self.ooi.primary_key, self.organization.code), - "text": _("Edit"), - } + {"url": get_ooi_url("ooi_edit", self.ooi.primary_key, self.organization.code), "text": _("Edit")} ) context["type"] = self.ooi_class.get_ooi_type() diff --git a/rocky/rocky/views/ooi_list.py b/rocky/rocky/views/ooi_list.py index 0c27d65fc7e..8e04ec06fc0 100644 --- a/rocky/rocky/views/ooi_list.py +++ b/rocky/rocky/views/ooi_list.py @@ -50,7 +50,7 @@ def get_context_data(self, **kwargs): context["scan_levels"] = [alias for _, alias in CUSTOM_SCAN_LEVEL.choices] context["organization_indemnification"] = self.get_organization_indemnification context["breadcrumbs"] = [ - {"url": reverse("ooi_list", kwargs={"organization_code": self.organization.code}), "text": _("Objects")}, + {"url": reverse("ooi_list", kwargs={"organization_code": self.organization.code}), "text": _("Objects")} ] return context @@ -96,10 +96,7 @@ def _set_scan_profiles( self.request, messages.ERROR, _("Could not raise clearance levels to L%s. Indemnification not present at organization %s.") - % ( - level, - self.organization.name, - ), + % (level, self.organization.name), ) return self.get(request, status=403, *args, **kwargs) except TrustedClearanceLevelTooLowException: @@ -111,10 +108,7 @@ def _set_scan_profiles( "You were trusted a clearance level of L%s. " "Contact your administrator to receive a higher clearance." ) - % ( - level, - self.organization_member.trusted_clearance_level, - ), + % (level, self.organization_member.trusted_clearance_level), ) return self.get(request, status=403, *args, **kwargs) except AcknowledgedClearanceLevelTooLowException: @@ -126,10 +120,7 @@ def _set_scan_profiles( "You acknowledged a clearance level of L%s. " "Please accept the clearance level below to proceed." ) - % ( - level, - self.organization_member.acknowledged_clearance_level, - ), + % (level, self.organization_member.acknowledged_clearance_level), ) return redirect(reverse("account_detail", kwargs={"organization_code": self.organization.code})) @@ -161,9 +152,7 @@ def _set_oois_to_inherit( self.octopoes_api_connector.save_many_scan_profiles(scan_profiles, valid_time=datetime.now(timezone.utc)) except (HTTPError, RemoteException, ConnectionError): messages.add_message( - request, - messages.ERROR, - _("An error occurred while setting clearance levels to inherit."), + request, messages.ERROR, _("An error occurred while setting clearance levels to inherit.") ) return self.get(request, status=500, *args, **kwargs) except ObjectNotFoundException: @@ -175,9 +164,7 @@ def _set_oois_to_inherit( return self.get(request, status=404, *args, **kwargs) messages.add_message( - request, - messages.SUCCESS, - _("Successfully set %d ooi(s) clearance level to inherit.") % len(selected_oois), + request, messages.SUCCESS, _("Successfully set %d ooi(s) clearance level to inherit.") % len(selected_oois) ) return self.get(request, *args, **kwargs) @@ -216,21 +203,10 @@ def get(self, request, *args, **kwargs): queryset = self.get_queryset() ooi_list = queryset[: OOIList.HARD_LIMIT] - exports = [ - { - "observed_at": str(self.observed_at), - "filters": str(filters), - } - ] + exports = [{"observed_at": str(self.observed_at), "filters": str(filters)}] for ooi in ooi_list: - exports.append( - { - "key": ooi.primary_key, - "name": ooi.human_readable, - "ooi_type": ooi.ooi_type, - } - ) + exports.append({"key": ooi.primary_key, "name": ooi.human_readable, "ooi_type": ooi.ooi_type}) if file_type == "json": response = HttpResponse( @@ -252,13 +228,7 @@ def get(self, request, *args, **kwargs): writer.writerow([str(self.observed_at), str(filters)]) writer.writerow(["key", "name", "ooi_type"]) for ooi in ooi_list: - writer.writerow( - [ - ooi.primary_key, - ooi.human_readable, - ooi.ooi_type, - ] - ) + writer.writerow([ooi.primary_key, ooi.human_readable, ooi.ooi_type]) return response diff --git a/rocky/rocky/views/ooi_report.py b/rocky/rocky/views/ooi_report.py index 9496d17bc76..6779c315e16 100644 --- a/rocky/rocky/views/ooi_report.py +++ b/rocky/rocky/views/ooi_report.py @@ -43,10 +43,7 @@ class OOIReportView(BaseOOIDetailView): def get(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: if self.observed_at > convert_date_to_datetime(datetime.now(timezone.utc)): - messages.error( - request, - _("You can't generate a report for an OOI on a date in the future."), - ) + messages.error(request, _("You can't generate a report for an OOI on a date in the future.")) return super().get(request, *args, **kwargs) def get_context_data(self, **kwargs): @@ -143,9 +140,7 @@ def get(self, request, *args, **kwargs): return redirect(reverse("finding_list", kwargs={"organization_code": self.organization.code})) return FileResponse( - report, - as_attachment=True, - filename=ReportsService.organization_report_file_name(self.organization.code), + report, as_attachment=True, filename=ReportsService.organization_report_file_name(self.organization.code) ) @@ -194,15 +189,7 @@ def add_boefje(cls, boefje, required): class DNSReport(Report): boefjes_required = {"dns-records", "dns-zone"} boefjes_optional = {"dns-sec"} - allowed_ooi_types = [ - DNSARecord, - DNSAAAARecord, - DNSMXRecord, - DNSNSRecord, - DNSSOARecord, - DNSCAARecord, - Hostname, - ] + allowed_ooi_types = [DNSARecord, DNSAAAARecord, DNSMXRecord, DNSNSRecord, DNSSOARecord, DNSCAARecord, Hostname] allowed_finding_types = [ "KAT-WEBSERVER-NO-IPV6", "KAT-NAMESERVER-NO-IPV6", diff --git a/rocky/rocky/views/ooi_tree.py b/rocky/rocky/views/ooi_tree.py index 032abe8aa44..ecd421816c8 100644 --- a/rocky/rocky/views/ooi_tree.py +++ b/rocky/rocky/views/ooi_tree.py @@ -51,10 +51,7 @@ class OOISummaryView(OOITreeView): template_name = "oois/ooi_summary.html" def get_last_breadcrumb(self): - return { - "url": get_ooi_url("ooi_summary", self.ooi.primary_key, self.organization.code), - "text": _("Summary"), - } + return {"url": get_ooi_url("ooi_summary", self.ooi.primary_key, self.organization.code), "text": _("Summary")} class OOIGraphView(OOITreeView): diff --git a/rocky/rocky/views/ooi_view.py b/rocky/rocky/views/ooi_view.py index d48205d9d91..9281ae195c8 100644 --- a/rocky/rocky/views/ooi_view.py +++ b/rocky/rocky/views/ooi_view.py @@ -154,10 +154,7 @@ def get_form(self, form_class=None) -> BaseRockyForm: return form def get_form_kwargs(self): - kwargs = { - "ooi_class": self.get_ooi_class(), - "connector": self.octopoes_api_connector, - } + kwargs = {"ooi_class": self.get_ooi_class(), "connector": self.octopoes_api_connector} kwargs.update(super().get_form_kwargs()) return kwargs diff --git a/rocky/rocky/views/organization_edit.py b/rocky/rocky/views/organization_edit.py index 2c7100792ab..afbc9cd1e3b 100644 --- a/rocky/rocky/views/organization_edit.py +++ b/rocky/rocky/views/organization_edit.py @@ -18,9 +18,7 @@ def get_object(self): def get_success_url(self): messages.add_message( - self.request, - messages.SUCCESS, - _("Organization %s successfully updated.") % (self.object.name), + self.request, messages.SUCCESS, _("Organization %s successfully updated.") % (self.object.name) ) return reverse("organization_settings", kwargs={"organization_code": self.object.code}) @@ -33,10 +31,7 @@ def get_context_data(self, **kwargs): "url": reverse("organization_settings", kwargs={"organization_code": self.object.code}), "text": self.object.name, }, - { - "url": reverse("organization_edit", kwargs={"organization_code": self.object.code}), - "text": _("Edit"), - }, + {"url": reverse("organization_edit", kwargs={"organization_code": self.object.code}), "text": _("Edit")}, ] return context diff --git a/rocky/rocky/views/organization_list.py b/rocky/rocky/views/organization_list.py index 6bf12f1956a..a73358b9677 100644 --- a/rocky/rocky/views/organization_list.py +++ b/rocky/rocky/views/organization_list.py @@ -5,10 +5,7 @@ from tools.view_helpers import OrganizationBreadcrumbsMixin -class OrganizationListView( - OrganizationBreadcrumbsMixin, - ListView, -): +class OrganizationListView(OrganizationBreadcrumbsMixin, ListView): template_name = "organizations/organization_list.html" def get_queryset(self) -> list[Organization]: diff --git a/rocky/rocky/views/organization_member_add.py b/rocky/rocky/views/organization_member_add.py index f4af5b320ef..4a19bc193b5 100644 --- a/rocky/rocky/views/organization_member_add.py +++ b/rocky/rocky/views/organization_member_add.py @@ -70,11 +70,10 @@ def build_breadcrumbs(self): breadcrumbs.append( { "url": reverse( - "organization_member_add_account_type", - kwargs={"organization_code": self.organization.code}, + "organization_member_add_account_type", kwargs={"organization_code": self.organization.code} ), "text": _("Add Account Type"), - }, + } ) return breadcrumbs @@ -116,8 +115,7 @@ def build_breadcrumbs(self): [ { "url": reverse( - "organization_member_add_account_type", - kwargs={"organization_code": self.organization.code}, + "organization_member_add_account_type", kwargs={"organization_code": self.organization.code} ), "text": _("Add Account Type"), }, @@ -193,16 +191,12 @@ def process_csv(self, form) -> None: logger.exception("Invalid group") except ValidationError: messages.add_message( - self.request, - messages.WARNING, - _("Invalid data for: '{email}'").format(email=email), + self.request, messages.WARNING, _("Invalid data for: '{email}'").format(email=email) ) logger.warning("Invalid data", exc_info=True) except ValueError: messages.add_message( - self.request, - messages.WARNING, - _("Invalid email address: '{email}'").format(email=email), + self.request, messages.WARNING, _("Invalid email address: '{email}'").format(email=email) ) logger.warning("Invalid email address: %s", email) @@ -227,9 +221,7 @@ def save_models( OrganizationMember(user=user, **member_kwargs).full_clean() # Do validation before saving the model member, member_created = OrganizationMember.objects.get_or_create( - user=user, - organization=self.organization, - defaults=member_kwargs, + user=user, organization=self.organization, defaults=member_kwargs ) member.groups.add(Group.objects.get(name=account_type)) diff --git a/rocky/rocky/views/organization_member_edit.py b/rocky/rocky/views/organization_member_edit.py index 3aba9498730..ed30cf559dd 100644 --- a/rocky/rocky/views/organization_member_edit.py +++ b/rocky/rocky/views/organization_member_edit.py @@ -37,9 +37,7 @@ def get_form(self): def get_success_url(self): messages.add_message( - self.request, - messages.SUCCESS, - _("Member %s successfully updated.") % (self.object.user.full_name), + self.request, messages.SUCCESS, _("Member %s successfully updated.") % (self.object.user.full_name) ) return reverse("organization_member_list", kwargs={"organization_code": self.organization.code}) diff --git a/rocky/rocky/views/organization_member_list.py b/rocky/rocky/views/organization_member_list.py index 3a53e04b07b..807ffa25d98 100644 --- a/rocky/rocky/views/organization_member_list.py +++ b/rocky/rocky/views/organization_member_list.py @@ -24,10 +24,7 @@ class PageActions(Enum): class OrganizationMemberListView( - OrganizationPermissionRequiredMixin, - OrganizationMemberBreadcrumbsMixin, - OrganizationView, - ListView, + OrganizationPermissionRequiredMixin, OrganizationMemberBreadcrumbsMixin, OrganizationView, ListView ): model = OrganizationMember context_object_name = "members" diff --git a/rocky/rocky/views/privacy_statement.py b/rocky/rocky/views/privacy_statement.py index c1d0e8ee21e..7bba1e1e434 100644 --- a/rocky/rocky/views/privacy_statement.py +++ b/rocky/rocky/views/privacy_statement.py @@ -9,8 +9,6 @@ class PrivacyStatementView(TemplateView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["breadcrumbs"] = [ - {"url": reverse("privacy_statement"), "text": _("Privacy Statement")}, - ] + context["breadcrumbs"] = [{"url": reverse("privacy_statement"), "text": _("Privacy Statement")}] return context diff --git a/rocky/rocky/views/scan_profile.py b/rocky/rocky/views/scan_profile.py index d82620b7cbe..e93fa7c2f86 100644 --- a/rocky/rocky/views/scan_profile.py +++ b/rocky/rocky/views/scan_profile.py @@ -55,8 +55,7 @@ def get(self, request, *args, **kwargs): def post(self, request, *args, **kwargs): super().post(request, *args, **kwargs) self.octopoes_api_connector.save_scan_profile( - EmptyScanProfile(reference=self.ooi.reference), - valid_time=datetime.now(timezone.utc), + EmptyScanProfile(reference=self.ooi.reference), valid_time=datetime.now(timezone.utc) ) return redirect(get_ooi_url("scan_profile_detail", self.ooi.primary_key, self.organization.code)) diff --git a/rocky/rocky/views/scans.py b/rocky/rocky/views/scans.py index 103fcf360af..81285d728af 100644 --- a/rocky/rocky/views/scans.py +++ b/rocky/rocky/views/scans.py @@ -14,9 +14,7 @@ class ScanListView(ObjectsBreadcrumbsMixin, OrganizationView, TemplateView): def build_breadcrumbs(self) -> list[Breadcrumb]: breadcrumbs = super().build_breadcrumbs() - breadcrumbs.append( - {"url": "", "text": "Scans"}, - ) + breadcrumbs.append({"url": "", "text": "Scans"}) return breadcrumbs diff --git a/rocky/rocky/views/task_detail.py b/rocky/rocky/views/task_detail.py index d46de98121a..54aa15c68fe 100644 --- a/rocky/rocky/views/task_detail.py +++ b/rocky/rocky/views/task_detail.py @@ -32,10 +32,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["breadcrumbs"] = [ - { - "url": reverse("task_list", kwargs={"organization_code": self.organization.code}), - "text": _("Tasks"), - }, + {"url": reverse("task_list", kwargs={"organization_code": self.organization.code}), "text": _("Tasks")}, { "url": reverse( "boefje_task_view", diff --git a/rocky/rocky/views/tasks.py b/rocky/rocky/views/tasks.py index 897f89eec5f..41ad6717e4a 100644 --- a/rocky/rocky/views/tasks.py +++ b/rocky/rocky/views/tasks.py @@ -90,7 +90,7 @@ def get_context_data(self, **kwargs): else: context["stats_error"] = False context["breadcrumbs"] = [ - {"url": reverse("task_list", kwargs={"organization_code": self.organization.code}), "text": _("Tasks")}, + {"url": reverse("task_list", kwargs={"organization_code": self.organization.code}), "text": _("Tasks")} ] return context @@ -108,10 +108,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["breadcrumbs"] = [ - { - "url": reverse("task_list", kwargs={"organization_code": self.organization.code}), - "text": _("Tasks"), - }, + {"url": reverse("task_list", kwargs={"organization_code": self.organization.code}), "text": _("Tasks")} ] return context diff --git a/rocky/tests/account/test_login.py b/rocky/tests/account/test_login.py index 78c2be7ed79..fe816cf0672 100644 --- a/rocky/tests/account/test_login.py +++ b/rocky/tests/account/test_login.py @@ -47,11 +47,7 @@ def test_login(superuser): response = client.post( "/en/login/", - { - "auth-username": superuser.email, - "auth-password": "SuperSuper123!!", - "login_rocky_view-current_step": "auth", - }, + {"auth-username": superuser.email, "auth-password": "SuperSuper123!!", "login_rocky_view-current_step": "auth"}, ) assert response.status_code == 200 @@ -60,10 +56,7 @@ def test_login(superuser): assertContains(response, "Insert the token generated by your token authenticator app.") assertContains(response, "Submit") - response = client.post( - "/en/login/", - {"token-otp_token": "123456", "login_rocky_view-current_step": "token"}, - ) + response = client.post("/en/login/", {"token-otp_token": "123456", "login_rocky_view-current_step": "token"}) assert response.status_code == 200 diff --git a/rocky/tests/conftest.py b/rocky/tests/conftest.py index 3a757900c63..c3a0a8d3968 100644 --- a/rocky/tests/conftest.py +++ b/rocky/tests/conftest.py @@ -82,10 +82,7 @@ def create_organization(name, organization_code): def create_member(user, organization): - Indemnification.objects.create( - user=user, - organization=organization, - ) + Indemnification.objects.create(user=user, organization=organization) return OrganizationMember.objects.create( user=user, @@ -132,9 +129,7 @@ def add_redteam_group_permissions(member): def add_client_group_permissions(member): group = Group.objects.get(name=GROUP_CLIENT) member.groups.add(group) - client_permissions = [ - Permission.objects.get(codename="can_scan_organization").id, - ] + client_permissions = [Permission.objects.get(codename="can_scan_organization").id] group.permissions.set(client_permissions) @@ -482,10 +477,7 @@ def hostname(network) -> Hostname: @pytest.fixture def website(ip_service: IPService, hostname: Hostname): - return Website( - ip_service=ip_service.reference, - hostname=hostname.reference, - ) + return Website(ip_service=ip_service.reference, hostname=hostname.reference) @pytest.fixture @@ -960,10 +952,7 @@ def finding_type_kat_invalid_dnssec() -> KATFindingType: @pytest.fixture def tree_data_tls_findings_and_suites(): return { - "root": { - "reference": "", - "children": {"ooi": [{"reference": "", "children": {}}]}, - }, + "root": {"reference": "", "children": {"ooi": [{"reference": "", "children": {}}]}}, "store": { "Finding|Network|testnetwork|KAT-0001": { "object_type": "Finding", @@ -1011,7 +1000,7 @@ def tree_data_tls_findings_and_suites(): "key_exchange_algorithm": "ECDH", "cipher_suite_code": "xc014", }, - ], + ] }, }, }, @@ -1182,20 +1171,12 @@ def get_tree( return self.tree[reference] def query( - self, - path: str, - valid_time: datetime, - source: Reference | str | None = None, - offset: int = 0, - limit: int = 50, + self, path: str, valid_time: datetime, source: Reference | str | None = None, offset: int = 0, limit: int = 50 ) -> list[OOI]: return self.queries[path][source] def query_many( - self, - path: str, - valid_time: datetime, - sources: list[OOI | Reference | str], + self, path: str, valid_time: datetime, sources: list[OOI | Reference | str] ) -> list[tuple[str, OOIType]]: result = [] diff --git a/rocky/tests/integration/test_reports.py b/rocky/tests/integration/test_reports.py index d175e42b094..12fe355bb47 100644 --- a/rocky/tests/integration/test_reports.py +++ b/rocky/tests/integration/test_reports.py @@ -65,10 +65,7 @@ def test_system_report(octopoes_api_connector: OctopoesAPIConnector, valid_time) data = report.collect_data([input_ooi], valid_time)[input_ooi] assert data["input_ooi"] == input_ooi - assert data["summary"] == { - "total_domains": 7, - "total_systems": 2, - } + assert data["summary"] == {"total_domains": 7, "total_systems": 2} assert data["services"] == { "IPAddressV4|test|192.0.2.3": { "hostnames": [ @@ -178,12 +175,7 @@ def test_aggregate_report(octopoes_api_connector: OctopoesAPIConnector, valid_ti assert data["basic_security"]["summary"]["Dicom"] == { "rpki": {"number_of_compliant": 1, "total": 1}, - "system_specific": { - "number_of_compliant": 0, - "total": 0, - "checks": {}, - "ips": {}, - }, + "system_specific": {"number_of_compliant": 0, "total": 0, "checks": {}, "ips": {}}, "safe_connections": {"number_of_compliant": 1, "total": 1}, } assert data["basic_security"]["summary"]["Mail"] == { @@ -227,12 +219,7 @@ def test_aggregate_report(octopoes_api_connector: OctopoesAPIConnector, valid_ti } assert data["basic_security"]["summary"]["Other"] == { "rpki": {"number_of_compliant": 1, "total": 1}, - "system_specific": { - "number_of_compliant": 0, - "total": 0, - "checks": {}, - "ips": {}, - }, + "system_specific": {"number_of_compliant": 0, "total": 0, "checks": {}, "ips": {}}, "safe_connections": {"number_of_compliant": 1, "total": 1}, } @@ -273,9 +260,7 @@ def test_multi_report( multi_report = MultiOrganizationReport(octopoes_api_connector) multi_report_data = collect_report_data( - octopoes_api_connector, - [str(report_data_object.reference), str(report_data_object_2.reference)], - valid_time, + octopoes_api_connector, [str(report_data_object.reference), str(report_data_object_2.reference)], valid_time ) multi_data = multi_report.post_process_data(multi_report_data) assert multi_data["organizations"] == [octopoes_api_connector.client, octopoes_api_connector_2.client] @@ -304,41 +289,25 @@ def test_multi_report( assert multi_data["asset_vulnerabilities"] == [ { "asset": "IPAddressV6|test|3e4d:64a2:cb49:bd48:a1ba:def3:d15d:9230", - "vulnerabilities": { - "CVE-2018-20677": None, - "CVE-2019-8331": None, - "RetireJS-jquerymigrate-f3a3": None, - }, + "vulnerabilities": {"CVE-2018-20677": None, "CVE-2019-8331": None, "RetireJS-jquerymigrate-f3a3": None}, "organisation": "test-test_multi_report", "services": ["Web"], }, { "asset": "IPAddressV4|test|192.0.2.3", - "vulnerabilities": { - "CVE-2018-20677": None, - "CVE-2019-8331": None, - "RetireJS-jquerymigrate-f3a3": None, - }, + "vulnerabilities": {"CVE-2018-20677": None, "CVE-2019-8331": None, "RetireJS-jquerymigrate-f3a3": None}, "organisation": "test-test_multi_report", "services": ["Dicom", "Mail", "Other", "Web"], }, { "asset": "IPAddressV6|test|3e4d:64a2:cb49:bd48:a1ba:def3:d15d:9230", - "vulnerabilities": { - "CVE-2018-20677": None, - "CVE-2019-8331": None, - "RetireJS-jquerymigrate-f3a3": None, - }, + "vulnerabilities": {"CVE-2018-20677": None, "CVE-2019-8331": None, "RetireJS-jquerymigrate-f3a3": None}, "organisation": "test-test_multi_report-2", "services": ["Web"], }, { "asset": "IPAddressV4|test|192.0.2.3", - "vulnerabilities": { - "CVE-2018-20677": None, - "CVE-2019-8331": None, - "RetireJS-jquerymigrate-f3a3": None, - }, + "vulnerabilities": {"CVE-2018-20677": None, "CVE-2019-8331": None, "RetireJS-jquerymigrate-f3a3": None}, "organisation": "test-test_multi_report-2", "services": ["Dicom", "Mail", "Other", "Web"], }, @@ -376,10 +345,7 @@ def test_multi_report( "safe_connections": {"number_of_compliant": 2, "total": 2}, }, } - assert multi_data["basic_security"]["safe_connections"] == { - "number_of_available": 10, - "number_of_ips": 10, - } + assert multi_data["basic_security"]["safe_connections"] == {"number_of_available": 10, "number_of_ips": 10} assert multi_data["basic_security"]["system_specific"] == { "Dicom": {"checks": {}, "total": 0}, "Mail": {"checks": {"DKIM": 2, "DMARC": 2, "SPF": 2}, "total": 2}, @@ -400,30 +366,10 @@ def test_multi_report( }, } assert multi_data["basic_security"]["rpki"] == { - "Dicom": { - "number_of_available": 4, - "number_of_ips": 4, - "number_of_valid": 4, - "rpki_ips": True, - }, - "Mail": { - "number_of_available": 4, - "number_of_ips": 4, - "number_of_valid": 4, - "rpki_ips": True, - }, - "Other": { - "number_of_available": 4, - "number_of_ips": 4, - "number_of_valid": 4, - "rpki_ips": True, - }, - "Web": { - "number_of_available": 4, - "number_of_ips": 4, - "number_of_valid": 4, - "rpki_ips": True, - }, + "Dicom": {"number_of_available": 4, "number_of_ips": 4, "number_of_valid": 4, "rpki_ips": True}, + "Mail": {"number_of_available": 4, "number_of_ips": 4, "number_of_valid": 4, "rpki_ips": True}, + "Other": {"number_of_available": 4, "number_of_ips": 4, "number_of_valid": 4, "rpki_ips": True}, + "Web": {"number_of_available": 4, "number_of_ips": 4, "number_of_valid": 4, "rpki_ips": True}, } assert multi_data["system_vulnerabilities"] == { "CVE-2018-20677": {"cvss": None, "Web": 4, "Dicom": 2, "Mail": 2, "Other": 2}, diff --git a/rocky/tests/katalogus/test_katalogus.py b/rocky/tests/katalogus/test_katalogus.py index 53cf9e612e7..067efb31cca 100644 --- a/rocky/tests/katalogus/test_katalogus.py +++ b/rocky/tests/katalogus/test_katalogus.py @@ -267,12 +267,7 @@ def test_enable_disable_plugin_no_clearance(rf, redteam_member, mocker): mock_requests.Client().get.return_value = mock_response mock_response.json.return_value = plugin - request = setup_request( - rf.post( - "plugin_enable_disable", - ), - redteam_member.user, - ) + request = setup_request(rf.post("plugin_enable_disable"), redteam_member.user) response = PluginEnableDisableView.as_view()( setup_request(request, redteam_member.user), @@ -308,12 +303,7 @@ def test_enable_disable_plugin_no_clearance_other_text(rf, redteam_member, mocke mock_requests.Client().get.return_value = mock_response mock_response.json.return_value = plugin - request = setup_request( - rf.post( - "plugin_enable_disable", - ), - redteam_member.user, - ) + request = setup_request(rf.post("plugin_enable_disable"), redteam_member.user) response = PluginEnableDisableView.as_view()( setup_request(request, redteam_member.user), @@ -345,12 +335,7 @@ def test_enable_disable_plugin_has_clearance(rf, redteam_member, mocker): mock_requests.Client().get.return_value = mock_response mock_response.json.return_value = plugin - request = setup_request( - rf.post( - "plugin_enable_disable", - ), - redteam_member.user, - ) + request = setup_request(rf.post("plugin_enable_disable"), redteam_member.user) response = PluginEnableDisableView.as_view()( setup_request(request, redteam_member.user), @@ -373,12 +358,7 @@ def test_enable_disable_normalizer(rf, redteam_member, mocker): mock_requests.Client().get.return_value = mock_response mock_response.json.return_value = plugin - request = setup_request( - rf.post( - "plugin_enable_disable", - ), - redteam_member.user, - ) + request = setup_request(rf.post("plugin_enable_disable"), redteam_member.user) response = PluginEnableDisableView.as_view()( setup_request(request, redteam_member.user), diff --git a/rocky/tests/katalogus/test_katalogus_plugin_add.py b/rocky/tests/katalogus/test_katalogus_plugin_add.py index 011ebf3682e..0cbbe30eaf7 100644 --- a/rocky/tests/katalogus/test_katalogus_plugin_add.py +++ b/rocky/tests/katalogus/test_katalogus_plugin_add.py @@ -5,13 +5,7 @@ from tests.conftest import setup_request -def test_plugin_settings_add_view( - rf, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema, -): +def test_plugin_settings_add_view(rf, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema): mock_mixins_katalogus().get_plugin.return_value = plugin_details mock_mixins_katalogus().get_plugin_schema.return_value = plugin_schema @@ -28,11 +22,7 @@ def test_plugin_settings_add_view( def test_plugin_settings_add_view_no_required( - rf, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema_no_required, + rf, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema_no_required ): mock_mixins_katalogus().get_plugin.return_value = plugin_details mock_mixins_katalogus().get_plugin_schema.return_value = plugin_schema_no_required @@ -49,13 +39,7 @@ def test_plugin_settings_add_view_no_required( assertContains(response, "Add settings and enable boefje") -def test_plugin_settings_add( - rf, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema, -): +def test_plugin_settings_add(rf, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema): mock_katalogus = mock_mixins_katalogus() mock_katalogus.get_plugin.return_value = plugin_details mock_katalogus.get_plugin_schema.return_value = plugin_schema @@ -71,11 +55,7 @@ def test_plugin_settings_add( def test_plugin_settings_add_no_required( - rf, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema_no_required, + rf, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema_no_required ): mock_katalogus = mock_mixins_katalogus() mock_katalogus.get_plugin.return_value = plugin_details @@ -92,11 +72,7 @@ def test_plugin_settings_add_no_required( def test_plugin_settings_add_wrong_property_but_required( - rf, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema, + rf, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema ): mock_mixins_katalogus().get_plugin.return_value = plugin_details mock_mixins_katalogus().get_plugin_schema.return_value = plugin_schema @@ -111,11 +87,7 @@ def test_plugin_settings_add_wrong_property_but_required( def test_plugin_settings_add_string_too_long( - rf, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema, + rf, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema ): mock_mixins_katalogus().get_plugin.return_value = plugin_details mock_mixins_katalogus().get_plugin_schema.return_value = plugin_schema @@ -130,11 +102,7 @@ def test_plugin_settings_add_string_too_long( def test_plugin_settings_add_error_message_about_integer_for_string_type( - rf, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema, + rf, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema ): mock_mixins_katalogus().get_plugin.return_value = plugin_details mock_mixins_katalogus().get_plugin_schema.return_value = plugin_schema @@ -151,11 +119,7 @@ def test_plugin_settings_add_error_message_about_integer_for_string_type( def test_plugin_settings_add_error_message_about_integer_too_small( - rf, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema, + rf, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema ): mock_mixins_katalogus().get_plugin.return_value = plugin_details mock_mixins_katalogus().get_plugin_schema.return_value = plugin_schema @@ -172,11 +136,7 @@ def test_plugin_settings_add_error_message_about_integer_too_small( def test_plugin_settings_add_error_message_about_integer_too_big( - rf, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema, + rf, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema ): mock_mixins_katalogus().get_plugin.return_value = plugin_details mock_mixins_katalogus().get_plugin_schema.return_value = plugin_schema @@ -200,20 +160,13 @@ def test_plugin_single_settings_add_view_no_schema(rf, superuser_member, mock_mi request = setup_request(rf.post("plugin_settings_add", data={"boefje_id": 123}), superuser_member.user) response = PluginSettingsAddView.as_view()( - request, - organization_code=superuser_member.organization.code, - plugin_type="boefje", - plugin_id="test-plugin", + request, organization_code=superuser_member.organization.code, plugin_type="boefje", plugin_id="test-plugin" ) assert response.status_code == 302 mock_katalogus.upsert_plugin_settings.assert_not_called() assert response.headers["Location"] == reverse( - "boefje_detail", - kwargs={ - "organization_code": superuser_member.organization.code, - "plugin_id": "test-boefje", - }, + "boefje_detail", kwargs={"organization_code": superuser_member.organization.code, "plugin_id": "test-boefje"} ) assert list(request._messages).pop().message == "Trying to add settings to boefje without schema" diff --git a/rocky/tests/katalogus/test_katalogus_plugin_delete.py b/rocky/tests/katalogus/test_katalogus_plugin_delete.py index 41e8fa62a36..7acb72d8c1c 100644 --- a/rocky/tests/katalogus/test_katalogus_plugin_delete.py +++ b/rocky/tests/katalogus/test_katalogus_plugin_delete.py @@ -6,13 +6,7 @@ from tests.conftest import setup_request -def test_plugin_settings_delete_view( - rf, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema, -): +def test_plugin_settings_delete_view(rf, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema): mock_mixins_katalogus().get_plugin.return_value = plugin_details mock_mixins_katalogus().get_plugin_schema.return_value = plugin_schema @@ -29,43 +23,25 @@ def test_plugin_settings_delete_view( assertContains(response, "Delete settings") -def test_plugin_settings_delete( - rf, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema, -): +def test_plugin_settings_delete(rf, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema): mock_katalogus = mock_mixins_katalogus() mock_katalogus.get_plugin.return_value = plugin_details mock_katalogus.get_plugin_schema.return_value = plugin_schema request = setup_request(rf.post("plugin_settings_delete"), superuser_member.user) response = PluginSettingsDeleteView.as_view()( - request, - organization_code=superuser_member.organization.code, - plugin_type="boefje", - plugin_id="test-boefje", + request, organization_code=superuser_member.organization.code, plugin_type="boefje", plugin_id="test-boefje" ) assert response.status_code == 302 assert response.headers["Location"] == reverse( - "boefje_detail", - kwargs={ - "organization_code": superuser_member.organization.code, - "plugin_id": "test-boefje", - }, + "boefje_detail", kwargs={"organization_code": superuser_member.organization.code, "plugin_id": "test-boefje"} ) assert list(request._messages).pop().message == "Settings for plugin TestBoefje successfully deleted." def test_plugin_settings_delete_failed( - rf, - mocker, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema, + rf, mocker, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema ): mock_katalogus = mock_mixins_katalogus() mock_katalogus.get_plugin.return_value = plugin_details @@ -76,19 +52,12 @@ def test_plugin_settings_delete_failed( request = setup_request(rf.post("plugin_settings_delete"), superuser_member.user) response = PluginSettingsDeleteView.as_view()( - request, - organization_code=superuser_member.organization.code, - plugin_type="boefje", - plugin_id="test-boefje", + request, organization_code=superuser_member.organization.code, plugin_type="boefje", plugin_id="test-boefje" ) assert response.status_code == 302 assert response.headers["Location"] == reverse( - "boefje_detail", - kwargs={ - "organization_code": superuser_member.organization.code, - "plugin_id": "test-boefje", - }, + "boefje_detail", kwargs={"organization_code": superuser_member.organization.code, "plugin_id": "test-boefje"} ) assert ( list(request._messages).pop().message @@ -97,12 +66,7 @@ def test_plugin_settings_delete_failed( def test_plugin_settings_delete_no_settings_present( - rf, - mocker, - superuser_member, - mock_mixins_katalogus, - plugin_details, - plugin_schema, + rf, mocker, superuser_member, mock_mixins_katalogus, plugin_details, plugin_schema ): mock_katalogus = mock_mixins_katalogus() mock_katalogus.get_plugin.return_value = plugin_details @@ -112,18 +76,11 @@ def test_plugin_settings_delete_no_settings_present( ) request = setup_request(rf.post("plugin_settings_delete"), superuser_member.user) response = PluginSettingsDeleteView.as_view()( - request, - organization_code=superuser_member.organization.code, - plugin_type="boefje", - plugin_id="test-boefje", + request, organization_code=superuser_member.organization.code, plugin_type="boefje", plugin_id="test-boefje" ) assert response.status_code == 302 assert response.headers["Location"] == reverse( - "boefje_detail", - kwargs={ - "organization_code": superuser_member.organization.code, - "plugin_id": "test-boefje", - }, + "boefje_detail", kwargs={"organization_code": superuser_member.organization.code, "plugin_id": "test-boefje"} ) assert list(request._messages).pop().message == "Plugin TestBoefje has no settings." diff --git a/rocky/tests/katalogus/test_katalogus_plugin_detail.py b/rocky/tests/katalogus/test_katalogus_plugin_detail.py index 03d717aa88f..05967f2e681 100644 --- a/rocky/tests/katalogus/test_katalogus_plugin_detail.py +++ b/rocky/tests/katalogus/test_katalogus_plugin_detail.py @@ -16,9 +16,7 @@ def test_plugin_detail_view( request = setup_request(rf.get("boefje_detail"), superuser_member.user) response = BoefjeDetailView.as_view()( - request, - organization_code=superuser_member.organization.code, - plugin_id="test-plugin", + request, organization_code=superuser_member.organization.code, plugin_id="test-plugin" ) assertContains(response, "TestBoefje") @@ -40,9 +38,7 @@ def test_plugin_detail_view_no_consumes( request = setup_request(rf.get("boefje_detail"), superuser_member.user) response = BoefjeDetailView.as_view()( - request, - organization_code=superuser_member.organization.code, - plugin_id="test-plugin", + request, organization_code=superuser_member.organization.code, plugin_id="test-plugin" ) assertContains(response, "TestBoefje does not need any input objects.") diff --git a/rocky/tests/objects/test_objects.py b/rocky/tests/objects/test_objects.py index f96d927c314..5b33afbc71a 100644 --- a/rocky/tests/objects/test_objects.py +++ b/rocky/tests/objects/test_objects.py @@ -40,10 +40,7 @@ def test_ooi_list_with_clearance_type_filter_and_clearance_level_filter( ): kwargs = {"organization_code": client_member.organization.code} url = reverse("ooi_list", kwargs=kwargs) - request = rf.get( - url, - {"clearance_level": [0, 1], "clearance_type": ["declared", "inherited"]}, - ) + request = rf.get(url, {"clearance_level": [0, 1], "clearance_type": ["declared", "inherited"]}) request.resolver_match = resolve(url) setup_request(request, client_member.user) @@ -81,11 +78,7 @@ def test_ooi_list_delete_multiple(rf, client_member, mock_organization_view_octo request = rf.post( url, - data={ - "ooi": ["Network|internet", "Hostname|internet|scanme.org"], - "scan-profile": "L0", - "action": "delete", - }, + data={"ooi": ["Network|internet", "Hostname|internet|scanme.org"], "scan-profile": "L0", "action": "delete"}, ) setup_request(request, client_member.user) response = OOIListView.as_view()(request, organization_code=client_member.organization.code) @@ -152,12 +145,7 @@ def test_update_scan_profile_single(rf, client_member, mock_organization_view_oc client_member.save() request = rf.post( - url, - data={ - "ooi": ["Hostname|internet|scanme.org"], - "scan-profile": "L4", - "action": "update-scan-profile", - }, + url, data={"ooi": ["Hostname|internet|scanme.org"], "scan-profile": "L4", "action": "update-scan-profile"} ) setup_request(request, client_member.user) @@ -172,12 +160,7 @@ def test_update_scan_profile_to_inherit(rf, client_member, mock_organization_vie url = reverse("ooi_list", kwargs=kwargs) request = rf.post( - url, - data={ - "ooi": ["Hostname|internet|scanme.org"], - "scan-profile": "inherit", - "action": "update-scan-profile", - }, + url, data={"ooi": ["Hostname|internet|scanme.org"], "scan-profile": "inherit", "action": "update-scan-profile"} ) setup_request(request, client_member.user) response = OOIListView.as_view()(request, organization_code=client_member.organization.code) @@ -192,12 +175,7 @@ def test_update_scan_profile_to_inherit_connection_error(rf, client_member, mock url = reverse("ooi_list", kwargs=kwargs) request = rf.post( - url, - data={ - "ooi": ["Hostname|internet|scanme.org"], - "scan-profile": "inherit", - "action": "update-scan-profile", - }, + url, data={"ooi": ["Hostname|internet|scanme.org"], "scan-profile": "inherit", "action": "update-scan-profile"} ) setup_request(request, client_member.user) response = OOIListView.as_view()(request, organization_code=client_member.organization.code) @@ -211,12 +189,7 @@ def test_update_scan_profile_to_inherit_object_not_found(rf, client_member, mock url = reverse("ooi_list", kwargs=kwargs) request = rf.post( - url, - data={ - "ooi": ["Hostname|internet|scanme.org"], - "scan-profile": "inherit", - "action": "update-scan-profile", - }, + url, data={"ooi": ["Hostname|internet|scanme.org"], "scan-profile": "inherit", "action": "update-scan-profile"} ) setup_request(request, client_member.user) response = OOIListView.as_view()(request, organization_code=client_member.organization.code) @@ -257,14 +230,7 @@ def test_update_scan_profiles_forbidden_trusted(rf, client_member, mock_organiza kwargs = {"organization_code": client_member.organization.code} url = reverse("ooi_list", kwargs=kwargs) - request = rf.post( - url, - data={ - "ooi": ["Network|internet"], - "scan-profile": "L1", - "action": "update-scan-profile", - }, - ) + request = rf.post(url, data={"ooi": ["Network|internet"], "scan-profile": "L1", "action": "update-scan-profile"}) client_member.trusted_clearance_level = -1 client_member.save() @@ -346,11 +312,7 @@ def test_delete_octopoes_down(rf, client_member, mock_organization_view_octopoes request = rf.post( "ooi_list", - data={ - "ooi": ["Network|internet", "Hostname|internet|scanme.org"], - "scan-profile": "L2", - "action": "delete", - }, + data={"ooi": ["Network|internet", "Hostname|internet|scanme.org"], "scan-profile": "L2", "action": "delete"}, ) client_member.trusted_clearance_level = 4 client_member.acknowledged_clearance_level = 4 @@ -367,11 +329,7 @@ def test_delete_object_not_found(rf, client_member, mock_organization_view_octop request = rf.post( "ooi_list", - data={ - "ooi": ["Network|internet", "Hostname|internet|scanme.org"], - "scan-profile": "L2", - "action": "delete", - }, + data={"ooi": ["Network|internet", "Hostname|internet|scanme.org"], "scan-profile": "L2", "action": "delete"}, ) setup_request(request, client_member.user) @@ -469,10 +427,7 @@ def test_delete_perms_object_list(request, member, rf, mock_organization_view_oc url = reverse("ooi_list", kwargs={"organization_code": member.organization.code}) request = rf.get(url) request.resolver_match = resolve(url) - response = OOIListView.as_view()( - setup_request(request, member.user), - organization_code=member.organization.code, - ) + response = OOIListView.as_view()(setup_request(request, member.user), organization_code=member.organization.code) assert response.status_code == 200 @@ -488,8 +443,7 @@ def test_delete_perms_object_list_clients(rf, client_member, mock_organization_v request = rf.get(url) request.resolver_match = resolve(url) response = OOIListView.as_view()( - setup_request(request, client_member.user), - organization_code=client_member.organization.code, + setup_request(request, client_member.user), organization_code=client_member.organization.code ) assert response.status_code == 200 diff --git a/rocky/tests/objects/test_objects_detail.py b/rocky/tests/objects/test_objects_detail.py index c1ea1e88f02..f793036d9ba 100644 --- a/rocky/tests/objects/test_objects_detail.py +++ b/rocky/tests/objects/test_objects_detail.py @@ -17,11 +17,7 @@ "children": {"ooi": [{"reference": "Network|testnetwork", "children": {}}]}, }, "store": { - "Network|testnetwork": { - "object_type": "Network", - "primary_key": "Network|testnetwork", - "name": "testnetwork", - }, + "Network|testnetwork": {"object_type": "Network", "primary_key": "Network|testnetwork", "name": "testnetwork"}, "Finding|Network|testnetwork|KAT-000": { "object_type": "Finding", "primary_key": "Finding|Network|testnetwork|KAT-000", @@ -37,11 +33,7 @@ "children": {"ooi": [{"reference": "Network|testnetwork", "children": {}}]}, }, "store": { - "Network|testnetwork": { - "object_type": "Network", - "primary_key": "Network|testnetwork", - "name": "testnetwork", - }, + "Network|testnetwork": {"object_type": "Network", "primary_key": "Network|testnetwork", "name": "testnetwork"}, "Question|/test|Network|testnetwork": { "ooi": "Question|/test|Network|testnetwork", "object_type": "Question", @@ -54,12 +46,7 @@ def test_ooi_detail( - rf, - client_member, - mock_scheduler, - mock_organization_view_octopoes, - lazy_task_list_with_boefje, - mocker, + rf, client_member, mock_scheduler, mock_organization_view_octopoes, lazy_task_list_with_boefje, mocker ): mocker.patch("katalogus.client.KATalogusClientV1") @@ -89,12 +76,7 @@ def test_ooi_detail( def test_question_detail( - rf, - client_member, - mock_scheduler, - mock_organization_view_octopoes, - lazy_task_list_with_boefje, - mocker, + rf, client_member, mock_scheduler, mock_organization_view_octopoes, lazy_task_list_with_boefje, mocker ): mocker.patch("katalogus.client.KATalogusClientV1") @@ -130,10 +112,7 @@ def test_answer_question( request = setup_request( rf.post( f"/en/{client_member.organization.code}/objects/details/?{query_string}", - data={ - "schema": '{"key": "value", "sa_tcp_ports": "314159,23"}', - "action": "submit_answer", - }, + data={"schema": '{"key": "value", "sa_tcp_ports": "314159,23"}', "action": "submit_answer"}, ), client_member.user, ) @@ -161,10 +140,7 @@ def test_answer_question_bad_schema( request = setup_request( rf.post( f"/en/{client_member.organization.code}/objects/details/?{query_string}", - data={ - "schema": '{"key": "value", "sa_tcp_ports": 314159}', - "action": "submit_answer", - }, + data={"schema": '{"key": "value", "sa_tcp_ports": 314159}', "action": "submit_answer"}, ), client_member.user, ) @@ -176,13 +152,7 @@ def test_answer_question_bad_schema( assertContains(response, f"314159 is not of type {quote_enc}string{quote_enc}", status_code=422) -def test_ooi_detail_start_scan( - rf, - client_member, - mock_organization_view_octopoes, - mocker, - network, -): +def test_ooi_detail_start_scan(rf, client_member, mock_organization_view_octopoes, mocker, network): mock_katalogus = mocker.patch("katalogus.client.KATalogusClientV1") mocker.patch("katalogus.views.mixins.schedule_task") @@ -206,10 +176,7 @@ def test_ooi_detail_start_scan( request = setup_request( rf.post( f"/en/{client_member.organization.code}/objects/details/?{query_string}", - data={ - "boefje_id": "nmap", - "action": "start_scan", - }, + data={"boefje_id": "nmap", "action": "start_scan"}, ), client_member.user, ) @@ -222,13 +189,7 @@ def test_ooi_detail_start_scan( def test_ooi_detail_start_scan_no_indemnification( - rf, - client_member, - mock_scheduler, - mock_organization_view_octopoes, - lazy_task_list_with_boefje, - mocker, - network, + rf, client_member, mock_scheduler, mock_organization_view_octopoes, lazy_task_list_with_boefje, mocker, network ): mocker.patch("katalogus.client.KATalogusClientV1") @@ -242,10 +203,7 @@ def test_ooi_detail_start_scan_no_indemnification( request = setup_request( rf.post( f"/en/{client_member.organization.code}/objects/details/?{query_string}", - data={ - "boefje_id": "nmap", - "action": "start_scan", - }, + data={"boefje_id": "nmap", "action": "start_scan"}, ), client_member.user, ) @@ -257,13 +215,7 @@ def test_ooi_detail_start_scan_no_indemnification( def test_ooi_detail_start_scan_no_action( - rf, - client_member, - mock_scheduler, - mock_organization_view_octopoes, - lazy_task_list_with_boefje, - mocker, - network, + rf, client_member, mock_scheduler, mock_organization_view_octopoes, lazy_task_list_with_boefje, mocker, network ): mocker.patch("katalogus.client.KATalogusClientV1") @@ -273,12 +225,7 @@ def test_ooi_detail_start_scan_no_action( # Passing query params in POST requests is not well-supported for RequestFactory it seems, hence the absolute path query_string = urlencode({"ooi_id": network.reference}, doseq=True) request = setup_request( - rf.post( - f"/en/{client_member.organization.code}/objects/details/?{query_string}", - data={ - "boefje_id": "nmap", - }, - ), + rf.post(f"/en/{client_member.organization.code}/objects/details/?{query_string}", data={"boefje_id": "nmap"}), client_member.user, ) response = OOIDetailView.as_view()(request, organization_code=client_member.organization.code) @@ -289,13 +236,7 @@ def test_ooi_detail_start_scan_no_action( @pytest.mark.parametrize("member", ["superuser_member", "admin_member", "redteam_member"]) def test_delete_perms_ooi_detail( - request, - member, - rf, - mock_scheduler, - mock_organization_view_octopoes, - lazy_task_list_with_boefje, - mocker, + request, member, rf, mock_scheduler, mock_organization_view_octopoes, lazy_task_list_with_boefje, mocker ): member = request.getfixturevalue(member) mocker.patch("katalogus.client.KATalogusClientV1") @@ -312,12 +253,7 @@ def test_delete_perms_ooi_detail( def test_delete_perms_ooi_detail_clients( - rf, - client_member, - mock_scheduler, - mock_organization_view_octopoes, - lazy_task_list_with_boefje, - mocker, + rf, client_member, mock_scheduler, mock_organization_view_octopoes, lazy_task_list_with_boefje, mocker ): mocker.patch("katalogus.client.KATalogusClientV1") mock_organization_view_octopoes().get_tree.return_value = ReferenceTree.model_validate(TREE_DATA) @@ -332,12 +268,7 @@ def test_delete_perms_ooi_detail_clients( def test_ooi_detail_start_scan_perms( - rf, - client_member, - mock_scheduler, - mock_organization_view_octopoes, - lazy_task_list_with_boefje, - mocker, + rf, client_member, mock_scheduler, mock_organization_view_octopoes, lazy_task_list_with_boefje, mocker ): mocker.patch("katalogus.client.KATalogusClientV1") request = setup_request(rf.get("ooi_detail", {"ooi_id": "Network|testnetwork"}), client_member.user) diff --git a/rocky/tests/objects/test_objects_findings.py b/rocky/tests/objects/test_objects_findings.py index 1f60f08c5ac..477a9a223ac 100644 --- a/rocky/tests/objects/test_objects_findings.py +++ b/rocky/tests/objects/test_objects_findings.py @@ -18,11 +18,7 @@ "children": {"ooi": [{"reference": "Network|testnetwork", "children": {}}]}, }, "store": { - "Network|testnetwork": { - "object_type": "Network", - "primary_key": "Network|testnetwork", - "name": "testnetwork", - }, + "Network|testnetwork": {"object_type": "Network", "primary_key": "Network|testnetwork", "name": "testnetwork"}, "Finding|Network|testnetwork|KAT-000": { "object_type": "Finding", "primary_key": "Finding|Network|testnetwork|KAT-000", @@ -45,11 +41,7 @@ "finding": "Finding|Network|testnetwork|KAT-000", "reason": "Hallo", }, - "Network|testnetwork": { - "object_type": "Network", - "primary_key": "Network|testnetwork", - "name": "testnetwork", - }, + "Network|testnetwork": {"object_type": "Network", "primary_key": "Network|testnetwork", "name": "testnetwork"}, "Finding|Network|testnetwork|KAT-000": { "object_type": "Finding", "primary_key": "Finding|Network|testnetwork|KAT-000", @@ -191,7 +183,7 @@ def test_muted_finding_button_not_presence(rf, mock_organization_view_octopoes, proof="proof", description="test description 123", reproduce="reproduce", - ), + ) ], ) mock_organization_view_octopoes().load_objects_bulk.return_value = { @@ -200,8 +192,7 @@ def test_muted_finding_button_not_presence(rf, mock_organization_view_octopoes, } response = FindingListView.as_view()( - setup_request(rf.get("finding_list"), redteam_member.user), - organization_code=redteam_member.organization.code, + setup_request(rf.get("finding_list"), redteam_member.user), organization_code=redteam_member.organization.code ) assert response.status_code == 200 @@ -237,8 +228,7 @@ def test_muted_finding_button_presence_more_findings_and_post( reproduce="reproduce", ) mock_organization_view_octopoes().list_findings.return_value = Paginated[Finding]( - count=2, - items=[finding_1, finding_2], + count=2, items=[finding_1, finding_2] ) mock_organization_view_octopoes().load_objects_bulk.return_value = { @@ -248,8 +238,7 @@ def test_muted_finding_button_presence_more_findings_and_post( } response = FindingListView.as_view()( - setup_request(rf.get("finding_list"), member.user), - organization_code=member.organization.code, + setup_request(rf.get("finding_list"), member.user), organization_code=member.organization.code ) assert response.status_code == 200 @@ -258,14 +247,7 @@ def test_muted_finding_button_presence_more_findings_and_post( assertContains(response, '') request = setup_request( - rf.post( - "finding_mute_bulk", - { - "finding": [finding_1, finding_2], - "reason": "testing", - }, - ), - member.user, + rf.post("finding_mute_bulk", {"finding": [finding_1, finding_2], "reason": "testing"}), member.user ) response_post = MuteFindingsBulkView.as_view()(request, organization_code=member.organization.code) @@ -274,14 +256,7 @@ def test_muted_finding_button_presence_more_findings_and_post( @pytest.mark.parametrize("member", ["admin_member", "client_member"]) -def test_can_mute_findings_perms( - rf, - request, - member, - mock_organization_view_octopoes, - network, - finding_types, -): +def test_can_mute_findings_perms(rf, request, member, mock_organization_view_octopoes, network, finding_types): member = request.getfixturevalue(member) mock_organization_view_octopoes().list_findings.return_value = Paginated[Finding]( count=2, @@ -310,8 +285,7 @@ def test_can_mute_findings_perms( } response = FindingListView.as_view()( - setup_request(rf.get("finding_list"), member.user), - organization_code=member.organization.code, + setup_request(rf.get("finding_list"), member.user), organization_code=member.organization.code ) assert response.status_code == 200 @@ -348,8 +322,7 @@ def test_findings_list_filtering( reproduce="reproduce", ) mock_organization_view_octopoes().list_findings.return_value = Paginated[Finding]( - count=2, - items=[finding_1, finding_2], + count=2, items=[finding_1, finding_2] ) mock_organization_view_octopoes().load_objects_bulk.return_value = { @@ -359,22 +332,13 @@ def test_findings_list_filtering( } response = FindingListView.as_view()( - setup_request(rf.get("finding_list"), member.user), - organization_code=member.organization.code, + setup_request(rf.get("finding_list"), member.user), organization_code=member.organization.code ) assert response.status_code == 200 assert len(response.context_data["object_list"]) == 2 - request_filtering = setup_request( - rf.get( - "finding_list", - { - "severity": "low", - }, - ), - member.user, - ) + request_filtering = setup_request(rf.get("finding_list", {"severity": "low"}), member.user) FindingListView.as_view()(request_filtering, organization_code=member.organization.code) assert mock_organization_view_octopoes().list_findings.mock_calls[2].kwargs["severities"] == {RiskLevelSeverity.LOW} diff --git a/rocky/tests/objects/test_objects_graph.py b/rocky/tests/objects/test_objects_graph.py index c214fd460d5..247de84be0d 100644 --- a/rocky/tests/objects/test_objects_graph.py +++ b/rocky/tests/objects/test_objects_graph.py @@ -11,11 +11,7 @@ "children": {"ooi": [{"reference": "Network|testnetwork", "children": {}}]}, }, "store": { - "Network|testnetwork": { - "object_type": "Network", - "primary_key": "Network|testnetwork", - "name": "testnetwork", - }, + "Network|testnetwork": {"object_type": "Network", "primary_key": "Network|testnetwork", "name": "testnetwork"}, "Finding|Network|testnetwork|KAT-000": { "object_type": "Finding", "primary_key": "Finding|Network|testnetwork|KAT-000", diff --git a/rocky/tests/objects/test_objects_report.py b/rocky/tests/objects/test_objects_report.py index c5001bed601..d0261fd6b2c 100644 --- a/rocky/tests/objects/test_objects_report.py +++ b/rocky/tests/objects/test_objects_report.py @@ -150,7 +150,7 @@ def test_organization_pdf_report(rf, client_member, mock_organization_view_octop proof="proof", description="test description 123", reproduce="reproduce", - ), + ) ] * 150, ) diff --git a/rocky/tests/objects/test_objects_scan_profile.py b/rocky/tests/objects/test_objects_scan_profile.py index 2534971b477..83f23b42858 100644 --- a/rocky/tests/objects/test_objects_scan_profile.py +++ b/rocky/tests/objects/test_objects_scan_profile.py @@ -17,11 +17,7 @@ "object_type": "Network", "primary_key": "Network|testnetwork", "name": "testnetwork", - "scan_profile": { - "scan_profile_type": "declared", - "reference": "Network|testnetwork", - "level": 1, - }, + "scan_profile": {"scan_profile_type": "declared", "reference": "Network|testnetwork", "level": 1}, }, "HostnameHTTPURL|https|internet|scanme.org|443|/": { "object_type": "HostnameHTTPURL", diff --git a/rocky/tests/objects/test_objects_tree.py b/rocky/tests/objects/test_objects_tree.py index 29d582ca250..21b2c9dbe20 100644 --- a/rocky/tests/objects/test_objects_tree.py +++ b/rocky/tests/objects/test_objects_tree.py @@ -11,11 +11,7 @@ "children": {"ooi": [{"reference": "Network|testnetwork", "children": {}}]}, }, "store": { - "Network|testnetwork": { - "object_type": "Network", - "primary_key": "Network|testnetwork", - "name": "testnetwork", - }, + "Network|testnetwork": {"object_type": "Network", "primary_key": "Network|testnetwork", "name": "testnetwork"}, "Finding|Network|testnetwork|KAT-000": { "object_type": "Finding", "primary_key": "Finding|Network|testnetwork|KAT-000", diff --git a/rocky/tests/onboarding/test_onboarding_admin_wizard.py b/rocky/tests/onboarding/test_onboarding_admin_wizard.py index 131a2e1a180..0fb30752d72 100644 --- a/rocky/tests/onboarding/test_onboarding_admin_wizard.py +++ b/rocky/tests/onboarding/test_onboarding_admin_wizard.py @@ -52,19 +52,13 @@ def test_onboarding_organization_setup(rf, superuser, adminuser, redteamuser, cl assert response_superuser.status_code == 200 with pytest.raises(PermissionDenied): - OnboardingOrganizationSetupView.as_view()( - setup_request(rf.get("step_organization_setup"), adminuser), - ) + OnboardingOrganizationSetupView.as_view()(setup_request(rf.get("step_organization_setup"), adminuser)) with pytest.raises(PermissionDenied): - OnboardingOrganizationSetupView.as_view()( - setup_request(rf.get("step_organization_setup"), redteamuser), - ) + OnboardingOrganizationSetupView.as_view()(setup_request(rf.get("step_organization_setup"), redteamuser)) with pytest.raises(PermissionDenied): - OnboardingOrganizationSetupView.as_view()( - setup_request(rf.get("step_organization_setup"), clientuser), - ) + OnboardingOrganizationSetupView.as_view()(setup_request(rf.get("step_organization_setup"), clientuser)) def test_onboarding_organization_update(rf, superuser_member, admin_member, redteam_member, client_member): diff --git a/rocky/tests/onboarding/test_onboarding_organization_steps.py b/rocky/tests/onboarding/test_onboarding_organization_steps.py index d22d67a2fa5..ce01b7fde8d 100644 --- a/rocky/tests/onboarding/test_onboarding_organization_steps.py +++ b/rocky/tests/onboarding/test_onboarding_organization_steps.py @@ -23,8 +23,7 @@ def test_onboarding_introduction(request, member, rf): member = request.getfixturevalue(member) response = OnboardingIntroductionView.as_view()( - setup_request(rf.get("step_introduction"), member.user), - organization_code=member.organization.code, + setup_request(rf.get("step_introduction"), member.user), organization_code=member.organization.code ) assert response.status_code == 200 @@ -37,8 +36,7 @@ def test_onboarding_introduction(request, member, rf): def test_onboarding_choose_report_info(request, member, rf): member = request.getfixturevalue(member) response = OnboardingChooseReportInfoView.as_view()( - setup_request(rf.get("step_choose_report_info"), member.user), - organization_code=member.organization.code, + setup_request(rf.get("step_choose_report_info"), member.user), organization_code=member.organization.code ) assert response.status_code == 200 @@ -54,8 +52,7 @@ def test_onboarding_choose_report_info(request, member, rf): def test_onboarding_choose_report_type(request, member, rf): member = request.getfixturevalue(member) response = OnboardingChooseReportTypeView.as_view()( - setup_request(rf.get("step_choose_report_type"), member.user), - organization_code=member.organization.code, + setup_request(rf.get("step_choose_report_type"), member.user), organization_code=member.organization.code ) assert response.status_code == 200 @@ -72,8 +69,7 @@ def test_onboarding_choose_report_type(request, member, rf): def test_onboarding_setup_scan(request, member, rf): member = request.getfixturevalue(member) response = OnboardingSetupScanOOIInfoView.as_view()( - setup_request(rf.get("step_setup_scan_ooi_info"), member.user), - organization_code=member.organization.code, + setup_request(rf.get("step_setup_scan_ooi_info"), member.user), organization_code=member.organization.code ) assert response.status_code == 200 @@ -113,10 +109,7 @@ def test_onboarding_setup_scan_detail_create_ooi( member = request.getfixturevalue(member) response = OnboardingSetupScanOOIAddView.as_view()( - setup_request( - rf.post("step_setup_scan_ooi_add", {"url": url.raw}), - member.user, - ), + setup_request(rf.post("step_setup_scan_ooi_add", {"url": url.raw}), member.user), ooi_type="URL", organization_code=member.organization.code, ) @@ -253,14 +246,7 @@ def test_onboarding_set_clearance_level( @pytest.mark.parametrize("member", ["superuser_member", "redteam_member"]) -def test_onboarding_select_plugins( - request, - member, - rf, - mocker, - mock_organization_view_octopoes, - url, -): +def test_onboarding_select_plugins(request, member, rf, mocker, mock_organization_view_octopoes, url): mocker.patch("onboarding.views.get_katalogus") member = request.getfixturevalue(member) request = setup_request(rf.get("step_setup_scan_select_plugins", {"ooi": url.primary_key}), member.user) @@ -280,12 +266,7 @@ def test_onboarding_select_plugins( @pytest.mark.parametrize("member", ["admin_member", "client_member"]) -def test_onboarding_select_plugins_perms( - request, - member, - rf, - url, -): +def test_onboarding_select_plugins_perms(request, member, rf, url): member = request.getfixturevalue(member) request = setup_request(rf.get("step_setup_scan_select_plugins", {"ooi": url.primary_key}), member.user) @@ -321,11 +302,7 @@ def test_onboarding_scanning_boefjes(request, member, rf, mock_organization_view setup_request( rf.post( "step_report", - { - "observed_at": valid_time.strftime("%Y-%m-%d"), - "ooi": url.primary_key, - "report_type": "dns-report", - }, + {"observed_at": valid_time.strftime("%Y-%m-%d"), "ooi": url.primary_key, "report_type": "dns-report"}, ), member.user, ), diff --git a/rocky/tests/reports/test_base_report.py b/rocky/tests/reports/test_base_report.py index 26298ed2f42..f5754dd59c8 100644 --- a/rocky/tests/reports/test_base_report.py +++ b/rocky/tests/reports/test_base_report.py @@ -37,10 +37,7 @@ def test_aggregate_report_choose_report_types( kwargs = {"organization_code": client_member.organization.code} url = reverse("aggregate_report_select_oois", kwargs=kwargs) - request = rf.get( - url, - {"observed_at": valid_time.strftime("%Y-%m-%d"), "ooi": "all"}, - ) + request = rf.get(url, {"observed_at": valid_time.strftime("%Y-%m-%d"), "ooi": "all"}) request.resolver_match = resolve(url) setup_request(request, client_member.user) diff --git a/rocky/tests/reports/test_dns_report.py b/rocky/tests/reports/test_dns_report.py index d4aa2f472e5..de667cf5669 100644 --- a/rocky/tests/reports/test_dns_report.py +++ b/rocky/tests/reports/test_dns_report.py @@ -4,9 +4,7 @@ def test_dns_report_no_findings(mock_octopoes_api_connector, valid_time, hostname, tree_data_no_findings): - mock_octopoes_api_connector.tree = { - hostname.reference: ReferenceTree.model_validate(tree_data_no_findings), - } + mock_octopoes_api_connector.tree = {hostname.reference: ReferenceTree.model_validate(tree_data_no_findings)} report = DNSReport(mock_octopoes_api_connector) data = report.generate_data(str(hostname.reference), valid_time) @@ -31,9 +29,7 @@ def test_dns_report_two_findings_one_finding_type( finding_type_kat_no_two_ipv6.reference: finding_type_kat_no_two_ipv6, } - mock_octopoes_api_connector.tree = { - hostname.reference: ReferenceTree.model_validate(tree_data_dns_findings), - } + mock_octopoes_api_connector.tree = {hostname.reference: ReferenceTree.model_validate(tree_data_dns_findings)} report = DNSReport(mock_octopoes_api_connector) data = report.generate_data(str(hostname.reference), valid_time) diff --git a/rocky/tests/reports/test_findings_report.py b/rocky/tests/reports/test_findings_report.py index 71f650812f3..9aba44d7bca 100644 --- a/rocky/tests/reports/test_findings_report.py +++ b/rocky/tests/reports/test_findings_report.py @@ -4,13 +4,9 @@ def test_findings_report_no_findings(mock_octopoes_api_connector, valid_time, hostname, tree_data_no_findings): - mock_octopoes_api_connector.oois = { - hostname.reference: hostname, - } + mock_octopoes_api_connector.oois = {hostname.reference: hostname} - mock_octopoes_api_connector.tree = { - hostname.reference: ReferenceTree.model_validate(tree_data_no_findings), - } + mock_octopoes_api_connector.tree = {hostname.reference: ReferenceTree.model_validate(tree_data_no_findings)} report = FindingsReport(mock_octopoes_api_connector) data = report.generate_data(str(hostname.reference), valid_time) @@ -30,9 +26,7 @@ def test_findings_report_two_findings_one_finding_type( } # This tree data contains four OOIs, three of which are findings that contain two different finding_types. - mock_octopoes_api_connector.tree = { - hostname.reference: ReferenceTree.model_validate(tree_data_findings), - } + mock_octopoes_api_connector.tree = {hostname.reference: ReferenceTree.model_validate(tree_data_findings)} report = FindingsReport(mock_octopoes_api_connector) data = report.generate_data(str(hostname.reference), valid_time) diff --git a/rocky/tests/reports/test_ipv6_report.py b/rocky/tests/reports/test_ipv6_report.py index 9c86df760f6..972b738e534 100644 --- a/rocky/tests/reports/test_ipv6_report.py +++ b/rocky/tests/reports/test_ipv6_report.py @@ -2,13 +2,9 @@ def test_ipv6_report_hostname_with_ipv6(mock_octopoes_api_connector, valid_time, hostname, ipaddressv6, ipaddressv4): - mock_octopoes_api_connector.oois = { - hostname.reference: hostname, - } + mock_octopoes_api_connector.oois = {hostname.reference: hostname} mock_octopoes_api_connector.queries = { - "Hostname. findings[1]["cvss"]["score"] mock_octopoes_api_connector.queries = { - "Hostname. list[_Service]: try: if name == search_query: service = _Service( - name, - int(port) if port else None, - transport_protocol if transport_protocol else None, - description, + name, int(port) if port else None, transport_protocol if transport_protocol else None, description ) services.append(service) except Exception: # noqa: S110 @@ -94,8 +91,7 @@ def table_to_2d(table_tag): # a colspan of 0 means “fill until the end” but can really only apply # to the last cell; ignore it elsewhere. colcount = max( - colcount, - sum(int(c.get("colspan", 1)) or 1 for c in cells[:-1]) + len(cells[-1:]) + len(rowspans_list), + colcount, sum(int(c.get("colspan", 1)) or 1 for c in cells[:-1]) + len(cells[-1:]) + len(rowspans_list) ) # update rowspan bookkeeping; 0 is a span to the bottom. rowspans_list += [int(c.get("rowspan", 1)) or len(rows) - r for c in cells] @@ -179,10 +175,7 @@ def port_info(number: str, protocol: str) -> tuple[str, str]: source = "https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers" descriptions = [] if not items: - return ( - f"No description found in wiki table for port {number} with protocol {protocol}", - source, - ) + return (f"No description found in wiki table for port {number} with protocol {protocol}", source) for item in items: if item.port == int(number) and protocol.lower() in item.protocols: diff --git a/rocky/tools/admin.py b/rocky/tools/admin.py index b3e9731bbb6..1d064e44a48 100644 --- a/rocky/tools/admin.py +++ b/rocky/tools/admin.py @@ -37,10 +37,7 @@ class OOIInformationAdmin(admin.ModelAdmin): def get_readonly_fields(self, request, obj=None): if obj is not None: # editing an existing object if not obj.value: - return self.readonly_fields + ( - "id", - "consult_api", - ) + return self.readonly_fields + ("id", "consult_api") return self.readonly_fields + ("id",) return self.readonly_fields diff --git a/rocky/tools/forms/base.py b/rocky/tools/forms/base.py index 7bf73024c8f..f6e7e3061c5 100644 --- a/rocky/tools/forms/base.py +++ b/rocky/tools/forms/base.py @@ -95,12 +95,7 @@ class CheckboxGroup(forms.CheckboxSelectMultiple): required_options: list[str] wrap_label = True - def __init__( - self, - required_options: list[str] | None = None, - *args, - **kwargs, - ) -> None: + def __init__(self, required_options: list[str] | None = None, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.required_options = required_options or [] @@ -142,14 +137,7 @@ def get_context(self, name, value, attrs): for index, (choice_value, choice_label) in enumerate(self.choices): selected = str(choice_value) in value if value is not None else False context["widget"]["options"].append( - self.create_option( - name, - choice_value, - choice_label, - selected, - index, - attrs=attrs, - ) + self.create_option(name, choice_value, choice_label, selected, index, attrs=attrs) ) context["widget"]["column_names"] = self.column_names diff --git a/rocky/tools/forms/finding_type.py b/rocky/tools/forms/finding_type.py index ee06c19c281..e6fb39e661d 100644 --- a/rocky/tools/forms/finding_type.py +++ b/rocky/tools/forms/finding_type.py @@ -32,14 +32,9 @@ class FindingTypeAddForm(BaseRockyForm): widget=forms.TextInput(attrs={"placeholder": _("Give the finding type a fitting title")}), ) description = forms.CharField( - label=_("Description"), - widget=forms.Textarea(attrs={"placeholder": _("Describe the finding type"), "rows": 3}), - ) - risk = forms.CharField( - label=_("Risk"), - widget=forms.Select(choices=RISK_RATING_CHOICES), - required=False, + label=_("Description"), widget=forms.Textarea(attrs={"placeholder": _("Describe the finding type"), "rows": 3}) ) + risk = forms.CharField(label=_("Risk"), widget=forms.Select(choices=RISK_RATING_CHOICES), required=False) solution = forms.CharField( label=_("Solution"), widget=forms.Textarea(attrs={"placeholder": _("How can this be solved?"), "rows": 3}), @@ -48,12 +43,7 @@ class FindingTypeAddForm(BaseRockyForm): ) references = forms.CharField( label=_("References"), - widget=forms.Textarea( - attrs={ - "placeholder": _("Please give some references on the solution"), - "rows": 3, - } - ), + widget=forms.Textarea(attrs={"placeholder": _("Please give some references on the solution"), "rows": 3}), required=False, help_text=_("Please give sources and references on the suggested solution"), ) @@ -63,19 +53,13 @@ class FindingTypeAddForm(BaseRockyForm): required=False, ) solution_chance = forms.CharField( - label=_("Solution chance"), - widget=forms.Select(choices=PIE_SCALE_CHOICES), - required=False, + label=_("Solution chance"), widget=forms.Select(choices=PIE_SCALE_CHOICES), required=False ) solution_impact = forms.CharField( - label=_("Solution impact"), - widget=forms.Select(choices=PIE_SCALE_CHOICES), - required=False, + label=_("Solution impact"), widget=forms.Select(choices=PIE_SCALE_CHOICES), required=False ) solution_effort = forms.CharField( - label=_("Solution effort"), - widget=forms.Select(choices=PIE_SCALE_EFFORT_CHOICES), - required=False, + label=_("Solution effort"), widget=forms.Select(choices=PIE_SCALE_EFFORT_CHOICES), required=False ) def clean_id(self): @@ -95,8 +79,7 @@ def check_finding_type_existence(self, finding_type_id): class FindingAddForm(BaseRockyForm): ooi_id = forms.CharField( - label="OOI", - widget=DataListInput(attrs={"placeholder": _("Click to select one of the available options")}), + label="OOI", widget=DataListInput(attrs={"placeholder": _("Click to select one of the available options")}) ) finding_type_ids = forms.CharField( label=_("Finding types"), @@ -117,17 +100,11 @@ class FindingAddForm(BaseRockyForm): required=False, ) description = forms.CharField( - label=_("Description"), - widget=forms.Textarea(attrs={"placeholder": _("Describe your finding"), "rows": 3}), + label=_("Description"), widget=forms.Textarea(attrs={"placeholder": _("Describe your finding"), "rows": 3}) ) reproduce = forms.CharField( label=_("Reproduce finding"), - widget=forms.Textarea( - attrs={ - "placeholder": _("Please explain how to reproduce your finding"), - "rows": 3, - } - ), + widget=forms.Textarea(attrs={"placeholder": _("Please explain how to reproduce your finding"), "rows": 3}), required=False, ) date = forms.DateTimeField( @@ -137,13 +114,7 @@ class FindingAddForm(BaseRockyForm): help_text=FINDING_DATETIME_HELP_TEXT, ) - def __init__( - self, - connector: OctopoesAPIConnector, - ooi_list: list[tuple[str, str]], - *args, - **kwargs, - ): + def __init__(self, connector: OctopoesAPIConnector, ooi_list: list[tuple[str, str]], *args, **kwargs): self.octopoes_connector = connector super().__init__(*args, **kwargs) self.set_choices_for_widget("ooi_id", ooi_list) diff --git a/rocky/tools/forms/ooi.py b/rocky/tools/forms/ooi.py index 849862f46af..0f3d6353dfc 100644 --- a/rocky/tools/forms/ooi.py +++ b/rocky/tools/forms/ooi.py @@ -10,20 +10,12 @@ class OOIReportSettingsForm(ObservedAtForm): depth = forms.IntegerField( - initial=DEPTH_DEFAULT, - min_value=1, - max_value=DEPTH_MAX, - required=False, - help_text=DEPTH_HELP_TEXT, + initial=DEPTH_DEFAULT, min_value=1, max_value=DEPTH_MAX, required=False, help_text=DEPTH_HELP_TEXT ) class OoiTreeSettingsForm(OOIReportSettingsForm): - ooi_type = forms.MultipleChoiceField( - label=_("Filter types"), - widget=forms.CheckboxSelectMultiple(), - required=False, - ) + ooi_type = forms.MultipleChoiceField(label=_("Filter types"), widget=forms.CheckboxSelectMultiple(), required=False) def __init__(self, ooi_types: list[str], *args, **kwargs): super().__init__(*args, **kwargs) @@ -50,14 +42,7 @@ class SelectOOIForm(BaseRockyForm): ), ) - def __init__( - self, - oois: list[OOI], - organization_code: str, - mandatory_fields: list | None = None, - *args, - **kwargs, - ): + def __init__(self, oois: list[OOI], organization_code: str, mandatory_fields: list | None = None, *args, **kwargs): super().__init__(*args, **kwargs) self.fields["ooi"].widget.attrs["organization_code"] = organization_code if mandatory_fields: @@ -68,28 +53,19 @@ def __init__( @staticmethod def _to_choice(ooi: OOI) -> tuple[str, Any]: - return str(ooi), ( - ooi, - ooi, - ooi.scan_profile.level if ooi.scan_profile else 0, - ) + return str(ooi), (ooi, ooi, ooi.scan_profile.level if ooi.scan_profile else 0) class SelectOOIFilterForm(BaseRockyForm): show_all = forms.NullBooleanField( label=_("Show objects that don't meet the Boefjes scan level."), - widget=forms.CheckboxInput( - attrs={"class": "submit-on-click"}, - ), + widget=forms.CheckboxInput(attrs={"class": "submit-on-click"}), ) class PossibleBoefjesFilterForm(BaseRockyForm): show_all = forms.NullBooleanField( - widget=LabeledCheckboxInput( - label=_("Show Boefjes that exceed the objects clearance level."), - autosubmit=True, - ), + widget=LabeledCheckboxInput(label=_("Show Boefjes that exceed the objects clearance level."), autosubmit=True) ) @@ -99,17 +75,8 @@ class SetClearanceLevelForm(forms.Form): help_text=_( "Boefjes that has a scan level below or equal to the clearance level, is permitted to scan an object." ), - error_messages={ - "level": { - "required": _("Please select a clearance level to proceed."), - }, - }, - widget=forms.Select( - choices=SCAN_LEVEL_CHOICES, - attrs={ - "aria-describedby": _("explanation-clearance-level"), - }, - ), + error_messages={"level": {"required": _("Please select a clearance level to proceed.")}}, + widget=forms.Select(choices=SCAN_LEVEL_CHOICES, attrs={"aria-describedby": _("explanation-clearance-level")}), ) diff --git a/rocky/tools/forms/ooi_form.py b/rocky/tools/forms/ooi_form.py index 372a26164ec..69555f3b7df 100644 --- a/rocky/tools/forms/ooi_form.py +++ b/rocky/tools/forms/ooi_form.py @@ -35,10 +35,7 @@ def clean(self): def get_fields(self) -> dict[str, forms.fields.Field]: return self.generate_form_fields() - def generate_form_fields( - self, - hidden_ooi_fields: dict[str, str] | None = None, - ) -> dict[str, forms.fields.Field]: + def generate_form_fields(self, hidden_ooi_fields: dict[str, str] | None = None) -> dict[str, forms.fields.Field]: fields = {} for name, field in self.ooi_class.model_fields.items(): annotation = field.annotation @@ -62,11 +59,7 @@ def generate_form_fields( fields[name] = forms.CharField(widget=forms.HiddenInput()) elif name in get_relations(self.ooi_class): fields[name] = generate_select_ooi_field( - self.api_connector, - name, - field, - get_relations(self.ooi_class)[name], - self.initial.get(name, None), + self.api_connector, name, field, get_relations(self.ooi_class)[name], self.initial.get(name, None) ) elif annotation in [IPv4Address, IPv6Address]: fields[name] = generate_ip_field(field) @@ -157,23 +150,16 @@ def generate_url_field(field: FieldInfo) -> forms.fields.Field: def default_field_options(name: str, field_info: FieldInfo) -> dict[str, str | bool]: - return { - "label": name, - "required": field_info.is_required(), - } + return {"label": name, "required": field_info.is_required()} class ClearanceFilterForm(BaseRockyForm): clearance_level = forms.CharField( - label=_("Filter by clearance level"), - widget=CheckboxGroup(choices=SCAN_LEVEL.choices), - required=False, + label=_("Filter by clearance level"), widget=CheckboxGroup(choices=SCAN_LEVEL.choices), required=False ) clearance_type = forms.CharField( - label=_("Filter by clearance type"), - widget=CheckboxGroup(choices=CLEARANCE_TYPE_CHOICES), - required=False, + label=_("Filter by clearance type"), widget=CheckboxGroup(choices=CLEARANCE_TYPE_CHOICES), required=False ) @@ -192,8 +178,5 @@ class ClearanceFilterForm(BaseRockyForm): class OOITypeMultiCheckboxForm(BaseRockyForm): ooi_type = forms.MultipleChoiceField( - label=_("Filter by OOI types"), - required=False, - choices=OOI_TYPE_CHOICES, - widget=forms.CheckboxSelectMultiple, + label=_("Filter by OOI types"), required=False, choices=OOI_TYPE_CHOICES, widget=forms.CheckboxSelectMultiple ) diff --git a/rocky/tools/forms/settings.py b/rocky/tools/forms/settings.py index 4774f022942..4ad15808b8f 100644 --- a/rocky/tools/forms/settings.py +++ b/rocky/tools/forms/settings.py @@ -30,18 +30,9 @@ ("high", _("high")), ] -PIE_SCALE_CHOICES: Choices = [ - BLANK_CHOICE, - ("low", _("low")), - ("medium", _("medium")), - ("high", _("high")), -] +PIE_SCALE_CHOICES: Choices = [BLANK_CHOICE, ("low", _("low")), ("medium", _("medium")), ("high", _("high"))] -CLEARANCE_TYPE_CHOICES = [ - ("declared", _("Declared")), - ("inherited", _("Inherited")), - ("empty", _("Empty")), -] +CLEARANCE_TYPE_CHOICES = [("declared", _("Declared")), ("inherited", _("Inherited")), ("empty", _("Empty"))] SCAN_LEVEL_CHOICES = [BLANK_CHOICE] + SCAN_LEVEL.choices MANUAL_FINDING_ID_PREFIX = "KAT-" diff --git a/rocky/tools/forms/upload_csv.py b/rocky/tools/forms/upload_csv.py index 2324d9fcda8..2437d105350 100644 --- a/rocky/tools/forms/upload_csv.py +++ b/rocky/tools/forms/upload_csv.py @@ -14,9 +14,7 @@ class UploadCSVForm(forms.Form): csv_file = forms.FileField( - label=_("Upload CSV file"), - help_text=_("Only accepts CSV file."), - allow_empty_file=False, + label=_("Upload CSV file"), help_text=_("Only accepts CSV file."), allow_empty_file=False ) def clean_csv_file(self): diff --git a/rocky/tools/forms/upload_raw.py b/rocky/tools/forms/upload_raw.py index f84802157ab..5cd899d606c 100644 --- a/rocky/tools/forms/upload_raw.py +++ b/rocky/tools/forms/upload_raw.py @@ -17,10 +17,7 @@ class UploadRawForm(forms.Form): required=True, widget=forms.TextInput(attrs={"placeholder": "text/html, image/jpeg, ..."}), ) - raw_file = forms.FileField( - label=_("Upload raw file"), - allow_empty_file=False, - ) + raw_file = forms.FileField(label=_("Upload raw file"), allow_empty_file=False) def clean_mime_types(self) -> set[str]: mime_types = self.cleaned_data["mime_types"] diff --git a/rocky/tools/management/commands/export_migrations.py b/rocky/tools/management/commands/export_migrations.py index 0fc4711db67..76b13d928aa 100644 --- a/rocky/tools/management/commands/export_migrations.py +++ b/rocky/tools/management/commands/export_migrations.py @@ -15,12 +15,7 @@ class Command(BaseCommand): def add_arguments(self, parser: CommandParser) -> None: parser.add_argument("app", action="store", type=str, help="Django app") parser.add_argument("from_id", action="store", type=int, help="Migration id to start from") - parser.add_argument( - "--output-folder", - action="store", - default="export_migrations", - help="Output folder", - ) + parser.add_argument("--output-folder", action="store", default="export_migrations", help="Output folder") def handle(self, **options) -> None: # Get the database we're operating from @@ -35,13 +30,11 @@ def handle(self, **options) -> None: # Find migration record to start from migration_match = MigrationRecorder.Migration.objects.get( - app=options["app"], - name__istartswith=f"{options['from_id']:04d}", + app=options["app"], name__istartswith=f"{options['from_id']:04d}" ) migrations_to_export = MigrationRecorder.Migration.objects.filter( - id__gte=migration_match.id, - app=options["app"], + id__gte=migration_match.id, app=options["app"] ) for migration in migrations_to_export: diff --git a/rocky/tools/management/commands/generate_report.py b/rocky/tools/management/commands/generate_report.py index 8782885979b..1280e2bd0e0 100644 --- a/rocky/tools/management/commands/generate_report.py +++ b/rocky/tools/management/commands/generate_report.py @@ -76,11 +76,7 @@ def handle(self, *args, **options): @staticmethod def get_findings_metadata(organization, valid_time, severities) -> list[dict[str, Any]]: - findings = FindingList( - OctopoesAPIConnector(settings.OCTOPOES_API, organization.code), - valid_time, - severities, - ) + findings = FindingList(OctopoesAPIConnector(settings.OCTOPOES_API, organization.code), valid_time, severities) return generate_findings_metadata(findings, severities) diff --git a/rocky/tools/management/commands/setup_test_users.py b/rocky/tools/management/commands/setup_test_users.py index cdb7973c5a1..696914757f2 100644 --- a/rocky/tools/management/commands/setup_test_users.py +++ b/rocky/tools/management/commands/setup_test_users.py @@ -32,11 +32,7 @@ def add_superuser(email: str, password: str): def add_test_user(email: str, password: str, group_name: str | None = None): - user_kwargs: dict[str, str | bool] = { - "email": email, - "password": password, - "full_name": "End-to-end user", - } + user_kwargs: dict[str, str | bool] = {"email": email, "password": password, "full_name": "End-to-end user"} add_user(user_kwargs, group_name) @@ -79,10 +75,7 @@ def add_user(user_kwargs: dict[str, str | bool], group_name: str | None = None): organization, created = Organization.objects.get_or_create(code="_dev") - organizationmember, created = OrganizationMember.objects.get_or_create( - user=user, - organization=organization, - ) + organizationmember, created = OrganizationMember.objects.get_or_create(user=user, organization=organization) if created: organizationmember.status = OrganizationMember.STATUSES.ACTIVE diff --git a/rocky/tools/migrations/0001_initial.py b/rocky/tools/migrations/0001_initial.py index e48cdb29e37..6bb80e59c86 100644 --- a/rocky/tools/migrations/0001_initial.py +++ b/rocky/tools/migrations/0001_initial.py @@ -10,51 +10,26 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] + dependencies = [migrations.swappable_dependency(settings.AUTH_USER_MODEL)] operations = [ migrations.CreateModel( name="Organization", fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("name", models.CharField(max_length=126)), - ( - "octopoes_host", - models.CharField(default=None, max_length=126, null=True), - ), + ("octopoes_host", models.CharField(default=None, max_length=126, null=True)), ], ), migrations.CreateModel( name="OrganizationMember", fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("authorized", models.BooleanField(default=False)), ( "status", models.CharField( - choices=[ - ("active", "active"), - ("new", "new"), - ("blocked", "blocked"), - ], + choices=[("active", "active"), ("new", "new"), ("blocked", "blocked")], default="new", max_length=64, ), @@ -64,33 +39,18 @@ class Migration(migrations.Migration): ("goal", models.CharField(max_length=256)), ( "organization", - models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to="tools.organization", - ), + models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to="tools.organization"), ), ( "user", - models.OneToOneField( - on_delete=django.db.models.deletion.DO_NOTHING, - to=settings.AUTH_USER_MODEL, - ), + models.OneToOneField(on_delete=django.db.models.deletion.DO_NOTHING, to=settings.AUTH_USER_MODEL), ), ], ), migrations.CreateModel( name="Job", fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - ), - ), + ("id", models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ("tool_module", models.CharField(max_length=128)), ("arguments", models.JSONField()), ("dispatches", models.JSONField(default=dict)), @@ -98,9 +58,7 @@ class Migration(migrations.Migration): ( "user", models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, + null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL ), ), ], @@ -108,29 +66,15 @@ class Migration(migrations.Migration): migrations.CreateModel( name="Indemnification", fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ( "organization", - models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to="tools.organization", - ), + models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to="tools.organization"), ), ( "user", models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, + null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL ), ), ], diff --git a/rocky/tools/migrations/0001_squashed_0041_merge_20230731_1131.py b/rocky/tools/migrations/0001_squashed_0041_merge_20230731_1131.py index 7c0d8750c48..f1d181981d3 100644 --- a/rocky/tools/migrations/0001_squashed_0041_merge_20230731_1131.py +++ b/rocky/tools/migrations/0001_squashed_0041_merge_20230731_1131.py @@ -135,11 +135,7 @@ class Migration(migrations.Migration): ), ), ], - options={ - "ordering": ("name",), - "abstract": False, - "unique_together": {("slug", "parent")}, - }, + options={"ordering": ("name",), "abstract": False, "unique_together": {("slug", "parent")}}, bases=(tagulous.models.models.BaseTagTreeModel, models.Model), ), migrations.CreateModel( @@ -252,8 +248,6 @@ class Migration(migrations.Migration): ("blocked", models.BooleanField(default=False)), ("groups", models.ManyToManyField(blank=True, to="auth.group")), ], - options={ - "unique_together": {("user", "organization")}, - }, + options={"unique_together": {("user", "organization")}}, ), ] diff --git a/rocky/tools/migrations/0002_alter_organization_octopoes_host.py b/rocky/tools/migrations/0002_alter_organization_octopoes_host.py index 83b62b018ad..06050cfb88f 100644 --- a/rocky/tools/migrations/0002_alter_organization_octopoes_host.py +++ b/rocky/tools/migrations/0002_alter_organization_octopoes_host.py @@ -4,14 +4,12 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0001_initial"), - ] + dependencies = [("tools", "0001_initial")] operations = [ migrations.AlterField( model_name="organization", name="octopoes_host", field=models.CharField(default=None, max_length=126, null=True, unique=True), - ), + ) ] diff --git a/rocky/tools/migrations/0003_change_orgazation_host_to_code.py b/rocky/tools/migrations/0003_change_orgazation_host_to_code.py index dc41e99ca7d..22643d5b6b2 100644 --- a/rocky/tools/migrations/0003_change_orgazation_host_to_code.py +++ b/rocky/tools/migrations/0003_change_orgazation_host_to_code.py @@ -5,23 +5,14 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0002_alter_organization_octopoes_host"), - ] + dependencies = [("tools", "0002_alter_organization_octopoes_host")] operations = [ - migrations.RemoveField( - model_name="organization", - name="octopoes_host", - ), + migrations.RemoveField(model_name="organization", name="octopoes_host"), migrations.AddField( model_name="job", name="organization", - field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to="tools.organization", - ), + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to="tools.organization"), ), migrations.AddField( model_name="organization", diff --git a/rocky/tools/migrations/0004_ooiinformation.py b/rocky/tools/migrations/0004_ooiinformation.py index bd52ad768f0..b743ab491bb 100644 --- a/rocky/tools/migrations/0004_ooiinformation.py +++ b/rocky/tools/migrations/0004_ooiinformation.py @@ -4,21 +4,16 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0003_change_orgazation_host_to_code"), - ] + dependencies = [("tools", "0003_change_orgazation_host_to_code")] operations = [ migrations.CreateModel( name="OOIInformation", fields=[ - ( - "id", - models.CharField(max_length=256, primary_key=True, serialize=False), - ), + ("id", models.CharField(max_length=256, primary_key=True, serialize=False)), ("last_updated", models.DateTimeField(auto_now=True)), ("data", models.JSONField(null=True)), ("consult_api", models.BooleanField(default=False)), ], - ), + ) ] diff --git a/rocky/tools/migrations/0005_scanprofile.py b/rocky/tools/migrations/0005_scanprofile.py index 9ea97a7de9a..b2ae2b9303f 100644 --- a/rocky/tools/migrations/0005_scanprofile.py +++ b/rocky/tools/migrations/0005_scanprofile.py @@ -6,39 +6,24 @@ class Migration(migrations.Migration): - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ("tools", "0004_ooiinformation"), - ] + dependencies = [migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("tools", "0004_ooiinformation")] operations = [ migrations.CreateModel( name="ScanProfile", fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("reference", models.CharField(max_length=256)), ( "level", models.PositiveSmallIntegerField( - choices=[(0, "L0"), (1, "L1"), (2, "L2"), (3, "L3"), (4, "L4")], - default=0, + choices=[(0, "L0"), (1, "L1"), (2, "L2"), (3, "L3"), (4, "L4")], default=0 ), ), ("new", models.BooleanField(default=True)), ( "organization", - models.ForeignKey( - on_delete=django.db.models.deletion.DO_NOTHING, - to="tools.organization", - ), + models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to="tools.organization"), ), ( "user", @@ -50,8 +35,6 @@ class Migration(migrations.Migration): ), ), ], - options={ - "unique_together": {("reference", "organization")}, - }, - ), + options={"unique_together": {("reference", "organization")}}, + ) ] diff --git a/rocky/tools/migrations/0006_alter_organization_name.py b/rocky/tools/migrations/0006_alter_organization_name.py index 2ee0b8b8a59..6d5c7c803c3 100644 --- a/rocky/tools/migrations/0006_alter_organization_name.py +++ b/rocky/tools/migrations/0006_alter_organization_name.py @@ -4,14 +4,10 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0005_scanprofile"), - ] + dependencies = [("tools", "0005_scanprofile")] operations = [ migrations.AlterField( - model_name="organization", - name="name", - field=models.CharField(max_length=126, unique=True), - ), + model_name="organization", name="name", field=models.CharField(max_length=126, unique=True) + ) ] diff --git a/rocky/tools/migrations/0007_update_job.py b/rocky/tools/migrations/0007_update_job.py index 2238aef8247..7596be3f514 100644 --- a/rocky/tools/migrations/0007_update_job.py +++ b/rocky/tools/migrations/0007_update_job.py @@ -4,19 +4,9 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0006_alter_organization_name"), - ] + dependencies = [("tools", "0006_alter_organization_name")] operations = [ - migrations.RenameField( - model_name="job", - old_name="created_at", - new_name="created", - ), - migrations.RenameField( - model_name="job", - old_name="tool_module", - new_name="module", - ), + migrations.RenameField(model_name="job", old_name="created_at", new_name="created"), + migrations.RenameField(model_name="job", old_name="tool_module", new_name="module"), ] diff --git a/rocky/tools/migrations/0008_organizationmember_verified.py b/rocky/tools/migrations/0008_organizationmember_verified.py index edc626bb552..aee6618d822 100644 --- a/rocky/tools/migrations/0008_organizationmember_verified.py +++ b/rocky/tools/migrations/0008_organizationmember_verified.py @@ -4,14 +4,8 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0007_update_job"), - ] + dependencies = [("tools", "0007_update_job")] operations = [ - migrations.AddField( - model_name="organizationmember", - name="verified", - field=models.BooleanField(default=False), - ), + migrations.AddField(model_name="organizationmember", name="verified", field=models.BooleanField(default=False)) ] diff --git a/rocky/tools/migrations/0009_scanprofile_is_source_ooi.py b/rocky/tools/migrations/0009_scanprofile_is_source_ooi.py index 40067f9bdb2..9e18b8eef9f 100644 --- a/rocky/tools/migrations/0009_scanprofile_is_source_ooi.py +++ b/rocky/tools/migrations/0009_scanprofile_is_source_ooi.py @@ -4,14 +4,8 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0008_organizationmember_verified"), - ] + dependencies = [("tools", "0008_organizationmember_verified")] operations = [ - migrations.AddField( - model_name="scanprofile", - name="is_source_ooi", - field=models.BooleanField(default=False), - ), + migrations.AddField(model_name="scanprofile", name="is_source_ooi", field=models.BooleanField(default=False)) ] diff --git a/rocky/tools/migrations/0010_alter_scanprofile_reference.py b/rocky/tools/migrations/0010_alter_scanprofile_reference.py index ef1bea89fff..e84bc5a51e6 100644 --- a/rocky/tools/migrations/0010_alter_scanprofile_reference.py +++ b/rocky/tools/migrations/0010_alter_scanprofile_reference.py @@ -4,14 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0009_scanprofile_is_source_ooi"), - ] + dependencies = [("tools", "0009_scanprofile_is_source_ooi")] - operations = [ - migrations.AlterField( - model_name="scanprofile", - name="reference", - field=models.TextField(), - ), - ] + operations = [migrations.AlterField(model_name="scanprofile", name="reference", field=models.TextField())] diff --git a/rocky/tools/migrations/0011_job_input_ooi.py b/rocky/tools/migrations/0011_job_input_ooi.py index 5c6273892d9..7cb17e17517 100644 --- a/rocky/tools/migrations/0011_job_input_ooi.py +++ b/rocky/tools/migrations/0011_job_input_ooi.py @@ -4,14 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0010_alter_scanprofile_reference"), - ] + dependencies = [("tools", "0010_alter_scanprofile_reference")] - operations = [ - migrations.AddField( - model_name="job", - name="input_ooi", - field=models.TextField(null=True), - ), - ] + operations = [migrations.AddField(model_name="job", name="input_ooi", field=models.TextField(null=True))] diff --git a/rocky/tools/migrations/0012_rename_module_job_boefje_name.py b/rocky/tools/migrations/0012_rename_module_job_boefje_name.py index f1efefdb2cc..5aa60b53094 100644 --- a/rocky/tools/migrations/0012_rename_module_job_boefje_name.py +++ b/rocky/tools/migrations/0012_rename_module_job_boefje_name.py @@ -4,14 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0011_job_input_ooi"), - ] + dependencies = [("tools", "0011_job_input_ooi")] - operations = [ - migrations.RenameField( - model_name="job", - old_name="module", - new_name="boefje_name", - ), - ] + operations = [migrations.RenameField(model_name="job", old_name="module", new_name="boefje_name")] diff --git a/rocky/tools/migrations/0013_boefjeconfig.py b/rocky/tools/migrations/0013_boefjeconfig.py index ef7325df65d..b1959087680 100644 --- a/rocky/tools/migrations/0013_boefjeconfig.py +++ b/rocky/tools/migrations/0013_boefjeconfig.py @@ -5,37 +5,22 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0012_rename_module_job_boefje_name"), - ] + dependencies = [("tools", "0012_rename_module_job_boefje_name")] operations = [ migrations.CreateModel( name="BoefjeConfig", fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("boefje", models.CharField(max_length=128)), ("enabled", models.BooleanField(default=False)), ( "organization", models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.DO_NOTHING, - to="tools.organization", + blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to="tools.organization" ), ), ], - options={ - "unique_together": {("boefje", "organization")}, - }, - ), + options={"unique_together": {("boefje", "organization")}}, + ) ] diff --git a/rocky/tools/migrations/0014_drop_dispatches_field.py b/rocky/tools/migrations/0014_drop_dispatches_field.py index b6d1d3863a0..0e271b67afb 100644 --- a/rocky/tools/migrations/0014_drop_dispatches_field.py +++ b/rocky/tools/migrations/0014_drop_dispatches_field.py @@ -4,18 +4,9 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0013_boefjeconfig"), - ] + dependencies = [("tools", "0013_boefjeconfig")] operations = [ - migrations.RemoveField( - model_name="job", - name="dispatches", - ), - migrations.RenameField( - model_name="job", - old_name="boefje_name", - new_name="boefje_id", - ), + migrations.RemoveField(model_name="job", name="dispatches"), + migrations.RenameField(model_name="job", old_name="boefje_name", new_name="boefje_id"), ] diff --git a/rocky/tools/migrations/0015_alter_job_input_ooi.py b/rocky/tools/migrations/0015_alter_job_input_ooi.py index 2704cc34d0b..c73e6f7a014 100644 --- a/rocky/tools/migrations/0015_alter_job_input_ooi.py +++ b/rocky/tools/migrations/0015_alter_job_input_ooi.py @@ -4,14 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0014_drop_dispatches_field"), - ] + dependencies = [("tools", "0014_drop_dispatches_field")] - operations = [ - migrations.AlterField( - model_name="job", - name="input_ooi", - field=models.TextField(null=True), - ), - ] + operations = [migrations.AlterField(model_name="job", name="input_ooi", field=models.TextField(null=True))] diff --git a/rocky/tools/migrations/0016_organization_signal_fields.py b/rocky/tools/migrations/0016_organization_signal_fields.py index d25d4291765..43c95225e86 100644 --- a/rocky/tools/migrations/0016_organization_signal_fields.py +++ b/rocky/tools/migrations/0016_organization_signal_fields.py @@ -5,9 +5,7 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0015_alter_job_input_ooi"), - ] + dependencies = [("tools", "0015_alter_job_input_ooi")] operations = [ migrations.AddField( diff --git a/rocky/tools/migrations/0017_alter_organizationmember_foreignkey.py b/rocky/tools/migrations/0017_alter_organizationmember_foreignkey.py index 8e034d79306..e27764a3ac2 100644 --- a/rocky/tools/migrations/0017_alter_organizationmember_foreignkey.py +++ b/rocky/tools/migrations/0017_alter_organizationmember_foreignkey.py @@ -5,19 +5,14 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0016_organization_signal_fields"), - ] + dependencies = [("tools", "0016_organization_signal_fields")] operations = [ migrations.AlterField( model_name="organizationmember", name="organization", field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="members", - to="tools.organization", + null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="members", to="tools.organization" ), - ), + ) ] diff --git a/rocky/tools/migrations/0018_alter_boefjeconfig_options.py b/rocky/tools/migrations/0018_alter_boefjeconfig_options.py index ef3445fab4a..c3771c7b940 100644 --- a/rocky/tools/migrations/0018_alter_boefjeconfig_options.py +++ b/rocky/tools/migrations/0018_alter_boefjeconfig_options.py @@ -4,13 +4,11 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0017_alter_organizationmember_foreignkey"), - ] + dependencies = [("tools", "0017_alter_organizationmember_foreignkey")] operations = [ migrations.AlterModelOptions( name="boefjeconfig", options={"permissions": [("can_enable_disable_boefje", "Can enable or disable boefje")]}, - ), + ) ] diff --git a/rocky/tools/migrations/0019_alter_scanprofile_remove_level_and_user.py b/rocky/tools/migrations/0019_alter_scanprofile_remove_level_and_user.py index 293a2559a93..6702b3371a3 100644 --- a/rocky/tools/migrations/0019_alter_scanprofile_remove_level_and_user.py +++ b/rocky/tools/migrations/0019_alter_scanprofile_remove_level_and_user.py @@ -4,21 +4,10 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0018_alter_boefjeconfig_options"), - ] + dependencies = [("tools", "0018_alter_boefjeconfig_options")] operations = [ - migrations.RemoveField( - model_name="job", - name="user", - ), - migrations.RemoveField( - model_name="scanprofile", - name="level", - ), - migrations.RemoveField( - model_name="scanprofile", - name="user", - ), + migrations.RemoveField(model_name="job", name="user"), + migrations.RemoveField(model_name="scanprofile", name="level"), + migrations.RemoveField(model_name="scanprofile", name="user"), ] diff --git a/rocky/tools/migrations/0020_auto_20220524_1324.py b/rocky/tools/migrations/0020_auto_20220524_1324.py index f9f2d77fb31..2f851530355 100644 --- a/rocky/tools/migrations/0020_auto_20220524_1324.py +++ b/rocky/tools/migrations/0020_auto_20220524_1324.py @@ -4,9 +4,7 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0019_alter_scanprofile_remove_level_and_user"), - ] + dependencies = [("tools", "0019_alter_scanprofile_remove_level_and_user")] operations = [ migrations.AlterModelOptions( @@ -18,13 +16,8 @@ class Migration(migrations.Migration): ) }, ), - migrations.RemoveField( - model_name="scanprofile", - name="is_source_ooi", - ), + migrations.RemoveField(model_name="scanprofile", name="is_source_ooi"), migrations.AddField( - model_name="organizationmember", - name="onboarded", - field=models.BooleanField(default=False), + model_name="organizationmember", name="onboarded", field=models.BooleanField(default=False) ), ] diff --git a/rocky/tools/migrations/0021_delete_boefjeconfig.py b/rocky/tools/migrations/0021_delete_boefjeconfig.py index 4d81f06cf71..5fd66415dd5 100644 --- a/rocky/tools/migrations/0021_delete_boefjeconfig.py +++ b/rocky/tools/migrations/0021_delete_boefjeconfig.py @@ -4,12 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0020_auto_20220524_1324"), - ] - - operations = [ - migrations.DeleteModel( - name="BoefjeConfig", - ), - ] + dependencies = [("tools", "0020_auto_20220524_1324")] + + operations = [migrations.DeleteModel(name="BoefjeConfig")] diff --git a/rocky/tools/migrations/0022_alter_organization_options.py b/rocky/tools/migrations/0022_alter_organization_options.py index 24caeb5a180..f530698addc 100644 --- a/rocky/tools/migrations/0022_alter_organization_options.py +++ b/rocky/tools/migrations/0022_alter_organization_options.py @@ -4,9 +4,7 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0021_delete_boefjeconfig"), - ] + dependencies = [("tools", "0021_delete_boefjeconfig")] operations = [ migrations.AlterModelOptions( @@ -18,5 +16,5 @@ class Migration(migrations.Migration): ("can_enable_disable_boefje", "Can enable or disable boefje"), ) }, - ), + ) ] diff --git a/rocky/tools/migrations/0023_delete_scanprofile.py b/rocky/tools/migrations/0023_delete_scanprofile.py index c8230d63144..9df04f11c8c 100644 --- a/rocky/tools/migrations/0023_delete_scanprofile.py +++ b/rocky/tools/migrations/0023_delete_scanprofile.py @@ -4,12 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0022_alter_organization_options"), - ] - - operations = [ - migrations.DeleteModel( - name="ScanProfile", - ), - ] + dependencies = [("tools", "0022_alter_organization_options")] + + operations = [migrations.DeleteModel(name="ScanProfile")] diff --git a/rocky/tools/migrations/0024_auto_20221005_1251.py b/rocky/tools/migrations/0024_auto_20221005_1251.py index b191a4d248b..528204886d9 100644 --- a/rocky/tools/migrations/0024_auto_20221005_1251.py +++ b/rocky/tools/migrations/0024_auto_20221005_1251.py @@ -5,9 +5,7 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0023_delete_scanprofile"), - ] + dependencies = [("tools", "0023_delete_scanprofile")] operations = [ migrations.AddField( diff --git a/rocky/tools/migrations/0025_auto_20221027_1233.py b/rocky/tools/migrations/0025_auto_20221027_1233.py index db343e73c61..ebf7d9ccbfd 100644 --- a/rocky/tools/migrations/0025_auto_20221027_1233.py +++ b/rocky/tools/migrations/0025_auto_20221027_1233.py @@ -5,9 +5,7 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0024_auto_20221005_1251"), - ] + dependencies = [("tools", "0024_auto_20221005_1251")] operations = [ migrations.AlterModelOptions( diff --git a/rocky/tools/migrations/0026_auto_20221031_1344.py b/rocky/tools/migrations/0026_auto_20221031_1344.py index 68bff706109..46b3620a474 100644 --- a/rocky/tools/migrations/0026_auto_20221031_1344.py +++ b/rocky/tools/migrations/0026_auto_20221031_1344.py @@ -24,12 +24,6 @@ def add_group_permissions(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ - ("tools", "0025_auto_20221027_1233"), - ("contenttypes", "0002_remove_content_type_name"), - ] - - operations = [ - migrations.RunPython(migrate_permissions), - migrations.RunPython(add_group_permissions), - ] + dependencies = [("tools", "0025_auto_20221027_1233"), ("contenttypes", "0002_remove_content_type_name")] + + operations = [migrations.RunPython(migrate_permissions), migrations.RunPython(add_group_permissions)] diff --git a/rocky/tools/migrations/0027_auto_20230103_1721.py b/rocky/tools/migrations/0027_auto_20230103_1721.py index 38f65376620..81299a9bbec 100644 --- a/rocky/tools/migrations/0027_auto_20230103_1721.py +++ b/rocky/tools/migrations/0027_auto_20230103_1721.py @@ -9,19 +9,11 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0026_auto_20221031_1344"), - ] + dependencies = [("tools", "0026_auto_20221031_1344")] operations = [ - migrations.RemoveField( - model_name="organization", - name="signal_group_id", - ), - migrations.RemoveField( - model_name="organization", - name="signal_username", - ), + migrations.RemoveField(model_name="organization", name="signal_group_id"), + migrations.RemoveField(model_name="organization", name="signal_username"), migrations.AlterField( model_name="organization", name="code", @@ -100,11 +92,7 @@ class Migration(migrations.Migration): ), ), ], - options={ - "ordering": ("name",), - "abstract": False, - "unique_together": {("slug", "parent")}, - }, + options={"ordering": ("name",), "abstract": False, "unique_together": {("slug", "parent")}}, bases=(tagulous.models.models.BaseTagTreeModel, models.Model), ), migrations.AddField( diff --git a/rocky/tools/migrations/0028_auto_20230117_1242.py b/rocky/tools/migrations/0028_auto_20230117_1242.py index 86b7bdc7109..9d24ee68136 100644 --- a/rocky/tools/migrations/0028_auto_20230117_1242.py +++ b/rocky/tools/migrations/0028_auto_20230117_1242.py @@ -6,10 +6,7 @@ class Migration(migrations.Migration): - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ("tools", "0027_auto_20230103_1721"), - ] + dependencies = [migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("tools", "0027_auto_20230103_1721")] operations = [ migrations.AlterField( @@ -24,20 +21,8 @@ class Migration(migrations.Migration): name="user", field=models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to=settings.AUTH_USER_MODEL), ), - migrations.AlterUniqueTogether( - name="organizationmember", - unique_together={("user", "organization")}, - ), - migrations.RemoveField( - model_name="organizationmember", - name="goal", - ), - migrations.RemoveField( - model_name="organizationmember", - name="member_role", - ), - migrations.RemoveField( - model_name="organizationmember", - name="signal_username", - ), + migrations.AlterUniqueTogether(name="organizationmember", unique_together={("user", "organization")}), + migrations.RemoveField(model_name="organizationmember", name="goal"), + migrations.RemoveField(model_name="organizationmember", name="member_role"), + migrations.RemoveField(model_name="organizationmember", name="signal_username"), ] diff --git a/rocky/tools/migrations/0029_alter_organizationtag_color.py b/rocky/tools/migrations/0029_alter_organizationtag_color.py index ee5add7bcf5..a665afbb7d0 100644 --- a/rocky/tools/migrations/0029_alter_organizationtag_color.py +++ b/rocky/tools/migrations/0029_alter_organizationtag_color.py @@ -4,9 +4,7 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0028_auto_20230117_1242"), - ] + dependencies = [("tools", "0028_auto_20230117_1242")] operations = [ migrations.AlterField( @@ -36,5 +34,5 @@ class Migration(migrations.Migration): default="color-1-light", max_length=20, ), - ), + ) ] diff --git a/rocky/tools/migrations/0029_set_user_full_name.py b/rocky/tools/migrations/0029_set_user_full_name.py index 8073d9a589b..fa9e092babe 100644 --- a/rocky/tools/migrations/0029_set_user_full_name.py +++ b/rocky/tools/migrations/0029_set_user_full_name.py @@ -14,9 +14,6 @@ def set_full_name(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ - ("account", "0001_initial"), - ("tools", "0028_auto_20230117_1242"), - ] + dependencies = [("account", "0001_initial"), ("tools", "0028_auto_20230117_1242")] operations = [migrations.RunPython(set_full_name)] diff --git a/rocky/tools/migrations/0030_auto_20230227_1458.py b/rocky/tools/migrations/0030_auto_20230227_1458.py index 36d527e401b..67d6b8e12f6 100644 --- a/rocky/tools/migrations/0030_auto_20230227_1458.py +++ b/rocky/tools/migrations/0030_auto_20230227_1458.py @@ -6,24 +6,12 @@ class Migration(migrations.Migration): - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ("tools", "0029_set_user_full_name"), - ] + dependencies = [migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("tools", "0029_set_user_full_name")] operations = [ - migrations.RemoveField( - model_name="organizationmember", - name="authorized", - ), - migrations.RemoveField( - model_name="organizationmember", - name="member_name", - ), - migrations.RemoveField( - model_name="organizationmember", - name="verified", - ), + migrations.RemoveField(model_name="organizationmember", name="authorized"), + migrations.RemoveField(model_name="organizationmember", name="member_name"), + migrations.RemoveField(model_name="organizationmember", name="verified"), migrations.AlterField( model_name="organizationmember", name="organization", diff --git a/rocky/tools/migrations/0031_merge_20230301_2012.py b/rocky/tools/migrations/0031_merge_20230301_2012.py index 124d44f49df..99d23979399 100644 --- a/rocky/tools/migrations/0031_merge_20230301_2012.py +++ b/rocky/tools/migrations/0031_merge_20230301_2012.py @@ -4,9 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0029_alter_organizationtag_color"), - ("tools", "0030_auto_20230227_1458"), - ] + dependencies = [("tools", "0029_alter_organizationtag_color"), ("tools", "0030_auto_20230227_1458")] operations: list = [] diff --git a/rocky/tools/migrations/0032_alter_organizationmember_user.py b/rocky/tools/migrations/0032_alter_organizationmember_user.py index d71b4edbfb1..9e094f35d2b 100644 --- a/rocky/tools/migrations/0032_alter_organizationmember_user.py +++ b/rocky/tools/migrations/0032_alter_organizationmember_user.py @@ -6,10 +6,7 @@ class Migration(migrations.Migration): - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ("tools", "0031_merge_20230301_2012"), - ] + dependencies = [migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("tools", "0031_merge_20230301_2012")] operations = [ migrations.AlterField( @@ -18,5 +15,5 @@ class Migration(migrations.Migration): field=models.ForeignKey( on_delete=django.db.models.deletion.PROTECT, related_name="members", to=settings.AUTH_USER_MODEL ), - ), + ) ] diff --git a/rocky/tools/migrations/0033_alter_organization_options.py b/rocky/tools/migrations/0033_alter_organization_options.py index 99e9c75017b..919b41c32a2 100644 --- a/rocky/tools/migrations/0033_alter_organization_options.py +++ b/rocky/tools/migrations/0033_alter_organization_options.py @@ -4,9 +4,7 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0032_alter_organizationmember_user"), - ] + dependencies = [("tools", "0032_alter_organizationmember_user")] operations = [ migrations.AlterModelOptions( @@ -20,5 +18,5 @@ class Migration(migrations.Migration): ("can_recalculate_bits", "Can recalculate bits"), ) }, - ), + ) ] diff --git a/rocky/tools/migrations/0033_auto_20230407_1113.py b/rocky/tools/migrations/0033_auto_20230407_1113.py index 87229fc3cbf..441298bc2fc 100644 --- a/rocky/tools/migrations/0033_auto_20230407_1113.py +++ b/rocky/tools/migrations/0033_auto_20230407_1113.py @@ -9,16 +9,10 @@ def migrate_organizationmember_status_blocked_to_blocked_attribute(apps, schema_ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0032_alter_organizationmember_user"), - ] + dependencies = [("tools", "0032_alter_organizationmember_user")] operations = [ - migrations.AddField( - model_name="organizationmember", - name="blocked", - field=models.BooleanField(default=False), - ), + migrations.AddField(model_name="organizationmember", name="blocked", field=models.BooleanField(default=False)), migrations.RunPython(migrate_organizationmember_status_blocked_to_blocked_attribute), migrations.AlterField( model_name="organizationmember", diff --git a/rocky/tools/migrations/0034_alter_organization_options.py b/rocky/tools/migrations/0034_alter_organization_options.py index a078dcf1a3a..693fa21acb9 100644 --- a/rocky/tools/migrations/0034_alter_organization_options.py +++ b/rocky/tools/migrations/0034_alter_organization_options.py @@ -4,9 +4,7 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0033_auto_20230407_1113"), - ] + dependencies = [("tools", "0033_auto_20230407_1113")] operations = [ migrations.AlterModelOptions( @@ -20,5 +18,5 @@ class Migration(migrations.Migration): ("can_delete_oois", "Can delete oois"), ) }, - ), + ) ] diff --git a/rocky/tools/migrations/0034_organizationmember_groups.py b/rocky/tools/migrations/0034_organizationmember_groups.py index 3f23c7e7a1a..800c56d1fd6 100644 --- a/rocky/tools/migrations/0034_organizationmember_groups.py +++ b/rocky/tools/migrations/0034_organizationmember_groups.py @@ -4,15 +4,10 @@ class Migration(migrations.Migration): - dependencies = [ - ("auth", "0012_alter_user_first_name_max_length"), - ("tools", "0033_auto_20230407_1113"), - ] + dependencies = [("auth", "0012_alter_user_first_name_max_length"), ("tools", "0033_auto_20230407_1113")] operations = [ migrations.AddField( - model_name="organizationmember", - name="groups", - field=models.ManyToManyField(blank=True, to="auth.Group"), - ), + model_name="organizationmember", name="groups", field=models.ManyToManyField(blank=True, to="auth.Group") + ) ] diff --git a/rocky/tools/migrations/0035_update_ooi_delete_perm.py b/rocky/tools/migrations/0035_update_ooi_delete_perm.py index 0bee4614926..588bda76340 100644 --- a/rocky/tools/migrations/0035_update_ooi_delete_perm.py +++ b/rocky/tools/migrations/0035_update_ooi_delete_perm.py @@ -26,12 +26,6 @@ def add_group_permissions(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ - ("tools", "0034_alter_organization_options"), - ("contenttypes", "0002_remove_content_type_name"), - ] - - operations = [ - migrations.RunPython(migrate_permissions), - migrations.RunPython(add_group_permissions), - ] + dependencies = [("tools", "0034_alter_organization_options"), ("contenttypes", "0002_remove_content_type_name")] + + operations = [migrations.RunPython(migrate_permissions), migrations.RunPython(add_group_permissions)] diff --git a/rocky/tools/migrations/0036_merge_20230504_1629.py b/rocky/tools/migrations/0036_merge_20230504_1629.py index 8fd69a1a278..9cc0ed67365 100644 --- a/rocky/tools/migrations/0036_merge_20230504_1629.py +++ b/rocky/tools/migrations/0036_merge_20230504_1629.py @@ -4,9 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0035_update_ooi_delete_perm"), - ("tools", "0035_update_perms_move_and_clear_groups"), - ] + dependencies = [("tools", "0035_update_ooi_delete_perm"), ("tools", "0035_update_perms_move_and_clear_groups")] operations: list = [] diff --git a/rocky/tools/migrations/0037_alter_organization_options.py b/rocky/tools/migrations/0037_alter_organization_options.py index ee136ab2aa4..a459d29a855 100644 --- a/rocky/tools/migrations/0037_alter_organization_options.py +++ b/rocky/tools/migrations/0037_alter_organization_options.py @@ -4,10 +4,7 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0036_merge_20230504_1629"), - ("tools", "0033_alter_organization_options"), - ] + dependencies = [("tools", "0036_merge_20230504_1629"), ("tools", "0033_alter_organization_options")] operations = [ migrations.AlterModelOptions( @@ -22,5 +19,5 @@ class Migration(migrations.Migration): ("can_recalculate_bits", "Can recalculate bits"), ) }, - ), + ) ] diff --git a/rocky/tools/migrations/0038_alter_organization_options.py b/rocky/tools/migrations/0038_alter_organization_options.py index 093ff75b443..5e9b7c27bfa 100644 --- a/rocky/tools/migrations/0038_alter_organization_options.py +++ b/rocky/tools/migrations/0038_alter_organization_options.py @@ -4,9 +4,7 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0037_alter_organization_options"), - ] + dependencies = [("tools", "0037_alter_organization_options")] operations = [ migrations.AlterModelOptions( @@ -24,5 +22,5 @@ class Migration(migrations.Migration): ("can_recalculate_bits", "Can recalculate bits"), ) }, - ), + ) ] diff --git a/rocky/tools/migrations/0038_delete_job.py b/rocky/tools/migrations/0038_delete_job.py index e18e8856116..2081689615f 100644 --- a/rocky/tools/migrations/0038_delete_job.py +++ b/rocky/tools/migrations/0038_delete_job.py @@ -4,12 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0037_alter_organization_options"), - ] - - operations = [ - migrations.DeleteModel( - name="Job", - ), - ] + dependencies = [("tools", "0037_alter_organization_options")] + + operations = [migrations.DeleteModel(name="Job")] diff --git a/rocky/tools/migrations/0039_merge_0038_alter_organization_options_0038_delete_job.py b/rocky/tools/migrations/0039_merge_0038_alter_organization_options_0038_delete_job.py index 86a01418cdd..204b2a8bc29 100644 --- a/rocky/tools/migrations/0039_merge_0038_alter_organization_options_0038_delete_job.py +++ b/rocky/tools/migrations/0039_merge_0038_alter_organization_options_0038_delete_job.py @@ -4,9 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0038_alter_organization_options"), - ("tools", "0038_delete_job"), - ] + dependencies = [("tools", "0038_alter_organization_options"), ("tools", "0038_delete_job")] operations: list = [] diff --git a/rocky/tools/migrations/0039_update_permissions.py b/rocky/tools/migrations/0039_update_permissions.py index 0147c4ec231..266b6da1cf9 100644 --- a/rocky/tools/migrations/0039_update_permissions.py +++ b/rocky/tools/migrations/0039_update_permissions.py @@ -32,11 +32,6 @@ def add_group_permissions(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ - ("tools", "0039_merge_0038_alter_organization_options_0038_delete_job"), - ] - - operations = [ - migrations.RunPython(migrate_permissions), - migrations.RunPython(add_group_permissions), - ] + dependencies = [("tools", "0039_merge_0038_alter_organization_options_0038_delete_job")] + + operations = [migrations.RunPython(migrate_permissions), migrations.RunPython(add_group_permissions)] diff --git a/rocky/tools/migrations/0040_admin_inherits_red_teamer_permissions.py b/rocky/tools/migrations/0040_admin_inherits_red_teamer_permissions.py index e883314e626..bc20c081cc7 100644 --- a/rocky/tools/migrations/0040_admin_inherits_red_teamer_permissions.py +++ b/rocky/tools/migrations/0040_admin_inherits_red_teamer_permissions.py @@ -36,11 +36,6 @@ def add_group_permissions(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ - ("tools", "0039_update_permissions"), - ] - - operations = [ - migrations.RunPython(migrate_permissions), - migrations.RunPython(add_group_permissions), - ] + dependencies = [("tools", "0039_update_permissions")] + + operations = [migrations.RunPython(migrate_permissions), migrations.RunPython(add_group_permissions)] diff --git a/rocky/tools/migrations/0040_update_admin_permission.py b/rocky/tools/migrations/0040_update_admin_permission.py index 84a507b8d33..ee396eb02d9 100644 --- a/rocky/tools/migrations/0040_update_admin_permission.py +++ b/rocky/tools/migrations/0040_update_admin_permission.py @@ -23,11 +23,6 @@ def add_admin_permission(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ - ("tools", "0039_update_permissions"), - ] - - operations = [ - migrations.RunPython(migrate_permissions), - migrations.RunPython(add_admin_permission), - ] + dependencies = [("tools", "0039_update_permissions")] + + operations = [migrations.RunPython(migrate_permissions), migrations.RunPython(add_admin_permission)] diff --git a/rocky/tools/migrations/0041_merge_20230731_1131.py b/rocky/tools/migrations/0041_merge_20230731_1131.py index d0da43cb26f..4c4569e823c 100644 --- a/rocky/tools/migrations/0041_merge_20230731_1131.py +++ b/rocky/tools/migrations/0041_merge_20230731_1131.py @@ -4,9 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ - ("tools", "0040_admin_inherits_red_teamer_permissions"), - ("tools", "0040_update_admin_permission"), - ] + dependencies = [("tools", "0040_admin_inherits_red_teamer_permissions"), ("tools", "0040_update_admin_permission")] operations: list = [] diff --git a/rocky/tools/ooi_helpers.py b/rocky/tools/ooi_helpers.py index 7d2c6238254..0b6cd2b3b18 100644 --- a/rocky/tools/ooi_helpers.py +++ b/rocky/tools/ooi_helpers.py @@ -86,11 +86,7 @@ def process_value(value: Any) -> Any: def get_ooi_dict(ooi: OOI) -> dict: - ooi_dict = { - "id": ooi.primary_key, - "ooi_type": ooi.get_ooi_type(), - "human_readable": ooi.human_readable, - } + ooi_dict = {"id": ooi.primary_key, "ooi_type": ooi.get_ooi_type(), "human_readable": ooi.human_readable} ignore_properties = ["primary_key", "scan_profile"] @@ -224,12 +220,7 @@ def include_type(ooi_type): def get_finding_type_from_finding(finding: Finding) -> FindingType: return TypeAdapter( KATFindingType | CVEFindingType | CWEFindingType | RetireJSFindingType | SnykFindingType | CAPECFindingType - ).validate_python( - { - "object_type": finding.finding_type.class_, - "id": finding.finding_type.natural_key, - } - ) + ).validate_python({"object_type": finding.finding_type.class_, "id": finding.finding_type.natural_key}) _EXCLUDED = [Finding] + FindingType.strict_subclasses() diff --git a/rocky/tools/validators.py b/rocky/tools/validators.py index 32d682a7db1..ef3f5acbf52 100644 --- a/rocky/tools/validators.py +++ b/rocky/tools/validators.py @@ -1,6 +1,5 @@ from django.core.validators import RegexValidator phone_validator = RegexValidator( - regex=r"^\+?1?\d{11,11}$", - message="Phone number must be entered in the format: '+99999999999'. 11 digits allowed.", + regex=r"^\+?1?\d{11,11}$", message="Phone number must be entered in the format: '+99999999999'. 11 digits allowed." ) diff --git a/rocky/tools/view_helpers.py b/rocky/tools/view_helpers.py index 741c59c868a..5aa54b0d971 100644 --- a/rocky/tools/view_helpers.py +++ b/rocky/tools/view_helpers.py @@ -135,7 +135,7 @@ def build_breadcrumbs(self): { "url": reverse("organization_settings", kwargs={"organization_code": self.organization.code}), "text": _("Settings"), - }, + } ] return breadcrumbs @@ -149,7 +149,7 @@ def build_breadcrumbs(self): { "url": reverse("organization_member_list", kwargs={"organization_code": self.organization.code}), "text": _("Members"), - }, + } ] return breadcrumbs @@ -209,10 +209,7 @@ def reschedule_task(request: HttpRequest, organization_code: str, task_id: str) return try: - new_p_item = PrioritizedItem( - data=task.p_item.data, - priority=1, - ) + new_p_item = PrioritizedItem(data=task.p_item.data, priority=1) schedule_task(request, organization_code, new_p_item) except SchedulerError as error: