Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable ruff format skip-magic-trailing-comma #2975

Merged
merged 6 commits into from
Oct 7, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
7 changes: 1 addition & 6 deletions boefjes/boefjes/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,7 @@

@click.command()
@click.argument("worker_type", type=click.Choice([q.value for q in WorkerManager.Queue]))
@click.option(
"--log-level",
type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"]),
help="Log level",
default="INFO",
)
@click.option("--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"]), help="Log level", default="INFO")
def cli(worker_type: str, log_level: str):
logger.setLevel(log_level)
logger.info("Starting runtime for %s", worker_type)
Expand Down
6 changes: 1 addition & 5 deletions boefjes/boefjes/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,7 @@ def get_scheduler_client():


def get_bytes_client():
return BytesAPIClient(
str(settings.bytes_api),
username=settings.bytes_username,
password=settings.bytes_password,
)
return BytesAPIClient(str(settings.bytes_api), username=settings.bytes_username, password=settings.bytes_password)


@app.get("/healthz")
Expand Down
6 changes: 1 addition & 5 deletions boefjes/boefjes/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,11 +269,7 @@ def get_runtime_manager(settings: Settings, queue: WorkerManager.Queue, log_leve
local_repository = get_local_repository()

session = sessionmaker(bind=get_engine())()
plugin_service = PluginService(
create_plugin_storage(session),
create_config_storage(session),
local_repository,
)
plugin_service = PluginService(create_plugin_storage(session), create_config_storage(session), local_repository)

item_handler: Handler
if queue is WorkerManager.Queue.BOEFJES:
Expand Down
9 changes: 2 additions & 7 deletions boefjes/boefjes/clients/bytes_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,7 @@ def __init__(self, base_url: str, username: str, password: str):
transport=(HTTPTransport(retries=6)),
)

self.credentials = {
"username": username,
"password": password,
}
self.credentials = {"username": username, "password": password}
self.headers: dict[str, str] = {}

def login(self) -> None:
Expand All @@ -65,9 +62,7 @@ def _get_authentication_headers(self) -> dict[str, str]:

def _get_token(self) -> str:
response = self._session.post(
"/token",
data=self.credentials,
headers={"content-type": "application/x-www-form-urlencoded"},
"/token", data=self.credentials, headers={"content-type": "application/x-www-form-urlencoded"}
)

return str(response.json()["access_token"])
Expand Down
19 changes: 4 additions & 15 deletions boefjes/boefjes/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,23 +87,12 @@ class Settings(BaseSettings):
..., examples=["http://localhost:8001"], description="Octopoes API URL", validation_alias="OCTOPOES_API"
)
api: AnyHttpUrl = Field(
...,
examples=["http://boefje:8000"],
description="The URL on which the boefjes API is available",
..., examples=["http://boefje:8000"], description="The URL on which the boefjes API is available"
)
# Boefje server settings
api_host: str = Field(
"0.0.0.0",
description="Host address of the Boefje API server",
)
api_port: int = Field(
8000,
description="Host port of the Boefje API server",
)
docker_network: str = Field(
"bridge",
description="Docker network to run Boefjes in",
)
api_host: str = Field("0.0.0.0", description="Host address of the Boefje API server")
api_port: int = Field(8000, description="Host port of the Boefje API server")
docker_network: str = Field("bridge", description="Docker network to run Boefjes in")
bytes_api: AnyHttpUrl = Field(
..., examples=["http://localhost:8002"], description="Bytes API URL", validation_alias="BYTES_API"
)
Expand Down
13 changes: 2 additions & 11 deletions boefjes/boefjes/dependencies/plugins.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,7 @@


class PluginService:
def __init__(
self,
plugin_storage: PluginStorage,
config_storage: ConfigStorage,
local_repo: LocalPluginRepository,
):
def __init__(self, plugin_storage: PluginStorage, config_storage: ConfigStorage, local_repo: LocalPluginRepository):
self.plugin_storage = plugin_storage
self.config_storage = config_storage
self.local_repo = local_repo
Expand Down Expand Up @@ -235,11 +230,7 @@ def _set_plugin_enabled(self, plugin: PluginType, organisation_id: str) -> Plugi

def get_plugin_service() -> Iterator[PluginService]:
def closure(session: Session):
return PluginService(
create_plugin_storage(session),
create_config_storage(session),
get_local_repository(),
)
return PluginService(create_plugin_storage(session), create_config_storage(session), get_local_repository())

yield from session_managed_iterator(closure)

Expand Down
4 changes: 1 addition & 3 deletions boefjes/boefjes/docker_boefjes_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@ def __init__(self, boefje_resource: Boefje, boefje_meta: BoefjeMeta):
self.docker_client = docker.from_env()
self.scheduler_client = SchedulerAPIClient(str(settings.scheduler_api))
self.bytes_api_client = BytesAPIClient(
str(settings.bytes_api),
username=settings.bytes_username,
password=settings.bytes_password,
str(settings.bytes_api), username=settings.bytes_username, password=settings.bytes_password
)

def run(self) -> None:
Expand Down
14 changes: 3 additions & 11 deletions boefjes/boefjes/job_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,7 @@
logger = structlog.get_logger(__name__)

bytes_api_client = BytesAPIClient(
str(settings.bytes_api),
username=settings.bytes_username,
password=settings.bytes_password,
str(settings.bytes_api), username=settings.bytes_username, password=settings.bytes_password
)


Expand All @@ -42,8 +40,7 @@ def get_environment_settings(boefje_meta: BoefjeMeta, schema: dict | None = None
try:
katalogus_api = str(settings.katalogus_api).rstrip("/")
response = httpx.get(
f"{katalogus_api}/v1/organisations/{boefje_meta.organization}/{boefje_meta.boefje.id}/settings",
timeout=30,
f"{katalogus_api}/v1/organisations/{boefje_meta.organization}/{boefje_meta.boefje.id}/settings", timeout=30
)
response.raise_for_status()
except HTTPError:
Expand Down Expand Up @@ -76,12 +73,7 @@ def get_environment_settings(boefje_meta: BoefjeMeta, schema: dict | None = None


class BoefjeHandler(Handler):
def __init__(
self,
job_runner: BoefjeJobRunner,
plugin_service: PluginService,
bytes_client: BytesAPIClient,
):
def __init__(self, job_runner: BoefjeJobRunner, plugin_service: PluginService, bytes_client: BytesAPIClient):
self.job_runner = job_runner
self.plugin_service = plugin_service
self.bytes_client = bytes_client
Expand Down
22 changes: 5 additions & 17 deletions boefjes/boefjes/katalogus/organisations.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@


def check_organisation_exists(
organisation_id: str,
storage: OrganisationStorage = Depends(get_organisations_store),
organisation_id: str, storage: OrganisationStorage = Depends(get_organisations_store)
) -> None:
"""
Checks if an organisation exists, if not, creates it.
Expand All @@ -24,36 +23,25 @@ def check_organisation_exists(


@router.get("", response_model=dict[str, Organisation])
def list_organisations(
storage: OrganisationStorage = Depends(get_organisations_store),
):
def list_organisations(storage: OrganisationStorage = Depends(get_organisations_store)):
return storage.get_all()


@router.get("/{organisation_id}", response_model=Organisation)
def get_organisation(
organisation_id: str,
storage: OrganisationStorage = Depends(get_organisations_store),
):
def get_organisation(organisation_id: str, storage: OrganisationStorage = Depends(get_organisations_store)):
try:
return storage.get_by_id(organisation_id)
except (KeyError, ObjectNotFoundException):
raise HTTPException(status.HTTP_404_NOT_FOUND, "Unknown organisation")


@router.post("/", status_code=status.HTTP_201_CREATED)
def add_organisation(
organisation: Organisation,
storage: OrganisationStorage = Depends(get_organisations_store),
):
def add_organisation(organisation: Organisation, storage: OrganisationStorage = Depends(get_organisations_store)):
with storage as store:
store.create(organisation)


@router.delete("/{organisation_id}")
def remove_organisation(
organisation_id: str,
storage: OrganisationStorage = Depends(get_organisations_store),
):
def remove_organisation(organisation_id: str, storage: OrganisationStorage = Depends(get_organisations_store)):
with storage as store:
store.delete_by_id(organisation_id)
32 changes: 7 additions & 25 deletions boefjes/boefjes/katalogus/plugins.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@
from boefjes.storage.interfaces import DuplicatePlugin, IntegrityError, NotAllowed, PluginStorage

router = APIRouter(
prefix="/organisations/{organisation_id}",
tags=["plugins"],
dependencies=[Depends(check_organisation_exists)],
prefix="/organisations/{organisation_id}", tags=["plugins"], dependencies=[Depends(check_organisation_exists)]
)

logger = structlog.get_logger(__name__)
Expand Down Expand Up @@ -57,10 +55,7 @@ def list_plugins(

# filter plugins by id, name or description
if filter_params.q is not None:
plugins = filter(
partial(_plugin_matches_query, query=filter_params.q),
plugins,
)
plugins = filter(partial(_plugin_matches_query, query=filter_params.q), plugins)

# filter plugins by type
if filter_params.type is not None:
Expand All @@ -86,9 +81,7 @@ def list_plugins(

@router.get("/plugins/{plugin_id}", response_model=PluginType)
def get_plugin(
plugin_id: str,
organisation_id: str,
plugin_service: PluginService = Depends(get_plugin_service),
plugin_id: str, organisation_id: str, plugin_service: PluginService = Depends(get_plugin_service)
) -> PluginType:
try:
with plugin_service as p:
Expand Down Expand Up @@ -165,11 +158,7 @@ def cron_valid(cls, cron: str | None) -> str | None:


@router.patch("/boefjes/{boefje_id}", status_code=status.HTTP_204_NO_CONTENT)
def update_boefje(
boefje_id: str,
boefje: BoefjeIn,
storage: PluginStorage = Depends(get_plugin_storage),
):
def update_boefje(boefje_id: str, boefje: BoefjeIn, storage: PluginStorage = Depends(get_plugin_storage)):
# todo: update boefje should be done in the plugin service
try:
with storage as p:
Expand Down Expand Up @@ -203,9 +192,7 @@ class NormalizerIn(BaseModel):

@router.patch("/normalizers/{normalizer_id}", status_code=status.HTTP_204_NO_CONTENT)
def update_normalizer(
normalizer_id: str,
normalizer: NormalizerIn,
storage: PluginStorage = Depends(get_plugin_storage),
normalizer_id: str, normalizer: NormalizerIn, storage: PluginStorage = Depends(get_plugin_storage)
):
with storage as p:
p.update_normalizer(normalizer_id, normalizer.model_dump(exclude_unset=True))
Expand All @@ -223,18 +210,13 @@ def get_plugin_schema(plugin_id: str, plugin_service: PluginService = Depends(ge


@router.get("/plugins/{plugin_id}/cover.jpg", include_in_schema=False)
def get_plugin_cover(
plugin_id: str,
plugin_service: PluginService = Depends(get_plugin_service),
) -> FileResponse:
def get_plugin_cover(plugin_id: str, plugin_service: PluginService = Depends(get_plugin_service)) -> FileResponse:
return FileResponse(plugin_service.cover(plugin_id))


@router.get("/plugins/{plugin_id}/description.md", include_in_schema=False)
def get_plugin_description(
plugin_id: str,
organisation_id: str,
plugin_service: PluginService = Depends(get_plugin_service),
plugin_id: str, organisation_id: str, plugin_service: PluginService = Depends(get_plugin_service)
) -> Response:
return Response(plugin_service.description(plugin_id, organisation_id))

Expand Down
20 changes: 4 additions & 16 deletions boefjes/boefjes/katalogus/root.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,34 +74,22 @@

@app.exception_handler(NotFound)
def entity_not_found_handler(request: Request, exc: NotFound):
return JSONResponse(
status_code=status.HTTP_404_NOT_FOUND,
content={"message": exc.message},
)
return JSONResponse(status_code=status.HTTP_404_NOT_FOUND, content={"message": exc.message})


@app.exception_handler(NotAllowed)
def not_allowed_handler(request: Request, exc: NotAllowed):
return JSONResponse(
status_code=status.HTTP_400_BAD_REQUEST,
content={"message": exc.message},
)
return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST, content={"message": exc.message})


@app.exception_handler(IntegrityError)
def integrity_error_handler(request: Request, exc: IntegrityError):
return JSONResponse(
status_code=status.HTTP_400_BAD_REQUEST,
content={"message": exc.message},
)
return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST, content={"message": exc.message})


@app.exception_handler(StorageError)
def storage_error_handler(request: Request, exc: StorageError):
return JSONResponse(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
content={"message": exc.message},
)
return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"message": exc.message})


class ServiceHealth(BaseModel):
Expand Down
11 changes: 2 additions & 9 deletions boefjes/boefjes/katalogus/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,14 @@


@router.get("", response_model=dict)
def list_settings(
organisation_id: str,
plugin_id: str,
plugin_service: PluginService = Depends(get_plugin_service),
):
def list_settings(organisation_id: str, plugin_id: str, plugin_service: PluginService = Depends(get_plugin_service)):
with plugin_service as p:
return p.get_all_settings(organisation_id, plugin_id)


@router.put("")
def upsert_settings(
organisation_id: str,
plugin_id: str,
values: dict,
plugin_service: PluginService = Depends(get_plugin_service),
organisation_id: str, plugin_id: str, values: dict, plugin_service: PluginService = Depends(get_plugin_service)
):
with plugin_service as p:
p.upsert_settings(values, organisation_id, plugin_id)
Expand Down
9 changes: 2 additions & 7 deletions boefjes/boefjes/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,17 +112,12 @@ def _parse_results(self, normalizer_meta: NormalizerMeta, results: Iterable[Norm

observations = [
NormalizerObservation(
type="observation",
input_ooi=normalizer_meta.raw_data.boefje_meta.input_ooi,
results=oois,
type="observation", input_ooi=normalizer_meta.raw_data.boefje_meta.input_ooi, results=oois
)
]
else:
observations = []

return NormalizerResults(
observations=observations,
declarations=declarations,
affirmations=affirmations,
scan_profiles=scan_profiles,
observations=observations, declarations=declarations, affirmations=affirmations, scan_profiles=scan_profiles
)
4 changes: 1 addition & 3 deletions boefjes/boefjes/migrations/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,7 @@ def run_migrations_online() -> None:

"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool
)

with connectable.connect() as connection:
Expand Down
Loading
Loading