Merge pull request #72 from SwiftNathalie/main

Change table name from `packages` to `scan`

alembic/versions/883af2539440_rename_package_to_scan.py

@@ -0,0 +1,40 @@
+"""Rename package to scan
+
+Revision ID: 883af2539440
+Revises: 5e46ee8ec64f
+Create Date: 2023-06-21 12:37:16.744892
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "883af2539440"
+down_revision = "5e46ee8ec64f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.rename_table("packages", "scans")
+    op.alter_column("scans", "package_id", new_column_name="scan_id")
+
+    op.alter_column("download_urls", "package_id", new_column_name="scan_id")
+    op.drop_constraint("download_urls_package_id_fkey", "download_urls", type_="foreignkey")
+    op.create_foreign_key("download_urls_scan_id_fkey", "download_urls", "scans", ["scan_id"], ["scan_id"])
+
+    op.alter_column("package_rules", "package_id", new_column_name="scan_id")
+    op.drop_constraint("package_rules_package_id_fkey", "package_rules", type_="foreignkey")
+    op.create_foreign_key("package_rules_scan_id_fkey", "package_rules", "scans", ["scan_id"], ["scan_id"])
+
+
+def downgrade() -> None:
+    op.rename_table("scans", "packages")
+    op.alter_column("packages", "scan_id", new_column_name="package_id")
+
+    op.alter_column("download_urls", "scan_id", new_column_name="package_id")
+    op.drop_constraint("download_urls_scan_id_fkey", "download_urls", type_="foreignkey")
+    op.create_foreign_key("download_urls_package_id_fkey", "download_urls", "packages", ["package_id"], ["package_id"])
+
+    op.alter_column("package_rules", "scan_id", new_column_name="package_id")
+    op.drop_constraint("package_rules_scan_id_fkey", "package_rules", type_="foreignkey")
+    op.create_foreign_key("package_rules_package_id_fkey", "package_rules", "packages", ["package_id"], ["package_id"])

src/mainframe/endpoints/job.py

@@ -11,7 +11,7 @@
 from mainframe.database import get_db
 from mainframe.dependencies import validate_token
 from mainframe.json_web_token import AuthenticationData
-from mainframe.models.orm import Package, Status
+from mainframe.models.orm import Scan, Status
 from mainframe.models.schemas import JobResult, NoJob
 
 router = APIRouter(tags=["job"])
@@ -36,48 +36,48 @@ async def get_job(
     """
 
     scalars = await session.scalars(
-        select(Package)
+        select(Scan)
         .where(
             or_(
-                Package.status == Status.QUEUED,
+                Scan.status == Status.QUEUED,
                 and_(
-                    Package.pending_at < datetime.now(timezone.utc) - timedelta(seconds=mainframe_settings.job_timeout),
-                    Package.status == Status.PENDING,
+                    Scan.pending_at < datetime.now(timezone.utc) - timedelta(seconds=mainframe_settings.job_timeout),
+                    Scan.status == Status.PENDING,
                 ),
             )
         )
-        .order_by(Package.pending_at.nulls_first(), Package.queued_at)
-        .options(selectinload(Package.download_urls))
+        .order_by(Scan.pending_at.nulls_first(), Scan.queued_at)
+        .options(selectinload(Scan.download_urls))
         .with_for_update()
     )
-    package = scalars.first()
+    scan = scalars.first()
 
-    if not package:
-        logger.info("No packages available to scan, job not given.", tag="no_packages")
-        return NoJob(detail="No available packages to scan. Try again later.")
+    if not scan:
+        logger.info("No scans available, job not given.", tag="no_packages")
+        return NoJob(detail="No scans available. Try again later.")
 
-    package.status = Status.PENDING
-    package.pending_at = datetime.now(timezone.utc)
-    package.pending_by = auth.subject
+    scan.status = Status.PENDING
+    scan.pending_at = datetime.now(timezone.utc)
+    scan.pending_by = auth.subject
     await session.commit()
 
-    distribution_urls = [distribution.url for distribution in package.download_urls]
+    distribution_urls = [distribution.url for distribution in scan.download_urls]
 
     await logger.ainfo(
         "Job given and status set to pending in database",
         package={
-            "name": package.name,
-            "status": package.status,
-            "pending_at": package.pending_at,
+            "name": scan.name,
+            "status": scan.status,
+            "pending_at": scan.pending_at,
             "pending_by": auth.subject,
-            "version": package.version,
+            "version": scan.version,
         },
         tag="job_given",
     )
 
     return JobResult(
-        name=package.name,
-        version=package.version,
+        name=scan.name,
+        version=scan.version,
         distributions=distribution_urls,
         hash=request.app.state.rules.rules_commit,
     )

src/mainframe/endpoints/package.py

@@ -11,7 +11,7 @@
 from mainframe.database import get_db
 from mainframe.dependencies import validate_token
 from mainframe.json_web_token import AuthenticationData
-from mainframe.models.orm import DownloadURL, Package, Rule, Status
+from mainframe.models.orm import DownloadURL, Rule, Scan, Status
 from mainframe.models.schemas import (
     BatchPackageQueueErr,
     Error,
@@ -39,34 +39,31 @@ async def submit_results(
     name = result.name
     version = result.version
 
-    row = await session.scalar(
-        select(Package)
-        .where(Package.name == name)
-        .where(Package.version == version)
-        .options(selectinload(Package.rules))
+    scan = await session.scalar(
+        select(Scan).where(Scan.name == name).where(Scan.version == version).options(selectinload(Scan.rules))
     )
 
     log = logger.bind(package={"name": name, "version": version})
 
-    if row is None:
+    if scan is None:
         error = HTTPException(404, f"Package `{name}@{version}` not found in database.")
         await log.aerror(
             f"Package {name}@{version} not found in database", error_message=error.detail, tag="package_not_found_db"
         )
         raise error
 
-    if row.status == Status.FINISHED:
+    if scan.status == Status.FINISHED:
         error = HTTPException(409, f"Package `{name}@{version}` is already in a FINISHED state.")
         await log.aerror(
             f"Package {name}@{version} already in a FINISHED state", error_message=error.detail, tag="already_finished"
         )
         raise error
 
-    row.status = Status.FINISHED
-    row.finished_at = dt.datetime.now(dt.timezone.utc)
-    row.inspector_url = result.inspector_url
-    row.score = result.score
-    row.finished_by = auth.subject
+    scan.status = Status.FINISHED
+    scan.finished_at = dt.datetime.now(dt.timezone.utc)
+    scan.inspector_url = result.inspector_url
+    scan.score = result.score
+    scan.finished_by = auth.subject
 
     for rule_name in result.rules_matched:
         rule = await session.scalar(select(Rule).where(Rule.name == rule_name))
@@ -80,15 +77,15 @@ async def submit_results(
             )
             raise error
 
-        row.rules.append(rule)
+        scan.rules.append(rule)
 
     await log.ainfo(
         "Scan results submitted",
         package={
             "name": name,
             "version": version,
-            "status": row.status,
-            "finished_at": row.finished_at,
+            "status": scan.status,
+            "finished_at": scan.finished_at,
             "inspector_url": result.inspector_url,
             "score": result.score,
             "finished_by": auth.subject,
@@ -148,13 +145,13 @@ async def lookup_package_info(
         )
         raise HTTPException(status_code=400)
 
-    query = select(Package).options(selectinload(Package.rules))
+    query = select(Scan).options(selectinload(Scan.rules))
     if nn_name:
-        query = query.where(Package.name == name)
+        query = query.where(Scan.name == name)
     if nn_version:
-        query = query.where(Package.version == version)
+        query = query.where(Scan.version == version)
     if nn_since:
-        query = query.where(Package.finished_at >= dt.datetime.fromtimestamp(since, tz=dt.timezone.utc))
+        query = query.where(Scan.finished_at >= dt.datetime.fromtimestamp(since, tz=dt.timezone.utc))
 
     data = await session.scalars(query)
 
@@ -191,7 +188,7 @@ async def batch_queue_package(
         except KeyError:
             err_packages[(name, version)] = f"Package {name}@{version} was not found on PyPI"
 
-    query = select(Package).where(tuple_(Package.name, Package.version).in_(ok_packages))
+    query = select(Scan).where(tuple_(Scan.name, Scan.version).in_(ok_packages))
     rows = await session.scalars(query)
 
     # This step filters out packages that are already in the database
@@ -204,7 +201,7 @@ async def batch_queue_package(
         err_packages[t] = f"Package {name}@{version} is already queued for scanning"
 
     new_packages = [
-        Package(
+        Scan(
             name=metadata.info.name,
             version=metadata.info.version,
             status=Status.QUEUED,
@@ -265,14 +262,14 @@ async def queue_package(
     version = package_metadata.info.version  # Use latest version if not provided
     log = logger.bind(package={"name": name, "version": version})
 
-    query = select(Package).where(Package.name == name).where(Package.version == version)
+    query = select(Scan).where(Scan.name == name).where(Scan.version == version)
     row = await session.scalar(query)
 
     if row is not None:
         await log.info(f"Package {name}@{version} already queued for scanning.", tag="already_queued")
         raise HTTPException(409, f"Package {name}@{version} is already queued for scanning")
 
-    new_package = Package(
+    new_package = Scan(
         name=name,
         version=version,
         status=Status.QUEUED,
@@ -295,4 +292,4 @@ async def queue_package(
         tag="package_added",
     )
 
-    return QueuePackageResponse(id=str(new_package.package_id))
+    return QueuePackageResponse(id=str(new_package.scan_id))

src/mainframe/endpoints/report.py

@@ -14,7 +14,7 @@
 from mainframe.database import get_db
 from mainframe.dependencies import get_ms_graph_client, validate_token
 from mainframe.json_web_token import AuthenticationData
-from mainframe.models.orm import Package
+from mainframe.models.orm import Scan
 from mainframe.models.schemas import Error, PackageSpecifier
 from mainframe.utils.mailer import send_email
 from mainframe.utils.pypi import file_path_from_inspector_url
@@ -132,7 +132,7 @@ async def report_package(
     version = package_metadata.info.version
     log = logger.bind(package={"name": name, "version": version})
 
-    query = select(Package).where(Package.name == name).options(selectinload(Package.rules))
+    query = select(Scan).where(Scan.name == name).options(selectinload(Scan.rules))
 
     rows = (await session.scalars(query)).fetchall()
 
@@ -163,7 +163,7 @@ async def report_package(
             )
             raise error
 
-    row = await session.scalar(query.where(Package.version == version))
+    row = await session.scalar(query.where(Scan.version == version))
     if row is None:
         error = HTTPException(
             404, detail=f"Package `{name}` has records in the database, but none with version `{version}`"

src/mainframe/models/orm.py

@@ -33,17 +33,17 @@ class Status(Enum):
 package_rules = Table(
     "package_rules",
     Base.metadata,
-    Column("package_id", ForeignKey("packages.package_id"), primary_key=True),
+    Column("scan_id", ForeignKey("scans.scan_id"), primary_key=True),
     Column("rule_name", ForeignKey("rules.name"), primary_key=True),
 )
 
 
-class Package(Base):
-    """The packages."""
+class Scan(Base):
+    """The scans."""
 
-    __tablename__: str = "packages"
+    __tablename__: str = "scans"
 
-    package_id: Mapped[uuid.UUID] = mapped_column(
+    scan_id: Mapped[uuid.UUID] = mapped_column(
         UUID(as_uuid=True),
         primary_key=True,
         server_default=FetchedValue(),
@@ -87,7 +87,7 @@ class DownloadURL(Base):
         default=uuid.uuid4,
     )
 
-    package_id: Mapped[str] = mapped_column(ForeignKey("packages.package_id"))
+    scan_id: Mapped[str] = mapped_column(ForeignKey("scans.scan_id"))
 
     url: Mapped[str]
 

tests/conftest.py

@@ -12,7 +12,7 @@
 from sqlalchemy import Engine, create_engine, insert
 from sqlalchemy.orm import Session, sessionmaker
 
-from mainframe.models.orm import Base, Package, Status
+from mainframe.models.orm import Base, Scan, Status
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__file__)
@@ -105,7 +105,7 @@ def test_data(request) -> list[dict]:
 def db_setup(engine: Engine, sm: sessionmaker, test_data: list[dict]) -> Generator[None, None, None]:
     Base.metadata.create_all(engine)
     with sm() as sess:
-        sess.execute(insert(Package), test_data)
+        sess.execute(insert(Scan), test_data)
         sess.commit()
         yield
     Base.metadata.drop_all(engine)

tests/test_job.py

@@ -6,11 +6,11 @@
 from sqlalchemy import func, select
 from sqlalchemy.orm import Session
 
-from mainframe.models.orm import Package, Status
+from mainframe.models.orm import Scan, Status
 
 
 def oldest_queued_package(db_session: Session):
-    return db_session.scalar(select(func.min(Package.queued_at)).where(Package.status == Status.QUEUED))
+    return db_session.scalar(select(func.min(Scan.queued_at)).where(Scan.status == Status.QUEUED))
 
 
 def test_min_queue_date_of_queued_rows(test_data: list[dict], db_session: Session):
@@ -24,7 +24,7 @@ def test_min_queue_date_of_queued_rows(test_data: list[dict], db_session: Sessio
 
 def fetch_pid_and_queue_time(name: str, version: str, db_session: Session) -> tuple[uuid.UUID, dt.datetime]:
     t = db_session.execute(
-        select(Package.package_id, Package.queued_at).where((Package.name == name) & (Package.version == version))
+        select(Scan.scan_id, Scan.queued_at).where((Scan.name == name) & (Scan.version == version))
     ).first()
     return typing.cast(tuple[uuid.UUID, dt.datetime], t)