Migrate dependency management to poetry

Switch dependency management to use poetry, which is much nicer than
pip-tools. This is largely based off of
<freedomofpress/securedrop-proxy#122> and
applies the same changes to the Makefile and CI.

.circleci/config.yml

@@ -1,58 +1,59 @@
 ---
 common-steps:
+  - &install_poetry
+    run:
+      name: Install Poetry
+      command: |
+        set -e
+        source /etc/os-release
+        if [[ "$VERSION_CODENAME" == "bullseye" ]]; then
+            # Install Poetry via PyPI
+            apt-get update && apt-get install --yes --no-install-recommends python3-pip
+            pip install poetry==1.6.1
+        elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then
+            # Install Poetry via system package
+            apt-get update && apt-get install --yes --no-install-recommends python3-poetry
+        else
+            echo "Unsupported Debian version: $VERSION_CODENAME"
+            exit 1
+        fi
+
   - &install_testing_dependencies
     run:
       name: Install testing dependencies
       command: |
-        set -e
         apt update && apt install -y git gnupg make python3-dev gnupg python3-venv libnotify-bin
+        poetry install --no-ansi
 
   - &install_build_dependencies
     run:
       name: Install build dependencies
       command: |
-        set -e
         apt update && apt install -y git make sudo
 
   - &run_unit_tests
     run:
       name: Install requirements and run unit tests
       command: |
-        set -e
-        export VERSION_CODENAME=$(~/project/scripts/codename)
-        make venv
-        source .venv/bin/activate
         export PYTHONPATH=$PYTHONPATH:.  # so alembic can get to Base metadata
         make test
 
   - &run_lint
     run:
       name: Run lint, type checking, code formatting
       command: |
-        set -e
-        export VERSION_CODENAME=$(~/project/scripts/codename)
-        make venv
-        source .venv/bin/activate
         make check-black lint
 
   - &check_security
     run:
       name: Run static analysis on source code to find security issues
       command: |
-        set -e
-        export VERSION_CODENAME=$(~/project/scripts/codename)
-        make venv
-        source .venv/bin/activate
         make semgrep
 
   - &check_python_dependencies_for_vulnerabilities
     run:
       name: Check Python dependencies for known vulnerabilities
       command: |
-        set -e
-        export VERSION_CODENAME=$(~/project/scripts/codename)
-        make venv
-        source .venv/bin/activate
         make safety
 
   - &install_packaging_dependencies
@@ -63,10 +64,10 @@ common-steps:
         mkdir ~/packaging && cd ~/packaging
         # local builds may not have an ssh url, so || true
         git config --global --unset url.ssh://[email protected] || true
-        git clone https://github.com/freedomofpress/securedrop-debian-packaging.git
-        cd securedrop-debian-packaging
-        apt-get update && apt-get install -y sudo make
+        git clone https://github.com/freedomofpress/securedrop-builder.git -b bc-build-reqs
+        cd securedrop-builder
         make install-deps
+        source .venv/bin/activate
         PKG_DIR=~/project make requirements
 
   - &check_packaging_requirements
@@ -77,24 +78,14 @@ common-steps:
         # Fail if unstaged changes exist (after `make requirements` in the previous run step).
         git diff --ignore-matching-lines=# --exit-code
 
-  - &check_testing_requirements
-    run:
-      name: Ensure that the same Python requirements are used for development/testing and production.
-      command: |
-        set -e
-        export VERSION_CODENAME=$(~/project/scripts/codename)
-        make venv
-        source .venv/bin/activate
-        make requirements
-        git diff --exit-code requirements/dev-${VERSION_CODENAME}-requirements.txt
-
   - &build_debian_package
     run:
       name: Build debian package
       command: |
-        cd ~/packaging/securedrop-debian-packaging
+        cd ~/packaging/securedrop-builder
         export PKG_VERSION=1000.0
         export PKG_PATH=~/project/
+        source .venv/bin/activate
         make securedrop-export
 
 version: 2.1
@@ -117,8 +108,9 @@ jobs:
     parameters: *parameters
     docker: *docker
     steps:
-      - *install_testing_dependencies
+      - *install_poetry
       - checkout
+      - *install_testing_dependencies
       - *run_unit_tests
       - store_test_results:
           path: test-results
@@ -127,33 +119,28 @@ jobs:
     parameters: *parameters
     docker: *docker
     steps:
-      - *install_testing_dependencies
+      - *install_poetry
       - checkout
+      - *install_testing_dependencies
       - *run_lint
 
   check-security:
     parameters: *parameters
     docker: *docker
     steps:
-      - *install_testing_dependencies
+      - *install_poetry
       - checkout
+      - *install_testing_dependencies
       - *check_security
 
   check-python-security:
     parameters: *parameters
     docker: *docker
     steps:
-      - *install_testing_dependencies
+      - *install_poetry
       - checkout
-      - *check_python_dependencies_for_vulnerabilities
-
-  check-testing-requirements:
-    parameters: *parameters
-    docker: *docker
-    steps:
       - *install_testing_dependencies
-      - checkout
-      - *check_testing_requirements
+      - *check_python_dependencies_for_vulnerabilities
 
 workflows:
   securedrop_export_ci:
@@ -166,8 +153,6 @@ workflows:
                 - bookworm
       - lint:
           matrix: *matrix
-      - check-testing-requirements:
-          matrix: *matrix
       - check-security:
           matrix: *matrix
       - check-python-security:

MANIFEST.in

@@ -2,8 +2,7 @@ include LICENSE
 include README.md
 include securedrop_export/VERSION
 include changelog.md
-include requirements/build-requirements.txt
-include requirements/requirements.txt
+include build-requirements.txt
 include securedrop_export/*.py
 include setup.py
 include files/send-to-usb.desktop

Makefile

@@ -1,80 +1,34 @@
 .PHONY: all
 all: help
 
-VERSION_CODENAME ?= bullseye
-
-.PHONY: venv
-venv:  ## Provision a Python 3 virtualenv for **development**
-	python3 -m venv .venv
-	.venv/bin/pip install --upgrade pip wheel
-	.venv/bin/pip install --require-hashes -r requirements/dev-${VERSION_CODENAME}-requirements.txt
-
 .PHONY: safety
 safety: ## Runs `safety check` to check python dependencies for vulnerabilities
-	pip install --upgrade safety && \
-		for req_file in `find . -type f -name '*requirements.txt'`; do \
-			echo "Checking file $$req_file" \
-			&& safety check --full-report -r $$req_file \
-			&& echo -e '\n' \
-			|| exit 1; \
-		done
-
-# Helper, not to be used directly
-.PHONY: sync-requirements
-sync-requirements:  ## Update dev-requirements.txt to pin to the same versions of prod dependencies
-	if test -f "requirements/dev-bullseye-requirements.txt"; then rm -r requirements/dev-bullseye-requirements.txt; fi
-	if test -f "requirements/dev-bookworm-requirements.txt"; then rm -r requirements/dev-bookworm-requirements.txt; fi
-	$(MAKE) dev-requirements
-
-# Helper, not to be used directly
-.PHONY: dev-requirements
-dev-requirements:  ## Update dev-*requirements.txt files if pinned versions do not comply with the dependency specifications in dev-*requirements.in
-	pip-compile --allow-unsafe --generate-hashes --output-file requirements/dev-bullseye-requirements.txt requirements/dev-bullseye-requirements.in
-	pip-compile --allow-unsafe --generate-hashes --output-file requirements/dev-bookworm-requirements.txt requirements/dev-bookworm-requirements.in
-
-.PHONY: requirements
-requirements:  ## Update *requirements.txt files if pinned versions do not comply with the dependency specifications in *requirements.in
-	pip-compile --generate-hashes --output-file requirements/requirements.txt requirements/requirements.in
-	$(MAKE) dev-requirements
-
-.PHONY: update-dependency
-update-dependency:  ## Add or upgrade a package to the latest version that complies with the dependency specifications in requirements.in
-	pip-compile --generate-hashes --upgrade-package $(PACKAGE) --output-file requirements/requirements.txt requirements/requirements.in
-	$(MAKE) sync-requirements
-
-.PHONY: update-dev-only-dependencies
-update-dev-only-dependencies:  ## Update dev-requirements.txt to pin to the latest versions of dev-only dependencies that comply with the dependency specifications in dev-requirements.in
-	$(MAKE) sync-requirements
-	@while read line; do \
-		pip-compile --allow-unsafe --generate-hashes --upgrade-package $file --output-file requirements/dev-bullseye-requirements.txt requirements/dev-bullseye-requirements.in; \
-	done < 'requirements/dev-bullseye-requirements.in'
-	@while read line; do \
-		pip-compile --allow-unsafe --generate-hashes --upgrade-package $file --output-file requirements/dev-bookworm-requirements.txt requirements/dev-bookworm-requirements.in; \
-	done < 'requirements/dev-bookworm-requirements.in'
+	@echo "Running safety against build requirements…"
+	@poetry run safety check --full-report -r build-requirements.txt
 
 .PHONY: check
 check: lint mypy semgrep test check-black ## Run linter and tests
 
 .PHONY: check-black
 check-black: ## Check Python source code formatting with black
-	@black --check --diff ./
+	@poetry run black --check --diff ./
 
 TESTS ?= tests
 .PHONY: test
 test:  ## Run tests
-	pytest -v --cov-report html --cov-report term-missing --cov=securedrop_export $$TESTS
+	poetry run pytest -v --cov-report html --cov-report term-missing --cov=securedrop_export $$TESTS
 
 .PHONY: lint
 lint:  ## Run linter
-	flake8 securedrop_export/ tests/
+	poetry run flake8 securedrop_export/ tests/
 
 .PHONY: mypy
 mypy:  ## Type check Python files
-	mypy .
+	poetry run mypy .
 
 .PHONY: black
 black: ## Format Python source code with black
-	@black ./
+	@poetry run black ./
 
 SEMGREP_FLAGS := --exclude "tests/" --error --strict --verbose
 
@@ -84,14 +38,14 @@ semgrep:semgrep-community semgrep-local
 .PHONY: semgrep-community
 semgrep-community:
 	@echo "Running semgrep with semgrep.dev community rules..."
-	@semgrep $(SEMGREP_FLAGS) --config "p/r2c-security-audit" --config "p/r2c-ci"
+	@poetry run semgrep $(SEMGREP_FLAGS) --config "p/r2c-security-audit" --config "p/r2c-ci"
 
 .PHONY: semgrep-local
 semgrep-local:
 	@echo "Running semgrep with local rules..."
-	@semgrep $(SEMGREP_FLAGS) --config ".semgrep"
+	@poetry run semgrep $(SEMGREP_FLAGS) --config ".semgrep"
 
-# Explaination of the below shell command should it ever break.
+# Explanation of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" and any make targets that might appear between : and ##
 # 2. Use sed-like syntax to remove the make targets
 # 3. Format the split fields into $$1) the target name (in blue) and $$2) the target descrption

README.md

@@ -6,21 +6,10 @@
 
 Code for exporting and printing files from the SecureDrop Qubes Workstation.
 
-## Getting Started
+## Quick Start
 
-Python 3 support is required. To get started:
-
-```
-virtualenv --python=python3.7 .venv
-source .venv/bin/activate
-pip install -r test-requirements.txt
-```
-
-To run the linter and tests:
-
-```
-make check
-```
+1. [Install Poetry](https://python-poetry.org/docs/#installing-with-the-official-installer)
+2. Run `make check` to verify the installation
 
 ## Supported Printers