Migrate dependency management to poetry

Switch dependency management to use poetry, which is much nicer than
pip-tools. This is largely based off of
<freedomofpress/securedrop-proxy#122> and
applies the same changes to the Makefile and CI.

.circleci/config.yml

@@ -1,12 +1,34 @@
 ---
 common-steps:
+  - &install_poetry
+    run:
+      name: Install Poetry
+      command: |
+        set -e
+        source /etc/os-release
+        if [[ "$VERSION_CODENAME" == "bullseye" ]]; then
+            # Install Poetry via PyPI
+            apt-get update && apt-get install --yes --no-install-recommends python3-pip
+            pip install poetry==1.6.1
+        elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then
+            # Install Poetry via system package
+            apt-get update && apt-get install --yes --no-install-recommends python3-poetry
+        else
+            echo "Unsupported Debian version: $VERSION_CODENAME"
+            exit 1
+        fi
+
+  - &install_testing_dependencies
+    run:
+      name: Install testing dependencies
+      command: |
+        apt-get install --yes --no-install-recommends git gnupg make
+        poetry install --no-ansi
+
   - &run_tests
     run:
       name: Install requirements and run tests
       command: |
-        apt-get update && apt-get install -y make python3-venv
-        make venv
-        source .venv/bin/activate
         make check
 
   - &install_packaging_dependencies
@@ -15,9 +37,10 @@ common-steps:
       command: |
         apt-get update && apt-get install -y git git-lfs make sudo
         mkdir ~/packaging && cd ~/packaging
-        git clone https://github.com/freedomofpress/securedrop-debian-packaging.git
-        cd securedrop-debian-packaging
+        git clone https://github.com/freedomofpress/securedrop-builder.git -b bc-build-reqs
+        cd securedrop-builder
         make install-deps
+        source .venv/bin/activate
         PKG_DIR=~/project make requirements
 
   - &verify_requirements
@@ -33,9 +56,10 @@ common-steps:
     run:
       name: Build debian package
       command: |
-        cd ~/packaging/securedrop-debian-packaging
+        cd ~/packaging/securedrop-builder
         export PKG_VERSION=1000.0
         export PKG_PATH=~/project/
+        source .venv/bin/activate
         make securedrop-log
 
 version: 2
@@ -45,6 +69,8 @@ jobs:
       - image: debian:bullseye
     steps:
       - checkout
+      - *install_poetry
+      - *install_testing_dependencies
       - *run_tests
 
   build-bullseye:

MANIFEST.in

@@ -1,11 +1,10 @@
 include LICENSE
 include README.md
 include changelog.md
-include requirements/build-requirements.txt
-include requirements/requirements.txt
+include build-requirements.txt
 include securedrop-log*
 include securedrop-redis-log
 include securedrop.Log
 include sd-rsyslog*
 include sdlog.conf
-include VERSION
+include VERSION

Makefile

@@ -1,66 +1,32 @@
 DEFAULT_GOAL: help
 SHELL := /bin/bash
 
-.PHONY: venv
-venv:  ## Provision a Python 3 virtualenv for **development**
-	python3 -m venv .venv
-	.venv/bin/pip install --upgrade pip wheel
-	.venv/bin/pip install --require-hashes -r requirements/dev-requirements.txt
-
 # Bandit is a static code analysis tool to detect security vulnerabilities in Python applications
 # https://wiki.openstack.org/wiki/Security/Projects/Bandit
 .PHONY: bandit
 bandit: ## Run bandit with medium level excluding test-related folders
-	pip install --upgrade pip && \
-        pip install --upgrade bandit!=1.6.0 && \
-	bandit -ll --recursive . --exclude tests,.venv
+	@echo "Running bandit security checks…"
+	@poetry run bandit -ll --recursive . --exclude tests,.venv
 
 .PHONY: safety
 safety: ## Runs `safety check` to check python dependencies for vulnerabilities
-	pip install --upgrade safety && \
-		for req_file in `find . -type f -name '*requirements.txt'`; do \
-			echo "Checking file $$req_file" \
-			&& safety check --full-report -r $$req_file \
-			&& echo -e '\n' \
-			|| exit 1; \
-		done
-
-.PHONY: sync-requirements
-sync-requirements:  ## Update dev-requirements.txt to pin to the same versions of prod dependencies
-	rm -r requirements/dev-requirements.txt && cp requirements/requirements.txt requirements/dev-requirements.txt
-	pip-compile --allow-unsafe --generate-hashes --output-file requirements/dev-requirements.txt requirements/requirements.in requirements/dev-requirements.in
-
-.PHONY: requirements
-requirements:  ## Update *requirements.txt files if pinned versions do not comply with the dependency specifications in *requirements.in
-	pip-compile --generate-hashes --output-file requirements/requirements.txt requirements/requirements.in
-	$(MAKE) sync-requirements
-
-.PHONY: update-dependency
-update-dependency:  ## Add or upgrade a package to the latest version that complies with the dependency specifications in requirements.in
-	pip-compile --generate-hashes --upgrade-package $(PACKAGE) --output-file requirements/requirements.txt requirements/requirements.in
-	$(MAKE) sync-requirements
-
-.PHONY: update-dev-only-dependencies
-update-dev-only-dependencies:  ## Update dev-requirements.txt to pin to the latest versions of dev-only dependencies that comply with the dependency specifications in dev-requirements.in
-	$(MAKE) sync-requirements
-	@while read line; do \
-		pip-compile --allow-unsafe --generate-hashes --upgrade-package $file --output-file requirements/dev-requirements.txt requirements/requirements.in requirements/dev-requirements.in; \
-	done < 'requirements/dev-requirements.in'
+	@echo "Running safety against build requirements…"
+	@poetry run safety check --full-report -r build-requirements.txt
 
 .PHONY: check
 check: flake8 mypy test
 
 .PHONY: flake8
 flake8: ## Run flake8 to lint Python files
-	flake8
+	@poetry run flake8
 
 mypy: ## Type check Python files
-	mypy
+	@poetry run mypy
 
 test: ## Run Python unit tests
-	python3 -m unittest
+	@poetry run python3 -m unittest
 
-# Explaination of the below shell command should it ever break.
+# Explanation of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" and any make targets that might appear between : and ##
 # 2. Use sed-like syntax to remove the make targets
 # 3. Format the split fields into $$1) the target name (in blue) and $$2) the target descrption

README.md

@@ -2,15 +2,15 @@
 
 # securedrop-log
 
-This is a Python module and qrexec service for logging in Qubes for [SecureDrop](https://securedrop.org).
+`securedrop-log` is part of the [SecureDrop
+Workstation](https://github.com/freedomofpress/securedrop-workstation) project.
 
-## How to upgrade the dependencies?
+This is a Python module and qrexec service for logging in Qubes.
 
-To upgrade one single Python dependency, say `redis`, run the following:
+#### Quick Start
 
-```bash
-PACKAGE=redis make upgrade-pip
-```
+1. [Install Poetry](https://python-poetry.org/docs/#installing-with-the-official-installer)
+2. Run `make test` to verify the installation
 
 ## How to use/try this?