Merge pull request #122 from freedomofpress/only-poetry

Use Poetry for dependency management

.circleci/config.yml

@@ -1,54 +1,64 @@
 ---
 common-steps:
+  - &install_poetry
+    run:
+      name: Install Poetry
+      command: |
+        set -e
+        source /etc/os-release
+        if [[ "$VERSION_CODENAME" == "bullseye" ]]; then
+            # Install Poetry via PyPI
+            apt-get update && apt-get install --yes --no-install-recommends python3-pip
+            pip install poetry==1.6.1
+        elif [[ "$VERSION_CODENAME" == "bookworm" ]]; then
+            # Install Poetry via system package
+            apt-get update && apt-get install --yes --no-install-recommends python3-poetry
+        else
+            echo "Unsupported Debian version: $VERSION_CODENAME"
+            exit 1
+        fi
+
   - &install_testing_dependencies
     run:
       name: Install testing dependencies
       command: |
-        set -e
-        apt update && apt install -y git gnupg make python3-dev python3-venv
+        apt-get install --yes --no-install-recommends git gnupg make
+        poetry install --no-ansi
 
   - &install_build_dependencies
     run:
       name: Install build dependencies
       command: |
         set -e
-        apt update && apt install -y git make sudo
+        apt-get update && apt-get install --yes git make sudo
 
   - &run_unit_tests
     run:
       name: Install requirements and run unit tests
       command: |
-        set -e
-        make venv
-        source .venv/bin/activate
         export PYTHONPATH=$PYTHONPATH:.  # so alembic can get to Base metadata
         make test
 
   - &run_lint
     run:
       name: Run lint, type checking, code formatting
       command: |
-        set -e
-        make venv
-        source .venv/bin/activate
         make lint
 
   - &check_security
     run:
       name: Run static analysis on source code to find security issues
       command: |
         set -e
-        make venv
-        source .venv/bin/activate
+        poetry update bandit
         make bandit
 
   - &check_python_dependencies_for_vulnerabilities
     run:
       name: Check Python dependencies for known vulnerabilities
       command: |
         set -e
-        make venv
-        source .venv/bin/activate
+        poetry update safety
         make safety
 
   - &install_packaging_dependencies
@@ -59,10 +69,11 @@ common-steps:
         mkdir ~/packaging && cd ~/packaging
         # local builds may not have an ssh url, so || true
         git config --global --unset url.ssh://[email protected] || true
-        git clone https://github.com/freedomofpress/securedrop-debian-packaging.git
-        cd securedrop-debian-packaging
+        git clone https://github.com/freedomofpress/securedrop-builder.git
+        cd securedrop-builder
         apt-get update && apt-get install -y sudo make
         make install-deps
+        source .venv/bin/activate
         PKG_DIR=~/project make requirements
 
   - &verify_requirements
@@ -78,9 +89,10 @@ common-steps:
     run:
       name: Build debian package
       command: |
-        cd ~/packaging/securedrop-debian-packaging
+        cd ~/packaging/securedrop-builder
         export PKG_VERSION=1000.0
         export PKG_PATH=~/project/
+        source .venv/bin/activate
         make securedrop-proxy
 
 version: 2.1
@@ -93,8 +105,8 @@ jobs:
     docker: &docker
       - image: debian:<< parameters.image >>
     steps:
-      - *install_build_dependencies
       - checkout
+      - *install_build_dependencies
       - *install_packaging_dependencies
       - *verify_requirements
       - *build_debian_package
@@ -103,8 +115,9 @@ jobs:
     parameters: *parameters
     docker: *docker
     steps:
-      - *install_testing_dependencies
       - checkout
+      - *install_poetry
+      - *install_testing_dependencies
       - *run_unit_tests
       - store_test_results:
           path: test-results
@@ -113,24 +126,27 @@ jobs:
     parameters: *parameters
     docker: *docker
     steps:
-      - *install_testing_dependencies
       - checkout
+      - *install_poetry
+      - *install_testing_dependencies
       - *run_lint
 
   check-security:
     parameters: *parameters
     docker: *docker
     steps:
-      - *install_testing_dependencies
       - checkout
+      - *install_poetry
+      - *install_testing_dependencies
       - *check_security
 
   check-python-security:
     parameters: *parameters
     docker: *docker
     steps:
-      - *install_testing_dependencies
       - checkout
+      - *install_poetry
+      - *install_testing_dependencies
       - *check_python_dependencies_for_vulnerabilities
 
 

MANIFEST.in

@@ -4,7 +4,6 @@ include securedrop_proxy/VERSION
 include changelog.md
 include config-example.yaml
 include qubes/securedrop.Proxy
-include requirements/build-requirements.txt
-include requirements/requirements.txt
+include build-requirements.txt
 include securedrop_proxy/*.py
 include setup.py

Makefile

@@ -3,83 +3,53 @@
 .PHONY: all
 all: help
 
-.PHONY: venv
-venv:
-	python3 -m venv .venv ## Provision a Python 3 virtualenv for **development**
-	.venv/bin/pip install --upgrade pip wheel
-	.venv/bin/pip install --require-hashes -r "requirements/dev-requirements.txt"
-
 .PHONY: bandit
 bandit: ## Run bandit with medium level excluding test-related folders
-	pip install --upgrade pip && \
-		pip install --upgrade bandit!=1.6.0 && \
-	bandit -ll --recursive securedrop_proxy
+	@echo "Running bandit security checks…"
+	@poetry run bandit -ll --recursive securedrop_proxy
 
 .PHONY: safety
 safety: ## Runs `safety check` to check python dependencies for vulnerabilities
-	pip install --upgrade safety && \
-		for req_file in `find . -type f -name '*requirements.txt'`; do \
-			echo "Checking file $$req_file" \
-			&& safety check --full-report -r $$req_file \
-			&& echo -e '\n' \
-			|| exit 1; \
-		done
+	@echo "Running safety against build requirements…"
+	@poetry run safety check --full-report -r build-requirements.txt
 
 .PHONY: lint
 lint: check-isort check-black mypy ## Run isort, black and flake8 and mypy
-	@flake8 securedrop_proxy tests
+	@poetry run flake8 securedrop_proxy tests
 
 .PHONY: mypy
 mypy: ## Run mypy static type checker
-	@mypy --ignore-missing-imports securedrop_proxy
+	@poetry run mypy --ignore-missing-imports securedrop_proxy
 
 .PHONY: black
 black: ## Run black for file formatting
-	@black securedrop_proxy tests
+	@echo "Running black (may result in changes in your working directory)…"
+	@poetry run black securedrop_proxy tests
 
 .PHONY: check-black
 check-black: ## Check Python source code formatting with black
-	@black --check --diff securedrop_proxy tests
+	@echo "Running black formatting check…"
+	@poetry run black --check --diff securedrop_proxy tests
 
 .PHONY: isort
 isort: ## Run isort for file formatting
-	@isort securedrop_proxy/*.py tests/*.py
+	@echo "Running isort (may result in changes in your working directory)…"
+	@poetry run isort securedrop_proxy/*.py tests/*.py
 
 .PHONY: check-isort
 check-isort: ## Check isort for file formatting
-	@isort --check-only --diff securedrop_proxy/*.py tests/*.py
-
-.PHONY: sync-requirements
-sync-requirements:  ## Update dev-requirements.txt to pin to the same versions of prod dependencies
-	rm -r requirements/dev-requirements.txt && cp requirements/requirements.txt requirements/dev-requirements.txt
-	pip-compile --allow-unsafe --generate-hashes --output-file requirements/dev-requirements.txt requirements/requirements.in requirements/dev-requirements.in
-
-.PHONY: requirements
-requirements:  ## Update *requirements.txt files if pinned versions do not comply with the dependency specifications in *requirements.in
-	pip-compile --generate-hashes --output-file requirements/requirements.txt requirements/requirements.in
-	$(MAKE) sync-requirements
-
-.PHONY: update-dependency
-update-dependency:  ## Add or upgrade a package to the latest version that complies with the dependency specifications in requirements.in
-	pip-compile --generate-hashes --upgrade-package $(PACKAGE) --output-file requirements/requirements.txt requirements/requirements.in
-	$(MAKE) sync-requirements
-
-.PHONY: update-dev-only-dependencies
-update-dev-only-dependencies:  ## Update dev-requirements.txt to pin to the latest versions of dev-only dependencies that comply with the dependency specifications in dev-requirements.in
-	$(MAKE) sync-requirements
-	@while read line; do \
-		pip-compile --allow-unsafe --generate-hashes --upgrade-package $file --output-file requirements/dev-requirements.txt requirements/requirements.in requirements/dev-requirements.in; \
-	done < 'requirements/dev-requirements.in'
+	@echo "Running isort module ordering check…"
+	@poetry run isort --check-only --diff securedrop_proxy/*.py tests/*.py
 
 .PHONY: test
 test: clean .coverage ## Runs tests with coverage
 
 .coverage:
-	@coverage run --source securedrop_proxy -m unittest
+	@poetry run coverage run --source securedrop_proxy -m unittest
 
 .PHONY: browse-coverage
 browse-coverage: .coverage ## Generates and opens HTML coverage report
-	@coverage html
+	@poetry run coverage html
 	@xdg-open htmlcov/index.html 2>/dev/null || open htmlcov/index.html 2>/dev/null
 
 .PHONY: check

README.md

@@ -29,27 +29,40 @@ https://github.com/freedomofpress/securedrop-workstation/issues/107.
 
 #### Quick Start
 
-To try the proxy script, create a virtual environment and install the
-requirements. In the root of the project directory, run
+1. [Install Poetry](https://python-poetry.org/docs/#installing-with-the-official-installer)
+2. Run `make test` to verify the installation
 
-```
-make venv
-source .venv/bin/activate
-make test
-```
+#### Managing Dependencies
 
-#### Update Dependencies
+We use Poetry to manage dependencies for this project.
 
-If you're adding or updating a dependency, you need to:
+### Development dependencies
 
-1. Modify either `dev-requirements.in` and `requirements.in` (depending on whether it is prod or dev only) and then run `make update-pip-dependencies`. This will generate `dev-requirements.txt` and `requirements.txt`.
+You can add development dependencies via  `poetry add <name> --group dev`.
+Make sure you commit changes to the lockfile along with changes to `pyproject.toml`.
 
-2. For building a debian package from this project, we use the requirements in
-`build-requirements.txt` which uses our pip mirror, i.e. the hashes in that file point to
-wheels on our pip mirror. A maintainer will need to add
-the updated dependency to our pip mirror (you can request this in the PR).
+To update the dependency to the latest version within the specified
+version constraints, simply run `poetry update <name>` and commit the resutling
+changes.
 
-3. Once the pip mirror is updated, you should checkout the [securedrop-debian-packaging repo](https://github.com/freedomofpress/securedrop-debian-packaging) and run `make requirements`. Commit the `build-requirements.txt` that results and add it to your PR.
+To update to a new major version (e.g., from 1.0.0 to 2.0.0), you will typically have to
+update `pyproject.toml`.
+
+### Production dependencies
+
+To add a production dependency, use `poetry add <name>`, and to update it,
+use `poetry update <name>`.
+
+For our production Debian packages, we use locally built wheels instead of
+downloading wheels from PyPI.
+
+This means that whenever you add or update a production dependency, you also
+have to build and commit a new wheel according to the process described in the
+[securedrop-builder](https://github.com/freedomofpress/securedrop-builder)
+repository.
+
+This will result in an updated `build-requirements.txt` file you can add to your
+PR in this repository.
 
 ## Making a Release