Merge pull request #467 from freedomofpress/poetry-support

Add support for building packages that use Poetry

.circleci/config.yml

@@ -60,6 +60,7 @@ common-steps:
     run:
       name: Build debian package
       command: |
+        source .venv/bin/activate
         export VERSION_CODENAME=$(~/project/scripts/codename)
         export PKG_PATH=~/packaging/$PKG_NAME/
         export PKG_VERSION=$VERSION_TO_BUILD

Makefile

@@ -46,17 +46,15 @@ requirements: ## Creates requirements files for the Python projects
 	./scripts/update-requirements
 
 .PHONY: build-wheels
-build-wheels: ## Builds the wheels and adds them to the localwheels directory
+build-wheels: ## Builds the wheels and adds them to the wheels subdirectory
 	./scripts/verify-sha256sum-signature $$(basename ${PKG_DIR})
 	./scripts/build-sync-wheels
 	./scripts/sync-sha256sums $$(basename ${PKG_DIR})
-	@printf "Done! Now please follow the instructions in\n"
-	@printf "https://github.com/freedomofpress/securedrop-debian-packaging-guide/"
-	@printf "to push these changes to the FPF PyPI index\n"
+	@echo Done!
 
 .PHONY: test
 test: ## Run simple test suite (skips reproducibility checks)
-	pytest -v tests/test_update_requirements.py tests/test_deb_package.py
+	pytest -v tests/test_update_requirements.py tests/test_deb_package.py tests/test_utils.py
 
 .PHONY: reprotest
 reprotest: ## Runs only reproducibility tests, for .deb and .whl files

README.md

@@ -4,7 +4,7 @@
 
 # securedrop-builder
 
-`securedrop-builder` is the tool we use to package Python projects into Debian packages for the [SecureDrop Workstation](https://github.com/freedomofpress/securedrop-workstation).
+`securedrop-builder` is the tool we use to package Python projects into Debian packages for the [SecureDrop Workstation](https://github.com/freedomofpress/securedrop-workstation). This repository also holds copies of reproducibly built wheels included with some Debian packages, in the `wheels` subdirectory of the package.
 
 * For instructions on how to build [SecureDrop](https://github.com/freedomofpress/securedrop) Debian packages, see https://developers.securedrop.org/en/latest/release_management.html.
 
@@ -100,8 +100,9 @@ the requirements files which are used for build of these packages (`build-requir
 using `make requirements` are kept up to date in latest `main` of those repositories.
 
 If new dependencies were added in the `build-requirements.txt` of that
-repo that are not in the FPF PyPI mirror (`./localwheels/` in this repository), then the maintainer needs
-to do the following (we are taking `securedrop-client` project as example):
+repo that are not in the `wheels` subdirectory for the package in this repository,
+then the maintainer needs to do the following (we are taking `securedrop-client` project
+as an example):
 
 ### 0. Enable the virtualenv
 
@@ -133,8 +134,7 @@ pytest==3.10.1
 
 Please build the wheel by using the following command.
 	PKG_DIR=/home/user/code/securedrop-client make build-wheels
-Then add the newly built wheels and sources to ./localwheels/.
-Also update the index HTML files accordingly commit your changes.
+Then add the newly built wheels and sources to the `wheels` subdirectory for the package.
 After these steps, please rerun the command again.
 ```
 
@@ -153,12 +153,13 @@ This above command will let you know about any new wheels + sources. It will
 build/download sources from PyPI (by verifying it against the sha256sums from
 the `requirements.txt` of the project).
 
-### 3. Commit changes to the localwheels directory (if only any update of wheels)
+### 3. Commit changes to the wheels directory (if only any update of wheels)
 
-Now add these built artifacts to version control:
+Now add these built artifacts to version control, from the relevant package
+directory:
 
 ```shell
-git add localwheels/
+git add wheels/
 git commit
 ```
 

scripts/build-sync-wheels

@@ -19,9 +19,8 @@ if os.geteuid() == 0:
     # sys.exit(1)
 if "VIRTUAL_ENV" not in os.environ:
     print(
-        "This script should be run in a virtualenv: "
-        "`source .venv/bin/activate`",
-        file=sys.stderr
+        "This script should be run in a virtualenv: `source .venv/bin/activate`",
+        file=sys.stderr,
     )
     sys.exit(1)
 
@@ -43,17 +42,18 @@ WHEEL_BUILD_DIR = "/tmp/pip-wheel-build"
 
 
 def main():
-    if "PKG_DIR" in os.environ and \
-            not any(arg.startswith("--pkg-dir") for arg in sys.argv):
+    if "PKG_DIR" in os.environ and not any(
+        arg.startswith("--pkg-dir") for arg in sys.argv
+    ):
         sys.argv.extend(["--pkg-dir", os.environ["PKG_DIR"]])
 
-    parser = argparse.ArgumentParser(
-        description="Builds and stores sources and wheels"
-    )
+    parser = argparse.ArgumentParser(description="Builds and stores sources and wheels")
     parser.add_argument("--pkg-dir", help="Package directory", required=True)
     parser.add_argument("--project", help="Project name to update")
     parser.add_argument(
-        "--clobber", action="store_true", default=False,
+        "--clobber",
+        action="store_true",
+        default=False,
         help="Whether to overwrite wheels and source tarballs",
     )
     parser.add_argument(
@@ -79,12 +79,25 @@ def main():
     if args.project is not None:
         project_name = args.project
     else:
-        project_name = utils.project_name(Path(args.pkg_dir))
+        project_name = utils.get_project_name(Path(args.pkg_dir))
     local_wheels = os.path.join(project_name, "wheels")
 
+    req_path = os.path.join(args.pkg_dir, args.requirements, "requirements.txt")
+    poetry_lock_path = os.path.join(args.pkg_dir, "poetry.lock")
+
+    use_poetry = False
+    # Check if requirements.txt exists, if not check for poetry.lock and create a temporary requirements.txt
     if not os.path.exists(req_path):
-        print("requirements.txt missing at {0}.".format(req_path))
-        sys.exit(3)
+        if os.path.exists(poetry_lock_path):
+            use_poetry = True
+            print(
+                f"requirements.txt was not found at {req_path}, but poetry.lock was found at {poetry_lock_path}, using."
+            )
+        else:
+            print(
+                f"requirements.txt not found at {req_path} and poetry.lock not found at {poetry_lock_path}."
+            )
+            sys.exit(3)
 
     if os.path.exists(WHEEL_BUILD_DIR):
         shutil.rmtree(WHEEL_BUILD_DIR)
@@ -93,9 +106,22 @@ def main():
         os.mkdir(WHEEL_BUILD_DIR)
 
     with tempfile.TemporaryDirectory() as tmpdir:
+        if use_poetry:
+            poetry_reqs = utils.get_requirements_from_poetry(Path(args.pkg_dir) / 'poetry.lock', Path(args.pkg_dir) / 'pyproject.toml')
+            req_path = os.path.join(tmpdir, 'requirements.txt')
+            with open(req_path, 'w') as req_file:
+                req_file.write(poetry_reqs)
+
         # The --require-hashes option will be used by default if there are
         # hashes in the requirements.txt file. We specify it anyway to guard
         # against use of a requirements.txt file without hashes.
+        #
+        # NOTE: Even with this invocation, pip may execute build steps, as
+        # part of its metadata collection process. Switching to
+        # manual downloading and hash verification may be preferable
+        # to make this process more resilient.
+        #
+        # See https://github.com/pypa/pip/issues/1884 for background.
         cmd = [
             "pip3",
             "download",
@@ -122,7 +148,16 @@ def main():
         for project in project_names:
             print(f"Building {project}")
             source_path = os.path.join(WHEEL_BUILD_DIR, project)
-            cmd = ["python3", "-m", "build", "--wheel", source_path, "--no-isolation", "-o", tmpdir]
+            cmd = [
+                "python3",
+                "-m",
+                "build",
+                "--wheel",
+                source_path,
+                "--no-isolation",
+                "-o",
+                tmpdir,
+            ]
             subprocess.check_call(cmd)
             print(f"build command used: {' '.join(cmd)}")
 

scripts/sync-sha256sums

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# A script to update the sha256sums from localwheels directory
+# A script to update the sha256sums from local wheels directory
 
 import os
 import subprocess