diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 15a87aa6..d30fff3c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -73,6 +73,7 @@ jobs: - rust_vendor - download_sequence - optimize_build + - extra_metadata steps: - name: Get source diff --git a/.mergify.yml b/.mergify.yml index 82865a81..23723192 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -75,6 +75,9 @@ pull_request_rules: - check-success=e2e (3.11, 1.75, optimize_build) - check-success=e2e (3.12, 1.75, optimize_build) + - check-success=e2e (3.11, 1.75, extra_metadata) + - check-success=e2e (3.12, 1.75, extra_metadata) + - "-draft" # At least 1 reviewer diff --git a/e2e/test_extra_metadata.sh b/e2e/test_extra_metadata.sh new file mode 100755 index 00000000..6f0c0569 --- /dev/null +++ b/e2e/test_extra_metadata.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test whether extra metadata was added in the wheels or not + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +fromager \ + --log-file="$OUTDIR/bootstrap.log" \ + --error-log-file="$OUTDIR/fromager-errors.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --settings-file="$SCRIPTDIR/build_settings.yaml" \ + bootstrap 'stevedore==5.2.0' + +find "$OUTDIR/wheels-repo/" -name '*.whl' +find "$OUTDIR/sdists-repo/" -name '*.tar.gz' +ls "$OUTDIR"/work-dir/*/build.log || true + +EXPECTED_FILES=" +$OUTDIR/wheels-repo/downloads/setuptools-*.whl +$OUTDIR/wheels-repo/downloads/pbr-*.whl +$OUTDIR/wheels-repo/downloads/stevedore-*.whl +" + +pass=true +for pattern in $EXPECTED_FILES; do + if [ ! -f "${pattern}" ]; then + echo "Did not find $pattern" 1>&2 + pass=false + fi +done + +$pass + +wheel unpack $OUTDIR/wheels-repo/downloads/stevedore-5.2.0-py3-none-any.whl -d $OUTDIR + +EXPECTED_FILES=" +$OUTDIR/stevedore-5.2.0/stevedore-5.2.0.dist-info/BUILD +$OUTDIR/stevedore-5.2.0/stevedore-5.2.0.dist-info/build-backend-requirements.txt +$OUTDIR/stevedore-5.2.0/stevedore-5.2.0.dist-info/build-system-requirements.txt +$OUTDIR/stevedore-5.2.0/stevedore-5.2.0.dist-info/build-sdist-requirements.txt +" + +pass=true +for pattern in $EXPECTED_FILES; do + if [ ! -f "${pattern}" ]; then + echo "Did not find $pattern" 1>&2 + pass=false + fi +done + +$pass + +for file in $EXPECTED_FILES; do + cat $file +done diff --git a/requirements.txt b/requirements.txt index 8e8d2cbb..e7674124 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ stevedore tomlkit tqdm virtualenv +wheel diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index a51d5fa6..39dc9093 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -3,6 +3,7 @@ import os import pathlib import platform +import shutil import sys import tempfile import typing @@ -10,6 +11,7 @@ from datetime import datetime import elfdeps +import tomlkit from packaging.requirements import Requirement from packaging.utils import parse_wheel_filename from packaging.version import Version @@ -129,6 +131,73 @@ def analyze_wheel_elfdeps( return requires, provides +def add_extra_metadata_to_wheels( + ctx: context.WorkContext, + req: Requirement, + version: Version, + extra_environ: dict[str, str], + sdist_root_dir: pathlib.Path, + wheel_file: pathlib.Path, +) -> None: + extra_data_plugin = overrides.find_override_method( + req.name, "add_extra_metadata_to_wheels" + ) + data_to_add = {} + if extra_data_plugin: + data_to_add = overrides.invoke( + extra_data_plugin, + ctx=ctx, + req=req, + version=version, + extra_environ=extra_environ, + sdist_root_dir=sdist_root_dir, + ) + if not isinstance(data_to_add, dict): + logger.warning( + f"{req.name}: unexpected return type from plugin add_extra_metadata_to_wheels. Expected dictionary. Will ignore" + ) + data_to_add = {} + + with tempfile.TemporaryDirectory() as dir_name: + cmd = ["wheel", "unpack", str(wheel_file), "-d", dir_name] + external_commands.run( + cmd, + cwd=dir_name, + network_isolation=ctx.network_isolation, + ) + + wheel_file_name_parts = wheel_file.name.split("-") + distribution_name = f"{wheel_file_name_parts[0]}-{wheel_file_name_parts[1]}" + dist_info_dir = ( + pathlib.Path(dir_name) + / distribution_name + / f"{distribution_name}.dist-info" + ) + if not dist_info_dir.exists(): + logger.debug( + f"{req.name}: could not add additional metadata. {dist_info_dir} does not exist" + ) + return + + build_file = dist_info_dir / "BUILD" + build_file.write_text( + tomlkit.dumps(ctx.settings.get_package_settings(req.name)) + ) + if data_to_add: + build_file.write_text(tomlkit.dumps(data_to_add)) + + req_files = sdist_root_dir.parent.glob("*-requirements.txt") + for req_file in req_files: + shutil.copy(req_file, dist_info_dir) + + cmd = ["wheel", "pack", str(dist_info_dir.parent), "-d", str(wheel_file.parent)] + external_commands.run( + cmd, + cwd=dir_name, + network_isolation=ctx.network_isolation, + ) + + def build_wheel( ctx: context.WorkContext, req: Requirement, @@ -172,6 +241,14 @@ def build_wheel( # TODO: raise error? return None wheel = wheels[0] + add_extra_metadata_to_wheels( + ctx=ctx, + req=req, + version=version, + extra_environ=extra_environ, + sdist_root_dir=sdist_root_dir, + wheel_file=wheel, + ) logger.info(f"{req.name}: built wheel '{wheel}' in {end - start}") analyze_wheel_elfdeps(ctx, req, wheel) return wheel