From 06560d5c9c41661826006c081cfeedc2a91cd012 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 19 Sep 2024 13:47:21 +0200 Subject: [PATCH] Don't write to Fromager's virtual env Until now, Fromager was installing build dependencies into its virtual environment. The build dependencies were necessary to run `pyproject_hooks`. Some hooks expect build dependencies and build system providers to be installed. Fromager now uses a new virtualenv that is bound to the `WorkContext`. The build dependencies are installed into the work environment. The hook caller uses the interpreter from the work env instead of Fromager's interpreter. The context environment will eventually go away after we have refactored the code to use the package's build env. Signed-off-by: Christian Heimes --- src/fromager/build_environment.py | 24 ++++++++++++++++++--- src/fromager/context.py | 17 +++++++++++++++ src/fromager/dependencies.py | 35 ++++++++++++++++++++++--------- src/fromager/sources.py | 9 +++++--- 4 files changed, 69 insertions(+), 16 deletions(-) diff --git a/src/fromager/build_environment.py b/src/fromager/build_environment.py index 4249c36e..939f96a7 100644 --- a/src/fromager/build_environment.py +++ b/src/fromager/build_environment.py @@ -79,10 +79,12 @@ def __init__( ctx: context.WorkContext, parent_dir: pathlib.Path, build_requirements: typing.Iterable[Requirement] | None, + clear: bool = False, ): self._ctx = ctx self.path = parent_dir.absolute() / f"build-{platform.python_version()}" self._build_requirements = build_requirements + self._clear = clear self._createenv() @property @@ -160,8 +162,23 @@ def _createenv(self) -> None: return logger.debug("creating build environment in %s", self.path) + cmd = [ + sys.executable, + "-m", + "virtualenv", + "--python", + sys.executable, + "--extra-search-dir", + str(self._ctx.wheels_downloads), + "--pip=bundle", + "--setuptools=bundle", + "--wheel=none", + ] + if self._clear: + cmd.append("--clear") + cmd.append(str(self.path)) external_commands.run( - [sys.executable, "-m", "virtualenv", str(self.path)], + cmd, network_isolation=False, ) logger.info("created build environment in %s", self.path) @@ -327,9 +344,10 @@ def _safe_install( req_type: RequirementType, ): logger.debug("installing %s %s", req_type, req) - external_commands.run( + build_env = ctx.get_build_env() + build_env.run( [ - sys.executable, + str(build_env.python), "-m", "pip", "-vvv", diff --git a/src/fromager/context.py b/src/fromager/context.py index e98e6200..2fbf94d8 100644 --- a/src/fromager/context.py +++ b/src/fromager/context.py @@ -11,6 +11,7 @@ from packaging.version import Version from . import ( + build_environment, constraints, dependency_graph, packagesettings, @@ -73,6 +74,7 @@ def __init__( self._build_order_filename = self.work_dir / "build-order.json" self._constraints_filename = self.work_dir / "constraints.txt" + self._build_env: build_environment.BuildEnvironment | None = None # Push items onto the stack as we start to resolve their # dependencies so at the end we have a list of items that need to @@ -91,6 +93,21 @@ def __init__( set() ) + def get_build_env(self) -> build_environment.BuildEnvironment: + """Get / create work virtual env + + The virtual environment is used for build dependencies for pyproject + hooks. + """ + if self._build_env is None: + self._build_env = build_environment.BuildEnvironment( + ctx=self, + parent_dir=self.work_dir, + build_requirements=(), + clear=True, # start with a clean env + ) + return self._build_env + @property def pip_wheel_server_args(self) -> list[str]: args = ["--index-url", self.wheel_server_url] diff --git a/src/fromager/dependencies.py b/src/fromager/dependencies.py index b220ccda..97316f69 100644 --- a/src/fromager/dependencies.py +++ b/src/fromager/dependencies.py @@ -133,7 +133,12 @@ def default_get_build_backend_dependencies( pyproject_toml = get_pyproject_contents(build_dir) extra_environ = pbi.get_extra_environ() hook_caller = get_build_backend_hook_caller( - build_dir, pyproject_toml, override_environ=extra_environ + ctx=ctx, + sdist_root_dir=sdist_root_dir, + build_dir=pbi.build_dir(sdist_root_dir), + pyproject_toml=pyproject_toml, + override_environ=extra_environ, + network_isolation=ctx.network_isolation, ) return hook_caller.get_requires_for_build_wheel() @@ -188,7 +193,12 @@ def default_get_build_sdist_dependencies( pyproject_toml = get_pyproject_contents(build_dir) extra_environ = pbi.get_extra_environ() hook_caller = get_build_backend_hook_caller( - build_dir, pyproject_toml, override_environ=extra_environ + ctx=ctx, + sdist_root_dir=sdist_root_dir, + build_dir=build_dir, + pyproject_toml=pyproject_toml, + override_environ=extra_environ, + network_isolation=ctx.network_isolation, ) return hook_caller.get_requires_for_build_wheel() @@ -238,12 +248,15 @@ def get_build_backend(pyproject_toml: dict[str, typing.Any]) -> dict[str, typing def get_build_backend_hook_caller( + ctx: context.WorkContext, sdist_root_dir: pathlib.Path, + build_dir: pathlib.Path, pyproject_toml: dict[str, typing.Any], override_environ: dict[str, typing.Any], *, network_isolation: bool = False, ) -> pyproject_hooks.BuildBackendHookCaller: + build_env = ctx.get_build_env() backend = get_build_backend(pyproject_toml) def _run_hook_with_extra_environ( @@ -252,25 +265,27 @@ def _run_hook_with_extra_environ( extra_environ: typing.Mapping[str, str] | None = None, ) -> None: """The BuildBackendHookCaller is going to pass extra_environ - and our build system may want to set some values, too. Merge - the 2 sets of values before calling the actual runner function. + and our build system may want to set some values, too. The hook + also needs env vars from the build environment's virtualenv. Merge + the 3 sets of values before calling the actual runner function. """ - full_environ: dict[str, typing.Any] = {} - if extra_environ is not None: - full_environ.update(extra_environ) - full_environ.update(override_environ) + extra_environ = dict(extra_environ) if extra_environ else {} + extra_environ.update(override_environ) + extra_environ.update(build_env.get_venv_environ(template_env=extra_environ)) external_commands.run( cmd, cwd=cwd, - extra_environ=full_environ, + extra_environ=extra_environ, network_isolation=network_isolation, ) return pyproject_hooks.BuildBackendHookCaller( - source_dir=str(sdist_root_dir), + # sources may be in a subdirectory (PyArrow, Triton, ...) + source_dir=str(build_dir), build_backend=backend["build-backend"], backend_path=backend["backend-path"], runner=_run_hook_with_extra_environ, + python_executable=str(build_env.python), ) diff --git a/src/fromager/sources.py b/src/fromager/sources.py index 258f7593..3afc319c 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -499,10 +499,13 @@ def pep517_build_sdist( ) -> pathlib.Path: """Use the PEP 517 API to build a source distribution from a modified source tree.""" pyproject_toml = dependencies.get_pyproject_contents(sdist_root_dir) + pbi = ctx.package_build_info(req) hook_caller = dependencies.get_build_backend_hook_caller( - sdist_root_dir, - pyproject_toml, - extra_environ, + ctx=ctx, + sdist_root_dir=sdist_root_dir, + build_dir=pbi.build_dir(sdist_root_dir), + pyproject_toml=pyproject_toml, + override_environ=extra_environ, network_isolation=ctx.network_isolation, ) sdist_filename = hook_caller.build_sdist(ctx.sdists_builds)