Don't write to Fromager's virtual env

Until now, Fromager was installing build dependencies into its virtual
environment. The build dependencies were necessary to run
`pyproject_hooks`. Some hooks expect build dependencies and build system
providers to be installed.

Fromager now uses a new virtualenv that is bound to the `WorkContext`.
The build dependencies are installed into the work environment. The hook
caller uses the interpreter from the work env instead of Fromager's
interpreter.

The context environment will eventually go away after we have refactored
the code to use the package's build env.

Signed-off-by: Christian Heimes <[email protected]>

src/fromager/build_environment.py

@@ -79,10 +79,12 @@ def __init__(
         ctx: context.WorkContext,
         parent_dir: pathlib.Path,
         build_requirements: typing.Iterable[Requirement] | None,
+        clear: bool = False,
     ):
         self._ctx = ctx
         self.path = parent_dir.absolute() / f"build-{platform.python_version()}"
         self._build_requirements = build_requirements
+        self._clear = clear
         self._createenv()
 
     @property
@@ -160,8 +162,23 @@ def _createenv(self) -> None:
             return
 
         logger.debug("creating build environment in %s", self.path)
+        cmd = [
+            sys.executable,
+            "-m",
+            "virtualenv",
+            "--python",
+            sys.executable,
+            "--extra-search-dir",
+            str(self._ctx.wheels_downloads),
+            "--pip=bundle",
+            "--setuptools=bundle",
+            "--wheel=none",
+        ]
+        if self._clear:
+            cmd.append("--clear")
+        cmd.append(str(self.path))
         external_commands.run(
-            [sys.executable, "-m", "virtualenv", str(self.path)],
+            cmd,
             network_isolation=False,
         )
         logger.info("created build environment in %s", self.path)
@@ -327,9 +344,10 @@ def _safe_install(
     req_type: RequirementType,
 ):
     logger.debug("installing %s %s", req_type, req)
-    external_commands.run(
+    build_env = ctx.get_build_env()
+    build_env.run(
         [
-            sys.executable,
+            str(build_env.python),
             "-m",
             "pip",
             "-vvv",

src/fromager/context.py

@@ -11,6 +11,7 @@
 from packaging.version import Version
 
 from . import (
+    build_environment,
     constraints,
     dependency_graph,
     packagesettings,
@@ -73,6 +74,7 @@ def __init__(
 
         self._build_order_filename = self.work_dir / "build-order.json"
         self._constraints_filename = self.work_dir / "constraints.txt"
+        self._build_env: build_environment.BuildEnvironment | None = None
 
         # Push items onto the stack as we start to resolve their
         # dependencies so at the end we have a list of items that need to
@@ -91,6 +93,21 @@ def __init__(
             set()
         )
 
+    def get_build_env(self) -> build_environment.BuildEnvironment:
+        """Get / create work virtual env
+
+        The virtual environment is used for build dependencies for pyproject
+        hooks.
+        """
+        if self._build_env is None:
+            self._build_env = build_environment.BuildEnvironment(
+                ctx=self,
+                parent_dir=self.work_dir,
+                build_requirements=(),
+                clear=True,  # start with a clean env
+            )
+        return self._build_env
+
     @property
     def pip_wheel_server_args(self) -> list[str]:
         args = ["--index-url", self.wheel_server_url]

src/fromager/dependencies.py

@@ -133,7 +133,12 @@ def default_get_build_backend_dependencies(
     pyproject_toml = get_pyproject_contents(build_dir)
     extra_environ = pbi.get_extra_environ()
     hook_caller = get_build_backend_hook_caller(
-        build_dir, pyproject_toml, override_environ=extra_environ
+        ctx=ctx,
+        sdist_root_dir=sdist_root_dir,
+        build_dir=pbi.build_dir(sdist_root_dir),
+        pyproject_toml=pyproject_toml,
+        override_environ=extra_environ,
+        network_isolation=ctx.network_isolation,
     )
     return hook_caller.get_requires_for_build_wheel()
 
@@ -188,7 +193,12 @@ def default_get_build_sdist_dependencies(
     pyproject_toml = get_pyproject_contents(build_dir)
     extra_environ = pbi.get_extra_environ()
     hook_caller = get_build_backend_hook_caller(
-        build_dir, pyproject_toml, override_environ=extra_environ
+        ctx=ctx,
+        sdist_root_dir=sdist_root_dir,
+        build_dir=build_dir,
+        pyproject_toml=pyproject_toml,
+        override_environ=extra_environ,
+        network_isolation=ctx.network_isolation,
     )
     return hook_caller.get_requires_for_build_wheel()
 
@@ -238,12 +248,15 @@ def get_build_backend(pyproject_toml: dict[str, typing.Any]) -> dict[str, typing
 
 
 def get_build_backend_hook_caller(
+    ctx: context.WorkContext,
     sdist_root_dir: pathlib.Path,
+    build_dir: pathlib.Path,
     pyproject_toml: dict[str, typing.Any],
     override_environ: dict[str, typing.Any],
     *,
     network_isolation: bool = False,
 ) -> pyproject_hooks.BuildBackendHookCaller:
+    build_env = ctx.get_build_env()
     backend = get_build_backend(pyproject_toml)
 
     def _run_hook_with_extra_environ(
@@ -252,25 +265,27 @@ def _run_hook_with_extra_environ(
         extra_environ: typing.Mapping[str, str] | None = None,
     ) -> None:
         """The BuildBackendHookCaller is going to pass extra_environ
-        and our build system may want to set some values, too. Merge
-        the 2 sets of values before calling the actual runner function.
+        and our build system may want to set some values, too. The hook
+        also needs env vars from the build environment's virtualenv. Merge
+        the 3 sets of values before calling the actual runner function.
         """
-        full_environ: dict[str, typing.Any] = {}
-        if extra_environ is not None:
-            full_environ.update(extra_environ)
-        full_environ.update(override_environ)
+        extra_environ = dict(extra_environ) if extra_environ else {}
+        extra_environ.update(override_environ)
+        extra_environ.update(build_env.get_venv_environ(template_env=extra_environ))
         external_commands.run(
             cmd,
             cwd=cwd,
-            extra_environ=full_environ,
+            extra_environ=extra_environ,
             network_isolation=network_isolation,
         )
 
     return pyproject_hooks.BuildBackendHookCaller(
-        source_dir=str(sdist_root_dir),
+        # sources may be in a subdirectory (PyArrow, Triton, ...)
+        source_dir=str(build_dir),
         build_backend=backend["build-backend"],
         backend_path=backend["backend-path"],
         runner=_run_hook_with_extra_environ,
+        python_executable=str(build_env.python),
     )
 
 

src/fromager/sources.py

@@ -499,10 +499,13 @@ def pep517_build_sdist(
 ) -> pathlib.Path:
     """Use the PEP 517 API to build a source distribution from a modified source tree."""
     pyproject_toml = dependencies.get_pyproject_contents(sdist_root_dir)
+    pbi = ctx.package_build_info(req)
     hook_caller = dependencies.get_build_backend_hook_caller(
-        sdist_root_dir,
-        pyproject_toml,
-        extra_environ,
+        ctx=ctx,
+        sdist_root_dir=sdist_root_dir,
+        build_dir=pbi.build_dir(sdist_root_dir),
+        pyproject_toml=pyproject_toml,
+        override_environ=extra_environ,
         network_isolation=ctx.network_isolation,
     )
     sdist_filename = hook_caller.build_sdist(ctx.sdists_builds)