PoC: automatically create a pinned extra with dependencies pinned from the lock file

docs/configuration.md

@@ -286,6 +286,21 @@ Use parallel execution when using the new (`>=1.1.0`) installer.
 Set the maximum number of retries in an unstable network.
 This setting has no effect if the server does not support HTTP range requests.
 
+### `installer.re-resolve`
+
+**Type**: `boolean`
+
+**Default**: `true`
+
+**Environment Variable**: `POETRY_INSTALLER_RE_RESOLVE`
+
+*Introduced in 2.0.0*
+
+If the config option is _not_ set and the lock file is at least version 2.1
+(created by Poetry 2.0 or above), the installer will not re-resolve dependencies
+but evaluate the locked markers to decide which of the locked dependencies have to
+be installed into the target environment.
+
 ### `solver.lazy-wheel`
 
 **Type**: `boolean`

src/poetry/config/config.py

@@ -131,6 +131,7 @@ class Config:
             "max-retries": 0,
         },
         "installer": {
+            "re-resolve": True,
             "parallel": True,
             "max-workers": None,
             "no-binary": None,
@@ -309,6 +310,7 @@ def _get_normalizer(name: str) -> Callable[[str], Any]:
             "virtualenvs.options.system-site-packages",
             "virtualenvs.options.prefer-active-python",
             "experimental.system-git-client",
+            "installer.re-resolve",
             "installer.parallel",
             "solver.lazy-wheel",
             "warnings.export",

src/poetry/console/commands/build.py

@@ -5,6 +5,9 @@
 from typing import ClassVar
 
 from cleo.helpers import option
+from poetry.core.packages.dependency_group import MAIN_GROUP
+from poetry.core.version.markers import MultiMarker
+from poetry.core.version.markers import parse_marker
 
 from poetry.console.commands.env_command import EnvCommand
 from poetry.utils.env import build_environment
@@ -68,8 +71,30 @@ def _build(
                 local=local_version_label
             )
 
-        for builder in builders:
-            builder(self.poetry, executable=executable).build(target_dir)
+        if not self.poetry.locker.is_locked_groups_and_markers():
+            raise RuntimeError("lock file version too old")
+        for package, info in self.poetry.locker.locked_packages().items():
+            if MAIN_GROUP not in info.groups:
+                continue
+            package.optional = True
+            dependency = package.to_dependency()
+            # We could just intersect 'extra == "pinned"' with the marker.
+            # However, since this extra is not part of the marker, we can avoid
+            # some work by building the MultiMarker manually. Besides,
+            # we can ensure that the extra is the first part of the marker.
+            extra_marker = parse_marker('extra == "pinned"')
+            locked_marker = info.get_marker({MAIN_GROUP})
+            if locked_marker.is_any():
+                dependency.marker = extra_marker
+            else:
+                # We explicitly do not MultiMarker.of() here
+                # because it is not necessary.
+                dependency.marker = MultiMarker(extra_marker, locked_marker)
+            self.poetry.package.add_dependency(dependency)
+        for builder_class in builders:
+            builder = builder_class(self.poetry, executable=executable)
+            builder._meta.provides_extra.append("pinned")
+            builder.build(target_dir)
 
     def handle(self) -> int:
         if not self.poetry.is_package_mode:

src/poetry/console/commands/config.py

@@ -75,6 +75,7 @@ def unique_config_values(self) -> dict[str, tuple[Any, Any]]:
             "virtualenvs.prompt": (str, str),
             "experimental.system-git-client": (boolean_validator, boolean_normalizer),
             "requests.max-retries": (lambda val: int(val) >= 0, int_normalizer),
+            "installer.re-resolve": (boolean_validator, boolean_normalizer),
             "installer.parallel": (boolean_validator, boolean_normalizer),
             "installer.max-workers": (lambda val: int(val) > 0, int_normalizer),
             "installer.no-binary": (

src/poetry/installation/installer.py

@@ -7,8 +7,7 @@
 from packaging.utils import canonicalize_name
 
 from poetry.installation.executor import Executor
-from poetry.installation.operations import Uninstall
-from poetry.installation.operations import Update
+from poetry.puzzle.transaction import Transaction
 from poetry.repositories import Repository
 from poetry.repositories import RepositoryPool
 from poetry.repositories.installed_repository import InstalledRepository
@@ -20,12 +19,14 @@
 
     from cleo.io.io import IO
     from packaging.utils import NormalizedName
+    from poetry.core.packages.package import Package
     from poetry.core.packages.path_dependency import PathDependency
     from poetry.core.packages.project_package import ProjectPackage
 
     from poetry.config.config import Config
     from poetry.installation.operations.operation import Operation
     from poetry.packages import Locker
+    from poetry.packages.transitive_package_info import TransitivePackageInfo
     from poetry.utils.env import Env
 
 
@@ -196,19 +197,20 @@ def _do_refresh(self) -> int:
         with solver.provider.use_source_root(
             source_root=self._env.path.joinpath("src")
         ):
-            ops = solver.solve(use_latest=use_latest).calculate_operations()
+            solved_packages = solver.solve(use_latest=use_latest).get_solved_packages()
 
-        lockfile_repo = LockfileRepository()
-        self._populate_lockfile_repo(lockfile_repo, ops)
-
-        self._write_lock_file(lockfile_repo, force=True)
+        self._write_lock_file(solved_packages, force=True)
 
         return 0
 
     def _do_install(self) -> int:
         from poetry.puzzle.solver import Solver
 
         locked_repository = Repository("poetry-locked")
+        reresolve = self._config.get("installer.re-resolve", True)
+        solved_packages: dict[Package, TransitivePackageInfo] = {}
+        lockfile_repo = LockfileRepository()
+
         if self._update:
             if not self._lock and self._locker.is_locked():
                 locked_repository = self._locker.locked_repository()
@@ -236,10 +238,17 @@ def _do_install(self) -> int:
             with solver.provider.use_source_root(
                 source_root=self._env.path.joinpath("src")
             ):
-                ops = solver.solve(use_latest=self._whitelist).calculate_operations()
+                solution = solver.solve(use_latest=self._whitelist)
+                solved_packages = solution.get_solved_packages()
 
-            lockfile_repo = LockfileRepository()
-            self._populate_lockfile_repo(lockfile_repo, ops)
+            if not self.executor.enabled:
+                # If we are only in lock mode, no need to go any further
+                self._write_lock_file(solved_packages)
+                return 0
+
+            for package in solved_packages:
+                if not lockfile_repo.has_package(package):
+                    lockfile_repo.add_package(package)
 
         else:
             self._io.write_line("<info>Installing dependencies from lock file</>")
@@ -249,6 +258,13 @@ def _do_install(self) -> int:
                     "pyproject.toml changed significantly since poetry.lock was last"
                     " generated. Run `poetry lock [--no-update]` to fix the lock file."
                 )
+            if not reresolve and not self._locker.is_locked_groups_and_markers():
+                if self._io.is_verbose():
+                    self._io.write_line(
+                        "<info>Cannot install without re-resolving"
+                        " because the lock file is not at least version 2.1</>"
+                    )
+                reresolve = True
 
             locker_extras = {
                 canonicalize_name(extra)
@@ -259,46 +275,63 @@ def _do_install(self) -> int:
                     raise ValueError(f"Extra [{extra}] is not specified.")
 
             locked_repository = self._locker.locked_repository()
-            lockfile_repo = locked_repository
-
-        if not self.executor.enabled:
-            # If we are only in lock mode, no need to go any further
-            self._write_lock_file(lockfile_repo)
-            return 0
+            if reresolve:
+                lockfile_repo = locked_repository
+            else:
+                solved_packages = self._locker.locked_packages()
 
         if self._io.is_verbose():
             self._io.write_line("")
             self._io.write_line(
                 "<info>Finding the necessary packages for the current system</>"
             )
 
-        if self._groups is not None:
-            root = self._package.with_dependency_groups(list(self._groups), only=True)
-        else:
-            root = self._package.without_optional_dependency_groups()
+        if reresolve:
+            if self._groups is not None:
+                root = self._package.with_dependency_groups(
+                    list(self._groups), only=True
+                )
+            else:
+                root = self._package.without_optional_dependency_groups()
 
-        # We resolve again by only using the lock file
-        packages = lockfile_repo.packages + locked_repository.packages
-        pool = RepositoryPool.from_packages(packages, self._config)
+            # We resolve again by only using the lock file
+            packages = lockfile_repo.packages + locked_repository.packages
+            pool = RepositoryPool.from_packages(packages, self._config)
 
-        solver = Solver(
-            root,
-            pool,
-            self._installed_repository.packages,
-            locked_repository.packages,
-            NullIO(),
-        )
-        # Everything is resolved at this point, so we no longer need
-        # to load deferred dependencies (i.e. VCS, URL and path dependencies)
-        solver.provider.load_deferred(False)
-
-        with solver.use_environment(self._env):
-            ops = solver.solve(use_latest=self._whitelist).calculate_operations(
-                with_uninstalls=self._requires_synchronization or self._update,
-                synchronize=self._requires_synchronization,
-                skip_directory=self._skip_directory,
-                extras=set(self._extras),
+            solver = Solver(
+                root,
+                pool,
+                self._installed_repository.packages,
+                locked_repository.packages,
+                NullIO(),
             )
+            # Everything is resolved at this point, so we no longer need
+            # to load deferred dependencies (i.e. VCS, URL and path dependencies)
+            solver.provider.load_deferred(False)
+
+            with solver.use_environment(self._env):
+                transaction = solver.solve(use_latest=self._whitelist)
+
+        else:
+            if self._groups is None:
+                groups = self._package.dependency_group_names()
+            else:
+                groups = set(self._groups)
+            transaction = Transaction(
+                locked_repository.packages,
+                solved_packages,
+                self._installed_repository.packages,
+                self._package,
+                self._env.marker_env,
+                groups,
+            )
+
+        ops = transaction.calculate_operations(
+            with_uninstalls=self._requires_synchronization or self._update,
+            synchronize=self._requires_synchronization,
+            skip_directory=self._skip_directory,
+            extras=set(self._extras),
+        )
 
         # Validate the dependencies
         for op in ops:
@@ -312,13 +345,17 @@ def _do_install(self) -> int:
 
         if status == 0 and self._update:
             # Only write lock file when installation is success
-            self._write_lock_file(lockfile_repo)
+            self._write_lock_file(solved_packages)
 
         return status
 
-    def _write_lock_file(self, repo: LockfileRepository, force: bool = False) -> None:
+    def _write_lock_file(
+        self,
+        packages: dict[Package, TransitivePackageInfo],
+        force: bool = False,
+    ) -> None:
         if not self.is_dry_run() and (force or self._update):
-            updated_lock = self._locker.set_lock_data(self._package, repo.packages)
+            updated_lock = self._locker.set_lock_data(self._package, packages)
 
             if updated_lock:
                 self._io.write_line("")
@@ -327,16 +364,5 @@ def _write_lock_file(self, repo: LockfileRepository, force: bool = False) -> Non
     def _execute(self, operations: list[Operation]) -> int:
         return self._executor.execute(operations)
 
-    def _populate_lockfile_repo(
-        self, repo: LockfileRepository, ops: Iterable[Operation]
-    ) -> None:
-        for op in ops:
-            if isinstance(op, Uninstall):
-                continue
-
-            package = op.target_package if isinstance(op, Update) else op.package
-            if not repo.has_package(package):
-                repo.add_package(package)
-
     def _get_installed(self) -> InstalledRepository:
         return InstalledRepository.load(self._env)