Add initial support for Poetry

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Added initial support for the Poetry package manager. ([#261](https://github.com/heroku/buildpacks-python/pull/261))
+
 ## [0.16.0] - 2024-08-30
 
 ### Changed

README.md

@@ -31,22 +31,25 @@ docker run --rm -it -e "PORT=8080" -p 8080:8080 sample-app
 
 ## Application Requirements
 
-A `requirements.txt` file must be present at the root of your application's repository.
+A `requirements.txt` or `poetry.lock` file must be present in the root (top-level) directory of your app's source code.
 
 ## Configuration
 
 ### Python Version
 
 By default, the buildpack will install the latest version of Python 3.12.
 
-To install a different version, add a `runtime.txt` file to your app’s root directory that declares the exact version number to use:
+To install a different version, add a `runtime.txt` file to your app's root directory that declares the exact version number to use:
 
 ```term
 $ cat runtime.txt
 python-3.12.5
 ```
 
-In the future this buildpack will also support specifying the Python version via a `.python-version` file (see [#6](https://github.com/heroku/buildpacks-python/issues/6)).
+In the future this buildpack will also support specifying the Python version using:
+
+- A `.python-version` file: [#6](https://github.com/heroku/buildpacks-python/issues/6)
+- `tool.poetry.dependencies.python` in `pyproject.toml`: [#260](https://github.com/heroku/buildpacks-python/issues/260)
 
 ## Contributing
 

requirements/poetry.txt

@@ -0,0 +1 @@
+poetry==1.8.3

src/detect.rs

@@ -41,29 +41,29 @@ pub(crate) fn is_python_project_directory(app_dir: &Path) -> io::Result<bool> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::package_manager::PACKAGE_MANAGER_FILE_MAPPING;
+    use crate::package_manager::SUPPORTED_PACKAGE_MANAGERS;
 
     #[test]
-    fn is_python_project_valid_project() {
+    fn is_python_project_directory_valid_project() {
         assert!(
             is_python_project_directory(Path::new("tests/fixtures/pyproject_toml_only")).unwrap()
         );
     }
 
     #[test]
-    fn is_python_project_empty() {
+    fn is_python_project_directory_empty() {
         assert!(!is_python_project_directory(Path::new("tests/fixtures/empty")).unwrap());
     }
 
     #[test]
-    fn is_python_project_io_error() {
+    fn is_python_project_directory_io_error() {
         assert!(is_python_project_directory(Path::new("tests/fixtures/empty/.gitkeep")).is_err());
     }
 
     #[test]
     fn known_python_project_files_contains_all_package_manager_files() {
-        assert!(PACKAGE_MANAGER_FILE_MAPPING
-            .iter()
-            .all(|(filename, _)| { KNOWN_PYTHON_PROJECT_FILES.contains(filename) }));
+        assert!(SUPPORTED_PACKAGE_MANAGERS.iter().all(|package_manager| {
+            KNOWN_PYTHON_PROJECT_FILES.contains(&package_manager.packages_file())
+        }));
     }
 }

src/errors.rs

@@ -1,6 +1,8 @@
 use crate::django::DjangoCollectstaticError;
 use crate::layers::pip::PipLayerError;
 use crate::layers::pip_dependencies::PipDependenciesLayerError;
+use crate::layers::poetry::PoetryLayerError;
+use crate::layers::poetry_dependencies::PoetryDependenciesLayerError;
 use crate::layers::python::PythonLayerError;
 use crate::package_manager::DeterminePackageManagerError;
 use crate::python_version::{PythonVersion, PythonVersionError, DEFAULT_PYTHON_VERSION};
@@ -48,6 +50,8 @@ fn on_buildpack_error(error: BuildpackError) {
         BuildpackError::DjangoDetection(error) => on_django_detection_error(&error),
         BuildpackError::PipDependenciesLayer(error) => on_pip_dependencies_layer_error(error),
         BuildpackError::PipLayer(error) => on_pip_layer_error(error),
+        BuildpackError::PoetryDependenciesLayer(error) => on_poetry_dependencies_layer_error(error),
+        BuildpackError::PoetryLayer(error) => on_poetry_layer_error(error),
         BuildpackError::PythonLayer(error) => on_python_layer_error(error),
         BuildpackError::PythonVersion(error) => on_python_version_error(error),
     };
@@ -68,18 +72,46 @@ fn on_determine_package_manager_error(error: DeterminePackageManagerError) {
             "determining which Python package manager to use for this project",
             &io_error,
         ),
-        // TODO: Should this mention the setup.py / pyproject.toml case?
+        DeterminePackageManagerError::MultipleFound(package_managers) => {
+            let files_found = package_managers
+                .into_iter()
+                .map(|package_manager| {
+                    format!(
+                        "{} ({})",
+                        package_manager.packages_file(),
+                        package_manager.name()
+                    )
+                })
+                .collect::<Vec<String>>()
+                .join("\n");
+            log_error(
+                "Multiple Python package manager files were found",
+                formatdoc! {"
+                    Exactly one package manager file must be present in your app's source code,
+                    however, several were found:
+
+                    {files_found}
+
+                    Decide which package manager you want to use with your app, and then delete
+                    the file(s) and any config from the others.
+                "},
+            );
+        }
         DeterminePackageManagerError::NoneFound => log_error(
-            "No Python package manager files were found",
+            "Couldn't find any supported Python package manager files",
             indoc! {"
-                A pip requirements file was not found in your application's source code.
-                This file is required so that your application's dependencies can be installed.
+                Your app must have either a pip requirements file ('requirements.txt')
+                or Poetry lockfile ('poetry.lock') in the root directory of its source
+                code, so your app's dependencies can be installed.
 
-                Please add a file named exactly 'requirements.txt' to the root directory of your
-                application, containing a list of the packages required by your application.
+                If your app already has one of those files, check that it:
 
-                For more information on what this file should contain, see:
-                https://pip.pypa.io/en/stable/reference/requirements-file-format/
+                1. Is in the top level directory (not a subdirectory).
+                2. Has the correct spelling (the filenames are case-sensitive).
+                3. Isn't excluded by '.gitignore' or 'project.toml'.
+
+                Otherwise, add a package manager file to your app. If your app has
+                no dependencies, then create an empty 'requirements.txt' file.
             "},
         ),
     };
@@ -235,6 +267,76 @@ fn on_pip_dependencies_layer_error(error: PipDependenciesLayerError) {
     };
 }
 
+fn on_poetry_layer_error(error: PoetryLayerError) {
+    match error {
+        PoetryLayerError::InstallPoetryCommand(error) => match error {
+            StreamedCommandError::Io(io_error) => log_io_error(
+                "Unable to install Poetry",
+                "running 'python' to install Poetry",
+                &io_error,
+            ),
+            StreamedCommandError::NonZeroExitStatus(exit_status) => log_error(
+                "Unable to install Poetry",
+                formatdoc! {"
+                    The command to install Poetry did not exit successfully ({exit_status}).
+
+                    See the log output above for more information.
+
+                    In some cases, this happens due to an unstable network connection.
+                    Please try again to see if the error resolves itself.
+
+                    If that does not help, check the status of PyPI (the upstream Python
+                    package repository service), here:
+                    https://status.python.org
+                "},
+            ),
+        },
+        PoetryLayerError::LocateBundledPip(io_error) => log_io_error(
+            "Unable to locate the bundled copy of pip",
+            "locating the pip wheel file bundled inside the Python 'ensurepip' module",
+            &io_error,
+        ),
+    };
+}
+
+fn on_poetry_dependencies_layer_error(error: PoetryDependenciesLayerError) {
+    match error {
+        PoetryDependenciesLayerError::CreateVenvCommand(error) => match error {
+            StreamedCommandError::Io(io_error) => log_io_error(
+                "Unable to create virtual environment",
+                "running 'python -m venv' to create a virtual environment",
+                &io_error,
+            ),
+            StreamedCommandError::NonZeroExitStatus(exit_status) => log_error(
+                "Unable to create virtual environment",
+                formatdoc! {"
+                    The 'python -m venv' command to create a virtual environment did
+                    not exit successfully ({exit_status}).
+
+                    See the log output above for more information.
+                "},
+            ),
+        },
+        PoetryDependenciesLayerError::PoetryInstallCommand(error) => match error {
+            StreamedCommandError::Io(io_error) => log_io_error(
+                "Unable to install dependencies using Poetry",
+                "running 'poetry install' to install the app's dependencies",
+                &io_error,
+            ),
+            // TODO: Add more suggestions here as to possible causes (similar to pip)
+            StreamedCommandError::NonZeroExitStatus(exit_status) => log_error(
+                "Unable to install dependencies using Poetry",
+                formatdoc! {"
+                    The 'poetry install --sync --only main' command to install the app's
+                    dependencies failed ({exit_status}).
+
+                    See the log output above for more information.
+                "},
+            ),
+        },
+    };
+}
+
 fn on_django_detection_error(error: &io::Error) {
     log_io_error(
         "Unable to determine if this is a Django-based app",

src/layers/mod.rs

@@ -1,4 +1,6 @@
 pub(crate) mod pip;
 pub(crate) mod pip_cache;
 pub(crate) mod pip_dependencies;
+pub(crate) mod poetry;
+pub(crate) mod poetry_dependencies;
 pub(crate) mod python;

src/layers/poetry.rs

@@ -0,0 +1,142 @@
+use crate::packaging_tool_versions::POETRY_VERSION;
+use crate::python_version::PythonVersion;
+use crate::utils::StreamedCommandError;
+use crate::{utils, BuildpackError, PythonBuildpack};
+use libcnb::build::BuildContext;
+use libcnb::data::layer_name;
+use libcnb::layer::{
+    CachedLayerDefinition, EmptyLayerCause, InvalidMetadataAction, LayerState, RestoredLayerAction,
+};
+use libcnb::layer_env::{LayerEnv, ModificationBehavior, Scope};
+use libcnb::Env;
+use libherokubuildpack::log::log_info;
+use serde::{Deserialize, Serialize};
+use std::io;
+use std::path::Path;
+use std::process::Command;
+
+/// Creates a build-only layer containing Poetry.
+pub(crate) fn install_poetry(
+    context: &BuildContext<PythonBuildpack>,
+    env: &mut Env,
+    python_version: &PythonVersion,
+    python_layer_path: &Path,
+) -> Result<(), libcnb::Error<BuildpackError>> {
+    let new_metadata = PoetryLayerMetadata {
+        arch: context.target.arch.clone(),
+        distro_name: context.target.distro_name.clone(),
+        distro_version: context.target.distro_version.clone(),
+        python_version: python_version.to_string(),
+        poetry_version: POETRY_VERSION.to_string(),
+    };
+
+    let layer = context.cached_layer(
+        layer_name!("poetry"),
+        CachedLayerDefinition {
+            build: true,
+            launch: false,
+            invalid_metadata_action: &|_| InvalidMetadataAction::DeleteLayer,
+            restored_layer_action: &|cached_metadata: &PoetryLayerMetadata, _| {
+                let cached_poetry_version = cached_metadata.poetry_version.clone();
+                if cached_metadata == &new_metadata {
+                    (RestoredLayerAction::KeepLayer, cached_poetry_version)
+                } else {
+                    (RestoredLayerAction::DeleteLayer, cached_poetry_version)
+                }
+            },
+        },
+    )?;
+
+    // Move the Python user base directory to this layer instead of under HOME:
+    // https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUSERBASE
+    let mut layer_env = LayerEnv::new().chainable_insert(
+        Scope::Build,
+        ModificationBehavior::Override,
+        "PYTHONUSERBASE",
+        layer.path(),
+    );
+
+    match layer.state {
+        LayerState::Restored {
+            cause: ref cached_poetry_version,
+        } => {
+            log_info(format!("Using cached Poetry {cached_poetry_version}"));
+        }
+        LayerState::Empty { ref cause } => {
+            match cause {
+                EmptyLayerCause::InvalidMetadataAction { .. } => {
+                    log_info("Discarding cached Poetry since its layer metadata can't be parsed");
+                }
+                EmptyLayerCause::RestoredLayerAction {
+                    cause: cached_poetry_version,
+                } => {
+                    log_info(format!("Discarding cached Poetry {cached_poetry_version}"));
+                }
+                EmptyLayerCause::NewlyCreated => {}
+            }
+
+            log_info(format!("Installing Poetry {POETRY_VERSION}"));
+
+            // We use the pip wheel bundled within Python's standard library to install Poetry.
+            // Whilst Poetry does still require pip for some tasks (such as package uninstalls),
+            // it bundles its own copy for use as a fallback. As such we don't need to install pip
+            // into the user site-packages (and in fact, Poetry wouldn't use this install anyway,
+            // since it only finds an external pip if it exists in the target venv).
+            let bundled_pip_module_path =
+                utils::bundled_pip_module_path(python_layer_path, python_version)
+                    .map_err(PoetryLayerError::LocateBundledPip)?;
+
+            utils::run_command_and_stream_output(
+                Command::new("python")
+                    .args([
+                        &bundled_pip_module_path.to_string_lossy(),
+                        "install",
+                        // There is no point using pip's cache here, since the layer itself will be cached.
+                        "--no-cache-dir",
+                        "--no-input",
+                        "--no-warn-script-location",
+                        "--quiet",
+                        "--user",
+                        format!("poetry=={POETRY_VERSION}").as_str(),
+                    ])
+                    .env_clear()
+                    .envs(&layer_env.apply(Scope::Build, env)),
+            )
+            .map_err(PoetryLayerError::InstallPoetryCommand)?;
+
+            layer.write_metadata(new_metadata)?;
+        }
+    }
+
+    layer.write_env(&layer_env)?;
+    // Required to pick up the automatic PATH env var. See: https://github.com/heroku/libcnb.rs/issues/842
+    layer_env = layer.read_env()?;
+    env.clone_from(&layer_env.apply(Scope::Build, env));
+
+    Ok(())
+}
+
+// Some of Poetry's dependencies contain compiled components so are platform-specific (unlike pure
+// Python packages). As such we have to take arch and distro into account for cache invalidation.
+#[derive(Deserialize, PartialEq, Serialize)]
+#[serde(deny_unknown_fields)]
+struct PoetryLayerMetadata {
+    arch: String,
+    distro_name: String,
+    distro_version: String,
+    python_version: String,
+    poetry_version: String,
+}
+
+/// Errors that can occur when installing Poetry into a layer.
+#[derive(Debug)]
+pub(crate) enum PoetryLayerError {
+    InstallPoetryCommand(StreamedCommandError),
+    LocateBundledPip(io::Error),
+}
+
+impl From<PoetryLayerError> for libcnb::Error<BuildpackError> {
+    fn from(error: PoetryLayerError) -> Self {
+        Self::BuildpackError(BuildpackError::PoetryLayer(error))
+    }
+}