Merge branch 'main' into django-stable

.github/workflows/security-scan.yaml

@@ -0,0 +1,16 @@
+name: Security scan
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - hotfix/*
+
+jobs:
+  python-scans:
+    name: Scan Python project
+    uses: canonical/starflow/.github/workflows/scan-python.yaml@main
+    with:
+      packages: python-apt-dev
+      osv-extra-args: '--config=source/osv-scanner.toml'
+      trivy-extra-args: '--severity HIGH,CRITICAL --ignore-unfixed --skip-dirs "tests/spread/**"'

.github/workflows/tests.yaml

@@ -81,7 +81,7 @@ jobs:
           sudo apt-get update
           echo "::endgroup::"
           echo "::group::apt-get install..."
-          sudo apt-get install -y python3 python3-dev libapt-pkg-dev libyaml-dev umoci
+          sudo apt-get install -y python3 python3-dev python3-poetry libapt-pkg-dev libyaml-dev umoci
           echo "::endgroup::"
           echo "::group::pip install"
           python -m pip install -U wheel setuptools pip

docs/.wordlist.txt

@@ -10,7 +10,7 @@ cli
 CMake
 cmake
 CMD
-CTRL
+Ctrl
 dataset
 Deadsnakes
 declaratively

docs/conf.py

@@ -85,6 +85,7 @@
     "common/craft-parts/reference/parts_steps.rst",
     "common/craft-parts/reference/step_execution_environment.rst",
     "common/craft-parts/reference/step_output_directories.rst",
+    "common/craft-parts/reference/plugins/poetry_plugin.rst",
     "common/craft-parts/reference/plugins/python_plugin.rst",
     # Extra non-craft-parts exclusions can be added after this comment
 ]

docs/reference/plugins.rst

@@ -21,7 +21,7 @@ Rockcraft.
    /common/craft-parts/reference/plugins/meson_plugin
    /common/craft-parts/reference/plugins/nil_plugin
    /common/craft-parts/reference/plugins/npm_plugin
-   /common/craft-parts/reference/plugins/poetry_plugin
+   plugins/poetry_plugin
    plugins/python_plugin
    /common/craft-parts/reference/plugins/qmake_plugin
    /common/craft-parts/reference/plugins/rust_plugin

docs/reference/plugins/_python_common.rst

@@ -0,0 +1,28 @@
+
+Dependencies
+------------
+
+Since none of the bases that are available for rocks contain a default Python
+installation, including a Python interpreter in Rockcraft projects is mandatory.
+Both the ``python`` and the ``poetry`` plugins also require the ``venv`` module
+to create the virtual environment where Python packages are installed at build
+time.
+
+The easiest way to do this is to include the ``python3-venv`` package in the
+``stage-packages`` of the part that uses the Python-based plugin. This will pull
+in the default Python interpreter for the ``build-base``, like Python 3.10 for
+Ubuntu 22.04. However, other versions can be used by explicitly declaring them -
+here's an example that uses ``python3.12-venv`` from the Deadsnakes ppa:
+
+.. code-block:: yaml
+
+   package-repositories:
+     - type: apt
+       ppa: deadsnakes/ppa
+       priority: always
+
+   parts:
+     my-part:
+       plugin: <python or poetry>
+       source: .
+       stage-packages: [python3.12-venv]

docs/reference/plugins/poetry_plugin.rst

@@ -0,0 +1,8 @@
+
+.. include:: /common/craft-parts/reference/plugins/poetry_plugin.rst
+   :end-before: .. _poetry-details-begin:
+
+.. include:: _python_common.rst
+
+.. include:: /common/craft-parts/reference/plugins/poetry_plugin.rst
+   :start-after: .. _poetry-details-end:

docs/reference/plugins/python_plugin.rst

@@ -2,32 +2,7 @@
 .. include:: /common/craft-parts/reference/plugins/python_plugin.rst
    :end-before: .. _python-details-begin:
 
-Dependencies
-------------
-
-Since none of the bases that are available for rocks contain a default Python
-installation, including a Python interpreter in Rockcraft projects is mandatory.
-The plugin also requires the ``venv`` module to create the virtual environment
-where Python packages are installed at build time.
-
-The easiest way to do this is to include the ``python3-venv`` package in the
-``stage-packages`` of the part that uses the Python plugin. This will pull in
-the default Python interpreter for the ``build-base``, like Python 3.10 for
-Ubuntu 22.04. However, other versions can be used by explicitly declaring them -
-here's an example that uses ``python3.12-venv`` from the Deadsnakes ppa:
-
-.. code-block:: yaml
-
-   package-repositories:
-     - type: apt
-       ppa: deadsnakes/ppa
-       priority: always
-
-   parts:
-     my-part:
-       plugin: python
-       source: .
-       stage-packages: [python3.12-venv]
+.. include:: _python_common.rst
 
 .. include:: /common/craft-parts/reference/plugins/python_plugin.rst
    :start-after: .. _python-details-end:

docs/tutorial/django.rst

@@ -68,7 +68,8 @@ The Django application should respond with
     it difficult to read on a terminal. Visit ``http://localhost:8000`` using a
     browser to see the fully rendered page.
 
-The Django application looks good, so you can stop it for now using ``ctrl+C``.
+The Django application looks good, so let's stop it for now by pressing
+:kbd:`Ctrl` + :kbd:`C`.
 
 Pack the Django application into a rock
 =======================================
@@ -208,7 +209,8 @@ You should expect to see something similar to this:
     2024-08-20T06:34:36.116Z [django] [2024-08-20 06:34:36 +0000] [18] [INFO] Booting worker with pid: 18
 
 You can also choose to follow the logs by using the ``-f`` option with the
-``pebble logs`` command above. To stop following the logs, press ``ctrl+c``.
+``pebble logs`` command above. To stop following the logs, press :kbd:`Ctrl` +
+:kbd:`C`.
 
 Cleanup
 ~~~~~~~

docs/tutorial/flask.rst

@@ -66,7 +66,8 @@ endpoint. You may need a new terminal for this, if you are using Multipass use
 
 The Flask application should respond with ``Hello, world!``.
 
-The Flask application looks good, so stop it for now using ``ctrl+C``.
+The Flask application looks good, so let's stop it for now by pressing
+:kbd:`Ctrl` + :kbd:`C`.
 
 Pack the Flask application into a rock
 ======================================
@@ -205,7 +206,8 @@ You should expect to see something similar to this:
     2024-06-21T03:41:45.078Z [flask] [2024-06-21 03:41:45 +0000] [18] [INFO] Booting worker with pid: 18
 
 You can also choose to follow the logs by using the ``-f`` option with the
-``pebble logs`` command above. To stop following the logs, press ``ctrl+c``.
+``pebble logs`` command above. To stop following the logs, press :kbd:`Ctrl` +
+:kbd:`C`.
 
 .. note::
 

docs/tutorial/node-app.rst

@@ -135,8 +135,8 @@ open another shell into the VM and access the app with curl:
 
 This should also print "Hello World from inside the rock!" to the terminal.
 
-You can now stop the running container by either interrupting it with CTRL+C or
-by running the following in another terminal:
+You can now stop the running container by either interrupting it with
+:kbd:`Ctrl` + :kbd:`C` or by running the following in another terminal:
 
 .. literalinclude:: code/node-app/task.yaml
     :language: bash

osv-scanner.toml

@@ -0,0 +1,4 @@
+[[IgnoredVulns]]
+id = "CVE-2024-35195"
+ignoreUntil = "2025-01-01T00:00:00Z"
+reason = "Needed for requests-unixsocket, which we're replacing with requests-unixsocket2"

requirements-jammy.txt

@@ -1 +1 @@
-https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/python-apt/2.4.0ubuntu2/python-apt_2.4.0ubuntu2.tar.xz ; sys_platform == 'linux'
+python-apt @  https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/python-apt/2.4.0ubuntu2/python-apt_2.4.0ubuntu2.tar.xz ; sys_platform == 'linux'

rockcraft/plugins/poetry_plugin.py

@@ -0,0 +1,47 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright 2024 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""The Rockcraft Poetry plugin."""
+
+
+from craft_parts.plugins import poetry_plugin
+from overrides import override  # type: ignore[reportUnknownVariableType]
+
+from rockcraft.plugins import python_common
+
+
+class PoetryPlugin(poetry_plugin.PoetryPlugin):
+    """A Poetry plugin for Rockcraft."""
+
+    @override
+    def _should_remove_symlinks(self) -> bool:
+        """Overridden because for ubuntu bases we must always remove the symlinks."""
+        return python_common.should_remove_symlinks(self._part_info)
+
+    @override
+    def _get_system_python_interpreter(self) -> str | None:
+        """Overridden because Python must always be provided by the parts."""
+        return None
+
+    @override
+    def _get_script_interpreter(self) -> str:
+        """Overridden because Python is always available in /bin."""
+        return python_common.get_script_interpreter()
+
+    @override
+    def get_build_commands(self) -> list[str]:
+        """Overridden to add a sitecustomize.py."""
+        return python_common.wrap_build_commands(super().get_build_commands())

rockcraft/plugins/python_common.py

@@ -0,0 +1,131 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright 2024 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Common functionality for Python-based plugins.
+
+This functionality extends Craft-parts' vanilla Python plugin to properly
+set the Python interpreter according to the rock's base. Specifically:
+
+- If the base is ubuntu, the venv-created symlinks in bin/ are removed
+altogether. This is because of the usrmerge; when the layer is added on
+top of the base ubuntu layer bin/ becomes a symlink to usr/bin/, so there
+already is a usable Python binary in bin/.
+- Since no base (bare or any ubuntu) provides Python by default, the base
+interpreter must always be provided by one of the parts. The easiest way
+to accomplish this is to add "python3-venv" as a stage-package.
+- The shebang in console scripts is hardcoded to "#!/bin/python3". In fact,
+every use of Python in the resulting image should be via /bin/python3.
+"""
+
+
+from textwrap import dedent
+
+import craft_parts
+
+# Template for the sitecustomize module that we'll add to the payload so that
+# the pip-installed packages are found regardless of how the interpreter is
+# called.
+SITECUSTOMIZE_TEMPLATE = dedent(
+    """
+    # sitecustomize added by Rockcraft.
+    import site
+    import sys
+
+    major, minor = sys.version_info.major, sys.version_info.minor
+    site_dir = f"/lib/python{major}.{minor}/site-packages"
+    dist_dir = "/usr/lib/python3/dist-packages"
+
+    # Add the directory that contains the venv-installed packages.
+    site.addsitedir(site_dir)
+
+    if dist_dir in sys.path:
+        # Make sure that this site-packages dir comes *before* the base-provided
+        # dist-packages dir in sys.path.
+        path = sys.path
+        site_index = path.index(site_dir)
+        dist_index = path.index(dist_dir)
+
+        if dist_index < site_index:
+            path[dist_index], path[site_index] = path[site_index], path[dist_index]
+
+    EOF
+    """
+).strip()
+
+
+def should_remove_symlinks(info: craft_parts.PartInfo) -> bool:
+    """Whether a given Python build should remove the python* venv symlinks.
+
+    :param info: the info for the Python-based part.
+    """
+    return bool(info.base != "bare")
+
+
+def get_script_interpreter() -> str:
+    """Python is always available in /bin."""
+    return "#!/bin/${PARTS_PYTHON_INTERPRETER}"
+
+
+def wrap_build_commands(parts_commands: list[str]) -> list[str]:
+    """Wrap the craft-parts build-commands with Rockraft specific code."""
+    commands: list[str] = []
+
+    # Detect whether PARTS_PYTHON_INTERPRETER is a full path (not supported)
+    commands.append(
+        dedent(
+            """
+            # Detect whether PARTS_PYTHON_INTERPRETER is an absolute path
+            if [[ "${PARTS_PYTHON_INTERPRETER}" = /* ]]; then
+                echo "Absolute paths in \"PARTS_PYTHON_INTERPRETER\" are not allowed: ${PARTS_PYTHON_INTERPRETER}"
+                exit 1
+            fi
+            """
+        )
+    )
+
+    commands.extend(parts_commands)
+
+    # Add a "sitecustomize.py" module to handle the very common case of the
+    # rock's interpreter being called as "python3"; in this case, because of
+    # the default $PATH, "/usr/bin/python3" ends up being called and that is
+    # *not* the venv-aware executable. This sitecustomize adds the location
+    # of the pip-installed packages.
+    commands.append(
+        dedent(
+            """
+            # Add a sitecustomize.py to import our venv-generated location
+            py_version=$(basename $payload_python)
+            py_dir=${CRAFT_PART_INSTALL}/usr/lib/${py_version}/
+            mkdir -p ${py_dir}
+            cat << EOF > ${py_dir}/sitecustomize.py
+            """
+        )
+    )
+    commands.append(SITECUSTOMIZE_TEMPLATE)
+
+    # Remove the pyvenv.cfg file that "marks" the virtual environment, because
+    # it's not necessary in the presence of the sitecustomize module and this
+    # way we get consistent behavior no matter how the interpreter is called.
+    commands.append(
+        dedent(
+            """
+            # Remove pyvenv.cfg file in favor of sitecustomize.py
+            rm ${CRAFT_PART_INSTALL}/pyvenv.cfg
+            """
+        )
+    )
+
+    return commands