ofek · ofek · Jun 20, 2023 · Jun 19, 2023
diff --git a/Cargo.toml b/Cargo.toml
@@ -40,7 +40,9 @@ passthrough = [
   "PYAPP_DISTRIBUTION_EMBED",
   "PYAPP_DISTRIBUTION_FORMAT",
   "PYAPP_DISTRIBUTION_PATH",
+  "PYAPP_DISTRIBUTION_PIP_AVAILABLE",
   "PYAPP_DISTRIBUTION_PYTHON_PATH",
+  "PYAPP_DISTRIBUTION_SITE_PACKAGES_PATH",
   "PYAPP_DISTRIBUTION_SOURCE",
   "PYAPP_DISTRIBUTION_VARIANT",
   "PYAPP_EXEC_CODE",
@@ -64,4 +66,5 @@ passthrough = [
   "PYAPP_PYTHON_VERSION",
   "PYAPP_SELF_COMMAND",
   "PYAPP_SKIP_INSTALL",
+  "PYAPP_UPGRADE_VIRTUALENV",
 ]
diff --git a/build.rs b/build.rs
@@ -11,6 +11,9 @@ use regex::Regex;
 
 const DEFAULT_PYTHON_VERSION: &str = "3.11";
 const KNOWN_DISTRIBUTION_FORMATS: &[&str] = &["tar|bzip2", "tar|gzip", "tar|zstd", "zip"];
+const DEFAULT_CPYTHON_SOURCE: &str =
+    "https://github.com/indygreg/python-build-standalone/releases/download/";
+const DEFAULT_PYPY_SOURCE: &str = "https://downloads.python.org/pypy/";
 
 // Python version in the form MAJOR.MINOR
 // Target OS https://doc.rust-lang.org/reference/conditional-compilation.html#target_os
@@ -154,6 +157,41 @@ const DEFAULT_CPYTHON_DISTRIBUTIONS: &[(&str, &str, &str, &str, &str, &str)] = &
     ("3.7", "macos", "x86_64", "", "", "https://github.com/indygreg/python-build-standalone/releases/download/20200823/cpython-3.7.9-x86_64-apple-darwin-pgo-20200823T2228.tar.zst"),
 ];
 
+// See https://downloads.python.org/pypy/
+#[rustfmt::skip]
+const DEFAULT_PYPY_DISTRIBUTIONS: &[(&str, &str, &str, &str, &str)] = &[
+    ("pypy3.10", "linux", "aarch64", "gnu",
+        "https://downloads.python.org/pypy/pypy3.10-v7.3.12-aarch64.tar.bz2"),
+    ("pypy3.10", "linux", "x86_64", "gnu",
+        "https://downloads.python.org/pypy/pypy3.10-v7.3.12-linux64.tar.bz2"),
+    ("pypy3.10", "windows", "x86_64", "msvc",
+        "https://downloads.python.org/pypy/pypy3.10-v7.3.12-win64.zip"),
+    ("pypy3.10", "macos", "aarch64", "",
+        "https://downloads.python.org/pypy/pypy3.10-v7.3.12-macos_arm64.tar.bz2"),
+    ("pypy3.10", "macos", "x86_64", "",
+        "https://downloads.python.org/pypy/pypy3.10-v7.3.12-macos_x86_64.tar.bz2"),
+    ("pypy3.9", "linux", "aarch64", "gnu",
+        "https://downloads.python.org/pypy/pypy3.9-v7.3.12-aarch64.tar.bz2"),
+    ("pypy3.9", "linux", "x86_64", "gnu",
+        "https://downloads.python.org/pypy/pypy3.9-v7.3.12-linux64.tar.bz2"),
+    ("pypy3.9", "windows", "x86_64", "msvc",
+        "https://downloads.python.org/pypy/pypy3.9-v7.3.12-win64.zip"),
+    ("pypy3.9", "macos", "aarch64", "",
+        "https://downloads.python.org/pypy/pypy3.9-v7.3.12-macos_arm64.tar.bz2"),
+    ("pypy3.9", "macos", "x86_64", "",
+        "https://downloads.python.org/pypy/pypy3.9-v7.3.12-macos_x86_64.tar.bz2"),
+    ("pypy2.7", "linux", "aarch64", "gnu",
+        "https://downloads.python.org/pypy/pypy2.7-v7.3.12-aarch64.tar.bz2"),
+    ("pypy2.7", "linux", "x86_64", "gnu",
+        "https://downloads.python.org/pypy/pypy2.7-v7.3.12-linux64.tar.bz2"),
+    ("pypy2.7", "windows", "x86_64", "msvc",
+        "https://downloads.python.org/pypy/pypy2.7-v7.3.12-win64.zip"),
+    ("pypy2.7", "macos", "aarch64", "",
+        "https://downloads.python.org/pypy/pypy2.7-v7.3.12-macos_arm64.tar.bz2"),
+    ("pypy2.7", "macos", "x86_64", "",
+        "https://downloads.python.org/pypy/pypy2.7-v7.3.12-macos_x86_64.tar.bz2"),
+];
+
 fn set_runtime_variable(name: &str, value: impl Display) {
     println!("cargo:rustc-env={}={}", name, value)
 }
@@ -250,8 +288,7 @@ fn get_distribution_source() -> String {
         abi
     };
 
-    for (python_version, platform, arch, abi, variant, url) in DEFAULT_CPYTHON_DISTRIBUTIONS.iter()
-    {
+    for (python_version, platform, arch, abi, variant, url) in DEFAULT_CPYTHON_DISTRIBUTIONS {
         if python_version == &selected_python_version
             && platform == &selected_platform
             && arch == &selected_arch
@@ -262,6 +299,16 @@ fn get_distribution_source() -> String {
         }
     }
 
+    for (python_version, platform, arch, abi, url) in DEFAULT_PYPY_DISTRIBUTIONS {
+        if python_version == &selected_python_version
+            && platform == &selected_platform
+            && arch == &selected_arch
+            && abi == &selected_abi
+        {
+            return url.to_string();
+        }
+    }
+
     panic!(
         "\n\nNo default distribution source found\nPython version: {}\nPlatform: {}\nArchitecture: {}\nABI: {}\nVariant: {}\n\n",
         selected_python_version, selected_platform, selected_arch, selected_abi, selected_variant
@@ -431,6 +478,18 @@ fn set_distribution() {
 
     set_distribution_format(&distribution_source);
     set_python_path(&distribution_source);
+    set_site_packages_path(&distribution_source);
+    set_distribution_pip_available(&distribution_source);
+
+    let python_isolation_flag = if get_python_version() == "pypy2.7" {
+        // https://docs.python.org/2/using/cmdline.html#cmdoption-e
+        // https://docs.python.org/2/using/cmdline.html#cmdoption-s
+        "-sE"
+    } else {
+        // https://docs.python.org/3/using/cmdline.html#cmdoption-I
+        "-I"
+    };
+    set_runtime_variable("PYAPP__PYTHON_ISOLATION_FLAG", python_isolation_flag);
 }
 
 fn set_distribution_format(distribution_source: &String) {
@@ -462,38 +521,123 @@ fn set_python_path(distribution_source: &str) {
     let on_windows = env::var("CARGO_CFG_TARGET_OS").unwrap() == "windows";
     let python_path = env::var(distribution_variable).unwrap_or_default();
     let relative_path = if !python_path.is_empty() {
-        &python_path
-    } else if distribution_source
-        .starts_with("https://github.com/indygreg/python-build-standalone/releases/download/")
-    {
+        python_path
+    } else if distribution_source.starts_with(DEFAULT_CPYTHON_SOURCE) {
         if get_python_version() == "3.7" {
             if on_windows {
-                r"python\install\python.exe"
+                r"python\install\python.exe".to_string()
             } else {
-                "python/install/bin/python3"
+                "python/install/bin/python3".to_string()
             }
         } else if on_windows {
-            r"python\python.exe"
+            r"python\python.exe".to_string()
+        } else {
+            "python/bin/python3".to_string()
+        }
+    } else if distribution_source.starts_with(DEFAULT_PYPY_SOURCE) {
+        let directory = distribution_source
+            .split('/')
+            .last()
+            .unwrap()
+            .trim_end_matches(".tar.bz2")
+            .trim_end_matches(".zip");
+        if on_windows {
+            format!(r"{}\pypy.exe", directory)
         } else {
-            "python/bin/python3"
+            format!("{}/bin/pypy3", directory)
         }
     } else if on_windows {
-        "python.exe"
+        "python.exe".to_string()
     } else {
-        "bin/python3"
+        "bin/python3".to_string()
     };
-    set_runtime_variable(distribution_variable, relative_path);
+    set_runtime_variable(distribution_variable, &relative_path);
 
     let installation_variable = "PYAPP__INSTALLATION_PYTHON_PATH";
     if is_enabled("PYAPP_FULL_ISOLATION") {
-        set_runtime_variable(installation_variable, relative_path);
+        set_runtime_variable(installation_variable, &relative_path);
     } else if on_windows {
         set_runtime_variable(installation_variable, r"Scripts\python.exe");
     } else {
         set_runtime_variable(installation_variable, "bin/python3");
     };
 }
 
+fn set_site_packages_path(distribution_source: &str) {
+    let distribution_variable = "PYAPP_DISTRIBUTION_SITE_PACKAGES_PATH";
+    let on_windows = env::var("CARGO_CFG_TARGET_OS").unwrap() == "windows";
+    let python_version = get_python_version();
+    let site_packages_path = env::var(distribution_variable).unwrap_or_default();
+    let relative_path = if !site_packages_path.is_empty() {
+        site_packages_path
+    } else if distribution_source.starts_with(DEFAULT_CPYTHON_SOURCE) {
+        if python_version == "3.7" {
+            if on_windows {
+                r"python\install\Lib\site-packages".to_string()
+            } else {
+                format!("python/install/lib/python{}/site-packages", python_version)
+            }
+        } else if on_windows {
+            r"python\Lib\site-packages".to_string()
+        } else {
+            format!("python/lib/python{}/site-packages", python_version)
+        }
+    } else if distribution_source.starts_with(DEFAULT_PYPY_SOURCE) {
+        let directory = distribution_source
+            .split('/')
+            .last()
+            .unwrap()
+            .trim_end_matches(".tar.bz2")
+            .trim_end_matches(".zip");
+        if python_version == "pypy2.7" {
+            if on_windows {
+                format!(r"{}\site-packages", directory)
+            } else {
+                format!("{}/site-packages", directory)
+            }
+        } else if on_windows {
+            format!(r"{}\Lib\site-packages", directory)
+        } else {
+            format!("{}/lib/{}/site-packages", directory, python_version)
+        }
+    } else if on_windows {
+        r"Lib\site-packages".to_string()
+    } else {
+        format!("lib/python{}/site-packages", python_version)
+    };
+    set_runtime_variable(distribution_variable, &relative_path);
+
+    let installation_variable = "PYAPP__INSTALLATION_SITE_PACKAGES_PATH";
+    if is_enabled("PYAPP_FULL_ISOLATION") {
+        set_runtime_variable(installation_variable, &relative_path);
+    } else if get_python_version() == "pypy2.7" {
+        set_runtime_variable(installation_variable, "site-packages");
+    } else if on_windows {
+        set_runtime_variable(installation_variable, r"Lib\site-packages");
+    } else {
+        set_runtime_variable(
+            installation_variable,
+            format!("lib/python{}/site-packages", python_version),
+        );
+    };
+}
+
+fn set_distribution_pip_available(distribution_source: &str) {
+    let variable = "PYAPP_DISTRIBUTION_PIP_AVAILABLE";
+    if is_enabled(variable)
+        // Enable if a default source is used and known to have pip installed already
+        || (!distribution_source.is_empty()
+            && !distribution_source.starts_with(DEFAULT_PYPY_SOURCE)
+            && env::var("PYAPP_DISTRIBUTION_SOURCE")
+                .unwrap_or_default()
+                .is_empty())
+    {
+        set_runtime_variable(variable, "1");
+    } else {
+        set_runtime_variable(variable, "0");
+    }
+}
+
 fn set_execution_mode() {
     let module_variable = "PYAPP_EXEC_MODULE";
     let module = env::var(module_variable).unwrap_or_default();
@@ -544,6 +688,15 @@ fn set_isolation_mode() {
     }
 }
 
+fn set_upgrade_virtualenv() {
+    let variable = "PYAPP_UPGRADE_VIRTUALENV";
+    if is_enabled(variable) || get_python_version() == "pypy2.7" {
+        set_runtime_variable(variable, "1");
+    } else {
+        set_runtime_variable(variable, "0");
+    }
+}
+
 fn set_pip_external() {
     let variable = "PYAPP_PIP_EXTERNAL";
     if is_enabled(variable) {
@@ -655,6 +808,7 @@ fn main() {
     set_distribution();
     set_execution_mode();
     set_isolation_mode();
+    set_upgrade_virtualenv();
     set_pip_external();
     set_pip_version();
     set_pip_extra_args();

diff --git a/docs/changelog.md b/docs/changelog.md
@@ -8,14 +8,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## Unreleased
 
+***Changed:***
+
+- Custom distributions should now define the relative path to the `site-packages` directory
+
 ***Added:***
 
-- Add support for distributions with `bzip2` compression
+- Add support for PyPy distributions
+- Add the `PYAPP_UPGRADE_VIRTUALENV` option to create virtual environments with `virtualenv` rather than the stdlib's `venv`
+- Add support for custom distributions with `bzip2` compression
 
 ***Fixed:***
 
 - Properly handle cases where temporary files are on different filesystems
 - Fix regression in the `metadata` management command on Windows
+- Improve error messages when running binaries that were misconfigured
 
 ## 0.8.0 - 2023-06-09
 

diff --git a/docs/config.md b/docs/config.md
@@ -68,6 +68,16 @@ Some distributions have [variants](https://gregoryszorc.com/docs/python-build-st
 | Linux | <ul><li><code>v1</code></li><li><code>v2</code></li><li><code>v3</code> (default)</li><li><code>v4</code></li></ul> |
 | Windows | <ul><li><code>shared</code> (default)</li><li><code>static</code></li></ul> |
 
+#### PyPy
+
+| ID |
+| --- |
+| `pypy2.7` |
+| `pypy3.9` |
+| `pypy3.10` |
+
+The source of distributions is the [PyPy](https://www.pypy.org) project.
+
 ### Custom
 
 You may explicitly set the `PYAPP_DISTRIBUTION_SOURCE` option which overrides the [known](#known) distribution settings. The source must be a URL that points to an archived version of the desired Python distribution.
@@ -89,6 +99,14 @@ The following formats are supported for the `PYAPP_DISTRIBUTION_FORMAT` option,
 
 You may set the relative path to the Python executable after unpacking the archive with the `PYAPP_DISTRIBUTION_PYTHON_PATH` option. The default is `python.exe` on Windows and `bin/python3` on all other platforms.
 
+#### Site packages location
+
+You may set the relative path to the [`site-packages`](https://docs.python.org/3/library/site.html) directory after unpacking the archive with the `PYAPP_DISTRIBUTION_SITE_PACKAGES_PATH` option. The default is `Lib\site-packages` on Windows and `lib/python<ID>/site-packages` on all other platforms where `<ID>` is the [distribution ID](#known) is defined.
+
+#### pip availability
+
+You may indicate whether pip is already installed by setting the `PYAPP_DISTRIBUTION_PIP_AVAILABLE` option to `true` or `1`. This elides the check for installation when [upgraded virtual environments](#virtual-environments) are enabled.
+
 ### Embedding ### {: #distribution-embedding }
 
 You may set the `PYAPP_DISTRIBUTION_EMBED` option to `true` or `1` to embed the distribution in the executable at build time to avoid fetching it at runtime. When distribution embedding is enabled, you can set the `PYAPP_DISTRIBUTION_PATH` option to use a local path rather than fetching the source.
@@ -104,7 +122,7 @@ You may set the `PYAPP_PIP_EXTERNAL` option to `true` or `1` to use the [standal
 By default, the latest version is used. You may use a specific `X.Y.Z` version by setting the `PYAPP_PIP_VERSION` option.
 
 !!! tip
-    This provides a significant installation speed up when [full isolation](#isolation) is not enabled.
+    This provides a significant installation speed up when [full isolation](#full-isolation) is not enabled.
 
 ### Extra arguments
 
@@ -114,10 +132,14 @@ You may set the `PYAPP_PIP_EXTRA_ARGS` option to provide extra arguments to the
 
 You may set the `PYAPP_PIP_ALLOW_CONFIG` option to `true` or `1` to allow the use of environment variables and other configuration at runtime.
 
-## Isolation
+## Full isolation
 
 You may set the `PYAPP_FULL_ISOLATION` option to `true` or `1` to provide each installation with a full copy of the distribution rather than a virtual environment.
 
+## Virtual environments
+
+When [full isolation](#full-isolation) is not enabled, you may set the `PYAPP_UPGRADE_VIRTUALENV` option to `true` or `1` to create virtual environments with [virtualenv](https://github.com/pypa/virtualenv) rather than the standard library's `venv` module.
+
 ## Skipping project installation
 
 You may set the `PYAPP_SKIP_INSTALL` option to `true` or `1` to skip installing the project in the distribution. This allows for entirely predefined distributions and thus no network calls at runtime if used in conjunction with [distribution embedding](#distribution-embedding).

diff --git a/docs/runtime.md b/docs/runtime.md
@@ -42,7 +42,7 @@ flowchart TD
     MNGCMD -- No --> EXECUTE
     MNGCMD -- Yes --> MANAGE[[Run management command]]
     click DISTEMBEDDED href "../config/#distribution-embedding"
-    click FULLISOLATION href "../config/#isolation"
+    click FULLISOLATION href "../config/#full-isolation"
     click EXTERNALPIP href "../config/#externally-managed"
     click PROJEMBEDDED href "../config/#project-embedding"
     click DEPFILE href "../config/#dependency-file"