Skip to content

Commit

Permalink
Add support for PyPy distributions
Browse files Browse the repository at this point in the history
  • Loading branch information
@ofek
ofek committed Jun 20, 2023
1 parent 11a297e commit eca859c
Show file tree
Hide file tree
Showing 9 changed files with 343 additions and 68 deletions.
3 changes: 3 additions & 0 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@ passthrough = [
"PYAPP_DISTRIBUTION_EMBED",
"PYAPP_DISTRIBUTION_FORMAT",
"PYAPP_DISTRIBUTION_PATH",
"PYAPP_DISTRIBUTION_PIP_AVAILABLE",
"PYAPP_DISTRIBUTION_PYTHON_PATH",
"PYAPP_DISTRIBUTION_SITE_PACKAGES_PATH",
"PYAPP_DISTRIBUTION_SOURCE",
"PYAPP_DISTRIBUTION_VARIANT",
"PYAPP_EXEC_CODE",
Expand All @@ -64,4 +66,5 @@ passthrough = [
"PYAPP_PYTHON_VERSION",
"PYAPP_SELF_COMMAND",
"PYAPP_SKIP_INSTALL",
"PYAPP_UPGRADE_VIRTUALENV",
]
182 changes: 168 additions & 14 deletions build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ use regex::Regex;

const DEFAULT_PYTHON_VERSION: &str = "3.11";
const KNOWN_DISTRIBUTION_FORMATS: &[&str] = &["tar|bzip2", "tar|gzip", "tar|zstd", "zip"];
const DEFAULT_CPYTHON_SOURCE: &str =
"https://github.com/indygreg/python-build-standalone/releases/download/";
const DEFAULT_PYPY_SOURCE: &str = "https://downloads.python.org/pypy/";

// Python version in the form MAJOR.MINOR
// Target OS https://doc.rust-lang.org/reference/conditional-compilation.html#target_os
Expand Down Expand Up @@ -154,6 +157,41 @@ const DEFAULT_CPYTHON_DISTRIBUTIONS: &[(&str, &str, &str, &str, &str, &str)] = &
("3.7", "macos", "x86_64", "", "", "https://github.com/indygreg/python-build-standalone/releases/download/20200823/cpython-3.7.9-x86_64-apple-darwin-pgo-20200823T2228.tar.zst"),
];

// See https://downloads.python.org/pypy/
#[rustfmt::skip]
const DEFAULT_PYPY_DISTRIBUTIONS: &[(&str, &str, &str, &str, &str)] = &[
("pypy3.10", "linux", "aarch64", "gnu",
"https://downloads.python.org/pypy/pypy3.10-v7.3.12-aarch64.tar.bz2"),
("pypy3.10", "linux", "x86_64", "gnu",
"https://downloads.python.org/pypy/pypy3.10-v7.3.12-linux64.tar.bz2"),
("pypy3.10", "windows", "x86_64", "msvc",
"https://downloads.python.org/pypy/pypy3.10-v7.3.12-win64.zip"),
("pypy3.10", "macos", "aarch64", "",
"https://downloads.python.org/pypy/pypy3.10-v7.3.12-macos_arm64.tar.bz2"),
("pypy3.10", "macos", "x86_64", "",
"https://downloads.python.org/pypy/pypy3.10-v7.3.12-macos_x86_64.tar.bz2"),
("pypy3.9", "linux", "aarch64", "gnu",
"https://downloads.python.org/pypy/pypy3.9-v7.3.12-aarch64.tar.bz2"),
("pypy3.9", "linux", "x86_64", "gnu",
"https://downloads.python.org/pypy/pypy3.9-v7.3.12-linux64.tar.bz2"),
("pypy3.9", "windows", "x86_64", "msvc",
"https://downloads.python.org/pypy/pypy3.9-v7.3.12-win64.zip"),
("pypy3.9", "macos", "aarch64", "",
"https://downloads.python.org/pypy/pypy3.9-v7.3.12-macos_arm64.tar.bz2"),
("pypy3.9", "macos", "x86_64", "",
"https://downloads.python.org/pypy/pypy3.9-v7.3.12-macos_x86_64.tar.bz2"),
("pypy2.7", "linux", "aarch64", "gnu",
"https://downloads.python.org/pypy/pypy2.7-v7.3.12-aarch64.tar.bz2"),
("pypy2.7", "linux", "x86_64", "gnu",
"https://downloads.python.org/pypy/pypy2.7-v7.3.12-linux64.tar.bz2"),
("pypy2.7", "windows", "x86_64", "msvc",
"https://downloads.python.org/pypy/pypy2.7-v7.3.12-win64.zip"),
("pypy2.7", "macos", "aarch64", "",
"https://downloads.python.org/pypy/pypy2.7-v7.3.12-macos_arm64.tar.bz2"),
("pypy2.7", "macos", "x86_64", "",
"https://downloads.python.org/pypy/pypy2.7-v7.3.12-macos_x86_64.tar.bz2"),
];

fn set_runtime_variable(name: &str, value: impl Display) {
println!("cargo:rustc-env={}={}", name, value)
}
Expand Down Expand Up @@ -250,8 +288,7 @@ fn get_distribution_source() -> String {
abi
};

for (python_version, platform, arch, abi, variant, url) in DEFAULT_CPYTHON_DISTRIBUTIONS.iter()
{
for (python_version, platform, arch, abi, variant, url) in DEFAULT_CPYTHON_DISTRIBUTIONS {
if python_version == &selected_python_version
&& platform == &selected_platform
&& arch == &selected_arch
Expand All @@ -262,6 +299,16 @@ fn get_distribution_source() -> String {
}
}

for (python_version, platform, arch, abi, url) in DEFAULT_PYPY_DISTRIBUTIONS {
if python_version == &selected_python_version
&& platform == &selected_platform
&& arch == &selected_arch
&& abi == &selected_abi
{
return url.to_string();
}
}

panic!(
"\n\nNo default distribution source found\nPython version: {}\nPlatform: {}\nArchitecture: {}\nABI: {}\nVariant: {}\n\n",
selected_python_version, selected_platform, selected_arch, selected_abi, selected_variant
Expand Down Expand Up @@ -431,6 +478,18 @@ fn set_distribution() {

set_distribution_format(&distribution_source);
set_python_path(&distribution_source);
set_site_packages_path(&distribution_source);
set_distribution_pip_available(&distribution_source);

let python_isolation_flag = if get_python_version() == "pypy2.7" {
// https://docs.python.org/2/using/cmdline.html#cmdoption-e
// https://docs.python.org/2/using/cmdline.html#cmdoption-s
"-sE"
} else {
// https://docs.python.org/3/using/cmdline.html#cmdoption-I
"-I"
};
set_runtime_variable("PYAPP__PYTHON_ISOLATION_FLAG", python_isolation_flag);
}

fn set_distribution_format(distribution_source: &String) {
Expand Down Expand Up @@ -462,38 +521,123 @@ fn set_python_path(distribution_source: &str) {
let on_windows = env::var("CARGO_CFG_TARGET_OS").unwrap() == "windows";
let python_path = env::var(distribution_variable).unwrap_or_default();
let relative_path = if !python_path.is_empty() {
&python_path
} else if distribution_source
.starts_with("https://github.com/indygreg/python-build-standalone/releases/download/")
{
python_path
} else if distribution_source.starts_with(DEFAULT_CPYTHON_SOURCE) {
if get_python_version() == "3.7" {
if on_windows {
r"python\install\python.exe"
r"python\install\python.exe".to_string()
} else {
"python/install/bin/python3"
"python/install/bin/python3".to_string()
}
} else if on_windows {
r"python\python.exe"
r"python\python.exe".to_string()
} else {
"python/bin/python3".to_string()
}
} else if distribution_source.starts_with(DEFAULT_PYPY_SOURCE) {
let directory = distribution_source
.split('/')
.last()
.unwrap()
.trim_end_matches(".tar.bz2")
.trim_end_matches(".zip");
if on_windows {
format!(r"{}\pypy.exe", directory)
} else {
"python/bin/python3"
format!("{}/bin/pypy3", directory)
}
} else if on_windows {
"python.exe"
"python.exe".to_string()
} else {
"bin/python3"
"bin/python3".to_string()
};
set_runtime_variable(distribution_variable, relative_path);
set_runtime_variable(distribution_variable, &relative_path);

let installation_variable = "PYAPP__INSTALLATION_PYTHON_PATH";
if is_enabled("PYAPP_FULL_ISOLATION") {
set_runtime_variable(installation_variable, relative_path);
set_runtime_variable(installation_variable, &relative_path);
} else if on_windows {
set_runtime_variable(installation_variable, r"Scripts\python.exe");
} else {
set_runtime_variable(installation_variable, "bin/python3");
};
}

fn set_site_packages_path(distribution_source: &str) {
let distribution_variable = "PYAPP_DISTRIBUTION_SITE_PACKAGES_PATH";
let on_windows = env::var("CARGO_CFG_TARGET_OS").unwrap() == "windows";
let python_version = get_python_version();
let site_packages_path = env::var(distribution_variable).unwrap_or_default();
let relative_path = if !site_packages_path.is_empty() {
site_packages_path
} else if distribution_source.starts_with(DEFAULT_CPYTHON_SOURCE) {
if python_version == "3.7" {
if on_windows {
r"python\install\Lib\site-packages".to_string()
} else {
format!("python/install/lib/python{}/site-packages", python_version)
}
} else if on_windows {
r"python\Lib\site-packages".to_string()
} else {
format!("python/lib/python{}/site-packages", python_version)
}
} else if distribution_source.starts_with(DEFAULT_PYPY_SOURCE) {
let directory = distribution_source
.split('/')
.last()
.unwrap()
.trim_end_matches(".tar.bz2")
.trim_end_matches(".zip");
if python_version == "pypy2.7" {
if on_windows {
format!(r"{}\site-packages", directory)
} else {
format!("{}/site-packages", directory)
}
} else if on_windows {
format!(r"{}\Lib\site-packages", directory)
} else {
format!("{}/lib/{}/site-packages", directory, python_version)
}
} else if on_windows {
r"Lib\site-packages".to_string()
} else {
format!("lib/python{}/site-packages", python_version)
};
set_runtime_variable(distribution_variable, &relative_path);

let installation_variable = "PYAPP__INSTALLATION_SITE_PACKAGES_PATH";
if is_enabled("PYAPP_FULL_ISOLATION") {
set_runtime_variable(installation_variable, &relative_path);
} else if get_python_version() == "pypy2.7" {
set_runtime_variable(installation_variable, "site-packages");
} else if on_windows {
set_runtime_variable(installation_variable, r"Lib\site-packages");
} else {
set_runtime_variable(
installation_variable,
format!("lib/python{}/site-packages", python_version),
);
};
}

fn set_distribution_pip_available(distribution_source: &str) {
let variable = "PYAPP_DISTRIBUTION_PIP_AVAILABLE";
if is_enabled(variable)
// Enable if a default source is used and known to have pip installed already
|| (!distribution_source.is_empty()
&& !distribution_source.starts_with(DEFAULT_PYPY_SOURCE)
&& env::var("PYAPP_DISTRIBUTION_SOURCE")
.unwrap_or_default()
.is_empty())
{
set_runtime_variable(variable, "1");
} else {
set_runtime_variable(variable, "0");
}
}

fn set_execution_mode() {
let module_variable = "PYAPP_EXEC_MODULE";
let module = env::var(module_variable).unwrap_or_default();
Expand Down Expand Up @@ -544,6 +688,15 @@ fn set_isolation_mode() {
}
}

fn set_upgrade_virtualenv() {
let variable = "PYAPP_UPGRADE_VIRTUALENV";
if is_enabled(variable) || get_python_version() == "pypy2.7" {
set_runtime_variable(variable, "1");
} else {
set_runtime_variable(variable, "0");
}
}

fn set_pip_external() {
let variable = "PYAPP_PIP_EXTERNAL";
if is_enabled(variable) {
Expand Down Expand Up @@ -655,6 +808,7 @@ fn main() {
set_distribution();
set_execution_mode();
set_isolation_mode();
set_upgrade_virtualenv();
set_pip_external();
set_pip_version();
set_pip_extra_args();
Expand Down
9 changes: 8 additions & 1 deletion docs/changelog.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),

## Unreleased

***Changed:***

- Custom distributions should now define the relative path to the `site-packages` directory

***Added:***

- Add support for distributions with `bzip2` compression
- Add support for PyPy distributions
- Add the `PYAPP_UPGRADE_VIRTUALENV` option to create virtual environments with `virtualenv` rather than the stdlib's `venv`
- Add support for custom distributions with `bzip2` compression

***Fixed:***

- Properly handle cases where temporary files are on different filesystems
- Fix regression in the `metadata` management command on Windows
- Improve error messages when running binaries that were misconfigured

## 0.8.0 - 2023-06-09

Expand Down
26 changes: 24 additions & 2 deletions docs/config.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,16 @@ Some distributions have [variants](https://gregoryszorc.com/docs/python-build-st
| Linux | <ul><li><code>v1</code></li><li><code>v2</code></li><li><code>v3</code> (default)</li><li><code>v4</code></li></ul> |
| Windows | <ul><li><code>shared</code> (default)</li><li><code>static</code></li></ul> |

#### PyPy

| ID |
| --- |
| `pypy2.7` |
| `pypy3.9` |
| `pypy3.10` |

The source of distributions is the [PyPy](https://www.pypy.org) project.

### Custom

You may explicitly set the `PYAPP_DISTRIBUTION_SOURCE` option which overrides the [known](#known) distribution settings. The source must be a URL that points to an archived version of the desired Python distribution.
Expand All @@ -89,6 +99,14 @@ The following formats are supported for the `PYAPP_DISTRIBUTION_FORMAT` option,

You may set the relative path to the Python executable after unpacking the archive with the `PYAPP_DISTRIBUTION_PYTHON_PATH` option. The default is `python.exe` on Windows and `bin/python3` on all other platforms.

#### Site packages location

You may set the relative path to the [`site-packages`](https://docs.python.org/3/library/site.html) directory after unpacking the archive with the `PYAPP_DISTRIBUTION_SITE_PACKAGES_PATH` option. The default is `Lib\site-packages` on Windows and `lib/python<ID>/site-packages` on all other platforms where `<ID>` is the [distribution ID](#known) is defined.

#### pip availability

You may indicate whether pip is already installed by setting the `PYAPP_DISTRIBUTION_PIP_AVAILABLE` option to `true` or `1`. This elides the check for installation when [upgraded virtual environments](#virtual-environments) are enabled.

### Embedding ### {: #distribution-embedding }

You may set the `PYAPP_DISTRIBUTION_EMBED` option to `true` or `1` to embed the distribution in the executable at build time to avoid fetching it at runtime. When distribution embedding is enabled, you can set the `PYAPP_DISTRIBUTION_PATH` option to use a local path rather than fetching the source.
Expand All @@ -104,7 +122,7 @@ You may set the `PYAPP_PIP_EXTERNAL` option to `true` or `1` to use the [standal
By default, the latest version is used. You may use a specific `X.Y.Z` version by setting the `PYAPP_PIP_VERSION` option.

!!! tip
This provides a significant installation speed up when [full isolation](#isolation) is not enabled.
This provides a significant installation speed up when [full isolation](#full-isolation) is not enabled.

### Extra arguments

Expand All @@ -114,10 +132,14 @@ You may set the `PYAPP_PIP_EXTRA_ARGS` option to provide extra arguments to the

You may set the `PYAPP_PIP_ALLOW_CONFIG` option to `true` or `1` to allow the use of environment variables and other configuration at runtime.

## Isolation
## Full isolation

You may set the `PYAPP_FULL_ISOLATION` option to `true` or `1` to provide each installation with a full copy of the distribution rather than a virtual environment.

## Virtual environments

When [full isolation](#full-isolation) is not enabled, you may set the `PYAPP_UPGRADE_VIRTUALENV` option to `true` or `1` to create virtual environments with [virtualenv](https://github.com/pypa/virtualenv) rather than the standard library's `venv` module.

## Skipping project installation

You may set the `PYAPP_SKIP_INSTALL` option to `true` or `1` to skip installing the project in the distribution. This allows for entirely predefined distributions and thus no network calls at runtime if used in conjunction with [distribution embedding](#distribution-embedding).
Expand Down
2 changes: 1 addition & 1 deletion docs/runtime.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ flowchart TD
MNGCMD -- No --> EXECUTE
MNGCMD -- Yes --> MANAGE[[Run management command]]
click DISTEMBEDDED href "../config/#distribution-embedding"
click FULLISOLATION href "../config/#isolation"
click FULLISOLATION href "../config/#full-isolation"
click EXTERNALPIP href "../config/#externally-managed"
click PROJEMBEDDED href "../config/#project-embedding"
click DEPFILE href "../config/#dependency-file"
Expand Down
Loading

0 comments on commit eca859c

Please sign in to comment.