From 3e6019f160f52c4cec09a29d697c3c7a207856d6 Mon Sep 17 00:00:00 2001 From: Amos Wenger Date: Wed, 4 Sep 2024 11:25:14 +0200 Subject: [PATCH] Clean up "performance allocators" and "performance flate2" backends Instead of having conditional cargo dependencies in both uv-dev and uv, and different `--features` invocations in CI, this introduces a series of `uv-performance-*` crates that do the right thing. These are enabled by default, but can be disabled when working on correctness alone, locally. --- .github/workflows/build-binaries.yml | 11 ++- Cargo.lock | 33 +++++---- Cargo.toml | 4 ++ crates/uv-dev/Cargo.toml | 13 ++-- crates/uv-dev/src/main.rs | 16 ----- crates/uv-distribution/Cargo.toml | 4 ++ crates/uv-extract/Cargo.toml | 6 +- .../uv-performance-flate2-backend/Cargo.toml | 10 +++ .../uv-performance-flate2-backend/src/lib.rs | 5 ++ .../Cargo.lock | 71 +++++++++++++++++++ .../Cargo.toml | 12 ++++ .../src/lib.rs | 18 +++++ crates/uv/Cargo.toml | 35 +++++---- crates/uv/src/lib.rs | 16 ----- pyproject.toml | 9 ++- 15 files changed, 189 insertions(+), 74 deletions(-) create mode 100644 crates/uv-performance-flate2-backend/Cargo.toml create mode 100644 crates/uv-performance-flate2-backend/src/lib.rs create mode 100644 crates/uv-performance-memory-allocator/Cargo.lock create mode 100644 crates/uv-performance-memory-allocator/Cargo.toml create mode 100644 crates/uv-performance-memory-allocator/src/lib.rs diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index eab2348049ee..74809604419a 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -352,8 +352,7 @@ jobs: *.tar.gz *.sha256 - # Like `linux-arm`, but use `--no-default-features --features flate2/rust_backend` when - # building uv. + # Like `linux-arm`, but use `--no-default-features` when building uv. linux-s390x: if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-build') }} runs-on: ubuntu-latest @@ -378,7 +377,7 @@ jobs: target: ${{ matrix.platform.target }} manylinux: auto docker-options: ${{ matrix.platform.maturin_docker_options }} - args: --release --locked --out dist --no-default-features --features flate2/rust_backend --features self-update + args: --release --locked --out dist --no-default-features --features self-update - uses: uraimo/run-on-arch-action@v2 if: matrix.platform.arch != 'ppc64' name: Test wheel @@ -421,8 +420,8 @@ jobs: *.tar.gz *.sha256 - # Like `linux-arm`, but use `--no-default-features --features flate2/rust_backend` when - # building uv, and install the `gcc-powerpc64-linux-gnu` package. + # Like `linux-arm`, but use `--no-default-features` when building uv, + # and install the `gcc-powerpc64-linux-gnu` package. linux-powerpc: if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-build') }} runs-on: ubuntu-latest @@ -453,7 +452,7 @@ jobs: target: ${{ matrix.platform.target }} manylinux: auto docker-options: ${{ matrix.platform.maturin_docker_options }} - args: --release --locked --out dist --no-default-features --features flate2/rust_backend --features self-update + args: --release --locked --out dist --no-default-features --features self-update before-script-linux: | if command -v yum &> /dev/null; then yum update -y diff --git a/Cargo.lock b/Cargo.lock index 9988e431d90f..75bbcbd0ca40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -326,15 +326,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "backtrace-ext" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" -dependencies = [ - "backtrace", -] - [[package]] name = "base64" version = "0.21.7" @@ -2091,8 +2082,6 @@ version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1" dependencies = [ - "backtrace", - "backtrace-ext", "cfg-if", "miette-derive", "owo-colors", @@ -4491,7 +4480,6 @@ dependencies = [ "itertools 0.13.0", "jiff", "miette", - "mimalloc", "owo-colors", "pep440_rs", "pep508_rs", @@ -4509,7 +4497,6 @@ dependencies = [ "tempfile", "textwrap", "thiserror", - "tikv-jemallocator", "tokio", "toml", "tracing", @@ -4531,6 +4518,8 @@ dependencies = [ "uv-git", "uv-installer", "uv-normalize", + "uv-performance-flate2-backend", + "uv-performance-memory-allocator", "uv-publish", "uv-python", "uv-requirements", @@ -4751,7 +4740,6 @@ dependencies = [ "fs-err", "itertools 0.13.0", "markdown", - "mimalloc", "owo-colors", "pep508_rs", "poloto", @@ -4763,7 +4751,6 @@ dependencies = [ "serde_json", "tagu", "textwrap", - "tikv-jemallocator", "tokio", "tracing", "tracing-durations-export", @@ -4774,6 +4761,7 @@ dependencies = [ "uv-installer", "uv-macros", "uv-options-metadata", + "uv-performance-memory-allocator", "uv-python", "uv-settings", "uv-workspace", @@ -4994,6 +4982,21 @@ dependencies = [ "serde", ] +[[package]] +name = "uv-performance-flate2-backend" +version = "0.1.0" +dependencies = [ + "flate2", +] + +[[package]] +name = "uv-performance-memory-allocator" +version = "0.1.0" +dependencies = [ + "mimalloc", + "tikv-jemallocator", +] + [[package]] name = "uv-pubgrub" version = "0.0.1" diff --git a/Cargo.toml b/Cargo.toml index 8e0c84697730..41ea24236fe5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,10 @@ exclude = [ "scripts", # Needs nightly "crates/uv-trampoline", + # Only used to pull in features, allocators, etc. — we specifically don't want them + # to be part of a workspace-wide cargo check, cargo clippy, etc. + "crates/uv-performance-memory-allocator", + "crates/uv-performance-flate2-backend", ] resolver = "2" diff --git a/crates/uv-dev/Cargo.toml b/crates/uv-dev/Cargo.toml index eb66987bf3e7..cc3f176266cc 100644 --- a/crates/uv-dev/Cargo.toml +++ b/crates/uv-dev/Cargo.toml @@ -52,12 +52,7 @@ tracing = { workspace = true } tracing-durations-export = { workspace = true, features = ["plot"] } tracing-subscriber = { workspace = true } walkdir = { workspace = true } - -[target.'cfg(target_os = "windows")'.dependencies] -mimalloc = { version = "0.1.43" } - -[target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64")))'.dependencies] -tikv-jemallocator = { version = "0.6.0" } +uv-performance-memory-allocator = { path = "../uv-performance-memory-allocator", optional = true } [[bin]] name = "uv-dev" @@ -65,7 +60,11 @@ name = "uv-dev" required-features = ["dev"] [features] -default = [] +default = ["performance"] # Actually build the dev CLI. dev = [] +performance = ["dep:uv-performance-memory-allocator"] render = ["poloto", "resvg", "tagu"] + +[package.metadata.cargo-shear] +ignored = ["flate2", "uv-performance-memory-allocator", "uv-performance-flate2-backend"] diff --git a/crates/uv-dev/src/main.rs b/crates/uv-dev/src/main.rs index 4ee22e757251..1d7c7b186e74 100644 --- a/crates/uv-dev/src/main.rs +++ b/crates/uv-dev/src/main.rs @@ -26,22 +26,6 @@ use crate::generate_options_reference::Args as GenerateOptionsReferenceArgs; use crate::render_benchmarks::RenderBenchmarksArgs; use crate::wheel_metadata::WheelMetadataArgs; -#[cfg(target_os = "windows")] -#[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; - -#[cfg(all( - not(target_os = "windows"), - not(target_os = "openbsd"), - any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "powerpc64" - ) -))] -#[global_allocator] -static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; - mod clear_compile; mod compile; mod generate_all; diff --git a/crates/uv-distribution/Cargo.toml b/crates/uv-distribution/Cargo.toml index d44386b2a220..1845a60951fd 100644 --- a/crates/uv-distribution/Cargo.toml +++ b/crates/uv-distribution/Cargo.toml @@ -55,3 +55,7 @@ zip = { workspace = true } [dev-dependencies] indoc = { version = "2.0.5" } insta = { version = "1.40.0", features = ["filters", "json", "redactions"] } + +[features] +default = [] +performance = ["uv-extract/performance"] diff --git a/crates/uv-extract/Cargo.toml b/crates/uv-extract/Cargo.toml index 0199059a46bc..b1f006c20581 100644 --- a/crates/uv-extract/Cargo.toml +++ b/crates/uv-extract/Cargo.toml @@ -30,8 +30,12 @@ thiserror = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true, features = ["compat"] } tracing = { workspace = true } -xz2 = { workspace = true, features = ["static"] } +xz2 = { workspace = true } zip = { workspace = true } +[features] +default = [] +performance = ["xz2/static"] + [package.metadata.cargo-shear] ignored = ["xz2"] diff --git a/crates/uv-performance-flate2-backend/Cargo.toml b/crates/uv-performance-flate2-backend/Cargo.toml new file mode 100644 index 000000000000..5dba55cc8f9a --- /dev/null +++ b/crates/uv-performance-flate2-backend/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "uv-performance-flate2-backend" +version = "0.1.0" +publish = false + +[target.'cfg(not(any(target_arch = "s390x", target_arch = "powerpc64")))'.dependencies] +flate2 = { version = "1.0.28", default-features = false, features = ["zlib-ng"] } + +[target.'cfg(any(target_arch = "s390x", target_arch = "powerpc64"))'.dependencies] +flate2 = { version = "1.0.28", default-features = false, features = ["rust_backend"] } diff --git a/crates/uv-performance-flate2-backend/src/lib.rs b/crates/uv-performance-flate2-backend/src/lib.rs new file mode 100644 index 000000000000..46340679c545 --- /dev/null +++ b/crates/uv-performance-flate2-backend/src/lib.rs @@ -0,0 +1,5 @@ +//! The sole purpose of this crate is to enable one of +//! `flate2/zlib-ng` (on most platforms) or `flate2/rust_backend` +//! (on s390x, powerpc64, etc. — anywhere libz-ng doesn't build) +//! +//! See `Cargo.toml` diff --git a/crates/uv-performance-memory-allocator/Cargo.lock b/crates/uv-performance-memory-allocator/Cargo.lock new file mode 100644 index 000000000000..dafd5aa60ef3 --- /dev/null +++ b/crates/uv-performance-memory-allocator/Cargo.lock @@ -0,0 +1,71 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cc" +version = "1.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +dependencies = [ + "shlex", +] + +[[package]] +name = "libc" +version = "0.2.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" + +[[package]] +name = "libmimalloc-sys" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23aa6811d3bd4deb8a84dde645f943476d13b248d818edcf8ce0b2f37f036b44" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "mimalloc" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68914350ae34959d83f732418d51e2427a794055d0b9529f48259ac07af65633" +dependencies = [ + "libmimalloc-sys", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3c60906412afa9c2b5b5a48ca6a5abe5736aec9eb48ad05037a677e52e4e2d" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cec5ff18518d81584f477e9bfdf957f5bb0979b0bac3af4ca30b5b3ae2d2865" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + +[[package]] +name = "uv-production-memory-allocator" +version = "0.1.0" +dependencies = [ + "mimalloc", + "tikv-jemallocator", +] diff --git a/crates/uv-performance-memory-allocator/Cargo.toml b/crates/uv-performance-memory-allocator/Cargo.toml new file mode 100644 index 000000000000..534461f30c3b --- /dev/null +++ b/crates/uv-performance-memory-allocator/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "uv-performance-memory-allocator" +version = "0.1.0" +publish = false + +[dependencies] + +[target.'cfg(all(target_os = "windows"))'.dependencies] +mimalloc = { version = "0.1.43" } + +[target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64")))'.dependencies] +tikv-jemallocator = { version = "0.6.0" } diff --git a/crates/uv-performance-memory-allocator/src/lib.rs b/crates/uv-performance-memory-allocator/src/lib.rs new file mode 100644 index 000000000000..50226a6b7419 --- /dev/null +++ b/crates/uv-performance-memory-allocator/src/lib.rs @@ -0,0 +1,18 @@ +//! The only purpose of this crate is to pull in `mimalloc` on windows and +//! `tikv-jemallocator` on most other platforms. + +#[cfg(target_os = "windows")] +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +#[cfg(all( + not(target_os = "windows"), + not(target_os = "openbsd"), + any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "powerpc64" + ) +))] +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; diff --git a/crates/uv/Cargo.toml b/crates/uv/Cargo.toml index c402fb11cbb3..a96f46faa36b 100644 --- a/crates/uv/Cargo.toml +++ b/crates/uv/Cargo.toml @@ -36,7 +36,7 @@ uv-git = { workspace = true } uv-installer = { workspace = true } uv-normalize = { workspace = true } uv-publish = { workspace = true } -uv-python = { workspace = true, features = ["schemars"]} +uv-python = { workspace = true, features = ["schemars"] } uv-requirements = { workspace = true } uv-resolver = { workspace = true } uv-scripts = { workspace = true } @@ -47,10 +47,15 @@ uv-types = { workspace = true } uv-virtualenv = { workspace = true } uv-warnings = { workspace = true } uv-workspace = { workspace = true } +uv-performance-memory-allocator = { path = "../uv-performance-memory-allocator", optional = true } +uv-performance-flate2-backend = { path = "../uv-performance-flate2-backend", optional = true } anstream = { workspace = true } anyhow = { workspace = true } -axoupdater = { workspace = true, features = ["github_releases", "tokio"], optional = true } +axoupdater = { workspace = true, features = [ + "github_releases", + "tokio", +], optional = true } clap = { workspace = true, features = ["derive", "string", "wrap_help"] } ctrlc = { workspace = true } flate2 = { workspace = true, default-features = false } @@ -62,7 +67,7 @@ indicatif = { workspace = true } indoc = { workspace = true } itertools = { workspace = true } jiff = { workspace = true } -miette = { workspace = true, features = ["fancy"] } +miette = { workspace = true, features = ["fancy-no-backtrace"] } owo-colors = { workspace = true } rayon = { workspace = true } regex = { workspace = true } @@ -84,12 +89,6 @@ url = { workspace = true } which = { workspace = true } zip = { workspace = true } -[target.'cfg(target_os = "windows")'.dependencies] -mimalloc = { version = "0.1.43" } - -[target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64")))'.dependencies] -tikv-jemallocator = { version = "0.6.0" } - [dev-dependencies] assert_cmd = { version = "2.0.16" } assert_fs = { version = "1.1.2" } @@ -108,10 +107,23 @@ tempfile = { workspace = true } zip = { workspace = true } [package.metadata.cargo-shear] -ignored = ["flate2"] +ignored = [ + "flate2", + "uv-performance-memory-allocator", + "uv-performance-flate2-backend", +] [features] -default = ["flate2/zlib-ng", "python", "pypi", "git"] +default = ["python", "pypi", "git", "performance"] +# Use better memory allocators, etc. — also turns-on self-update. +performance = [ + "performance-memory-allocator", + "performance-flate2-backend", + "uv-distribution/performance", +] +performance-memory-allocator = ["dep:uv-performance-memory-allocator"] +performance-flate2-backend = ["dep:uv-performance-flate2-backend"] + # Introduces a dependency on a local Python installation. python = [] # Introduces a dependency on a local Python installation with specific patch versions. @@ -122,4 +134,3 @@ pypi = [] git = [] # Adds self-update functionality. self-update = ["axoupdater", "uv-cli/self-update"] - diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index ada7dc88532d..c614905a05c4 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -36,22 +36,6 @@ use crate::settings::{ PublishSettings, }; -#[cfg(target_os = "windows")] -#[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; - -#[cfg(all( - not(target_os = "windows"), - not(target_os = "openbsd"), - any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "powerpc64" - ) -))] -#[global_allocator] -static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; - pub(crate) mod commands; pub(crate) mod logging; pub(crate) mod printer; diff --git a/pyproject.toml b/pyproject.toml index 58bda92c2c42..8cafffd617e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,14 @@ manifest-path = "crates/uv/Cargo.toml" module-name = "uv" python-source = "python" strip = true -include = [{ path = "rust-toolchain.toml", format = ["sdist", "wheel"] }, { path = "LICENSE-APACHE", format = "sdist" }, { path = "LICENSE-MIT", format = "sdist" }] +include = [ + { path = "rust-toolchain.toml", format = ["sdist", "wheel"] }, + # this one isn't discovered by maturin because it's behind a feature flag + { path = "crates/uv-performance-memory-allocator/**/*", format = ["sdist", "wheel"] }, + { path = "crates/uv-performance-flate2-backend/**/*", format = ["sdist", "wheel"] }, + { path = "LICENSE-APACHE", format = "sdist" }, + { path = "LICENSE-MIT", format = "sdist" }, +] [tool.rooster] major_labels = [] # We do not use the major version number yet