From cc637c1b9aa702932136de84ef7818c6d115aaab Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sat, 16 Nov 2019 20:52:47 +0700 Subject: [PATCH] Use reusable venvs Replace pip install --target with venvs. As setuptools breakes out of the venv to load distutils build_ext command, first create a copy of the Python distribution and hack it. This also has the benefit that the venv can be used to build extensions outside of PyOxidizer. Also add venv_path to PipRequirementsFile, allowing the same venv to be incrementally populated in multiple rules, and the venv re-used across PyOxidizer build runs. Fixes https://github.com/indygreg/PyOxidizer/issues/162 Fixes https://github.com/indygreg/PyOxidizer/issues/170 Closes https://github.com/indygreg/PyOxidizer/issues/194 --- pyoxidizer/Cargo.toml | 2 + pyoxidizer/src/app_packaging/config.rs | 3 + .../src/app_packaging/packaging_rule.rs | 623 ++++++------------ pyoxidizer/src/distutils/_msvccompiler.py | 4 + pyoxidizer/src/distutils/unixccompiler.py | 4 + pyoxidizer/src/py_packaging/distribution.rs | 213 +++++- pyoxidizer/src/py_packaging/distutils.rs | 78 +-- pyoxidizer/src/starlark/python_packaging.rs | 17 +- 8 files changed, 454 insertions(+), 490 deletions(-) diff --git a/pyoxidizer/Cargo.toml b/pyoxidizer/Cargo.toml index 2ffcd761f..f01ba4177 100644 --- a/pyoxidizer/Cargo.toml +++ b/pyoxidizer/Cargo.toml @@ -27,7 +27,9 @@ cc = "1.0" clap = "2.32" codemap = "0.1" codemap-diagnostic = "0.1" +copy_dir = "0.1.2" encoding_rs = "0.8" +filetime = "0.2" git2 = "0.9" glob = "0.3" goblin = "0.0" diff --git a/pyoxidizer/src/app_packaging/config.rs b/pyoxidizer/src/app_packaging/config.rs index 80e0ce2cd..a18089914 100644 --- a/pyoxidizer/src/app_packaging/config.rs +++ b/pyoxidizer/src/app_packaging/config.rs @@ -26,6 +26,7 @@ pub enum InstallLocation { #[derive(Clone, Debug, PartialEq)] pub struct PackagingSetupPyInstall { pub path: String, + pub venv_path: Option, pub extra_env: HashMap, pub extra_global_arguments: Vec, pub optimize_level: i64, @@ -87,6 +88,7 @@ pub struct PackagingPackageRoot { #[derive(Clone, Debug, PartialEq)] pub struct PackagingPipInstallSimple { pub package: String, + pub venv_path: Option, pub extra_env: HashMap, pub optimize_level: i64, pub excludes: Vec, @@ -99,6 +101,7 @@ pub struct PackagingPipInstallSimple { pub struct PackagingPipRequirementsFile { // TODO resolve to a PathBuf. pub requirements_path: String, + pub venv_path: Option, pub extra_env: HashMap, pub optimize_level: i64, pub include_source: bool, diff --git a/pyoxidizer/src/app_packaging/packaging_rule.rs b/pyoxidizer/src/app_packaging/packaging_rule.rs index 4cb03fab9..2f691ab43 100644 --- a/pyoxidizer/src/app_packaging/packaging_rule.rs +++ b/pyoxidizer/src/app_packaging/packaging_rule.rs @@ -15,8 +15,10 @@ use super::config::{ PackagingStdlibExtensionsPolicy, PackagingVirtualenv, PythonPackaging, }; use super::state::BuildContext; -use crate::py_packaging::distribution::{is_stdlib_test_package, ParsedPythonDistribution}; -use crate::py_packaging::distutils::{prepare_hacked_distutils, read_built_extensions}; +use crate::py_packaging::distribution::{ + is_stdlib_test_package, resolve_python_paths, ParsedPythonDistribution, +}; +use crate::py_packaging::distutils::read_built_extensions; use crate::py_packaging::fsscan::{ find_python_resources, is_package_from_path, PythonFileResource, }; @@ -109,30 +111,6 @@ fn resource_full_name(resource: &PythonFileResource) -> &str { } } -struct PythonPaths { - main: PathBuf, - site_packages: PathBuf, -} - -/// Resolve the location of Python modules given a base install path. -fn resolve_python_paths(base: &Path, python_version: &str, is_windows: bool) -> PythonPaths { - let mut p = base.to_path_buf(); - - if is_windows { - p.push("Lib"); - } else { - p.push("lib"); - p.push(format!("python{}", &python_version[0..3])); - } - - let site_packages = p.join("site-packages"); - - PythonPaths { - main: p, - site_packages, - } -} - fn resolve_built_extensions( state_dir: &Path, res: &mut Vec, @@ -149,6 +127,122 @@ fn resolve_built_extensions( Ok(()) } +/// Processes resources in a path +/// Args includes and excludes are ignored if None or an empty Vec. +fn process_resources( + logger: &slog::Logger, + path: &PathBuf, + location: &ResourceLocation, + state_dir: Option<&PathBuf>, + include_source: bool, + optimize_level: i64, + includes: Option<&Vec>, + excludes: Option<&Vec>, +) -> Vec { + let mut res = Vec::new(); + + let path_s = path.display().to_string(); + warn!(logger, "processing resources from {}", path_s); + + for resource in find_python_resources(path) { + let full_name = resource_full_name(&resource); + + let excluded = match includes { + Some(values) => values.iter().any(|v| { + let prefix = v.clone() + "."; + full_name != v && !full_name.starts_with(&prefix) + }), + None => false, + }; + + if excluded { + info!( + logger, + "whitelist skipping {}", full_name + ); + continue; + } + + let excluded = match excludes { + Some(values) => match values.is_empty() { + true => false, + false => values.iter().all(|v| { + let prefix = v.clone() + "."; + full_name == v || full_name.starts_with(&prefix) + }), + }, + None => false, + }; + + if excluded { + info!( + logger, + "blacklist skipping {}", full_name + ); + continue; + } + + match resource { + PythonFileResource::Source { + full_name, path, .. + } => { + let is_package = is_package_from_path(&path); + let source = fs::read(path).expect("error reading source file"); + + if include_source { + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::ModuleSource { + name: full_name.clone(), + source: source.clone(), + is_package, + }, + }); + } + + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::ModuleBytecode { + name: full_name.clone(), + source, + optimize_level: optimize_level as i32, + is_package, + }, + }); + } + + PythonFileResource::Resource(resource) => { + let data = fs::read(resource.path).expect("error reading resource file"); + + res.push(PythonResourceAction { + action: ResourceAction::Add, + location: location.clone(), + resource: PythonResource::Resource { + package: resource.package.clone(), + name: resource.stem.clone(), + data, + }, + }); + } + + _ => {} + } + } + + match state_dir { + Some(dir) => { + if dir.exists() { + resolve_built_extensions(&dir, &mut res, &location).unwrap(); + } + } + None => {} + }; + + res +} + fn resolve_stdlib_extensions_policy( logger: &slog::Logger, dist: &ParsedPythonDistribution, @@ -330,165 +424,44 @@ fn resolve_stdlib( } fn resolve_virtualenv( + logger: &slog::Logger, dist: &ParsedPythonDistribution, rule: &PackagingVirtualenv, ) -> Vec { - let mut res = Vec::new(); - let location = ResourceLocation::new(&rule.install_location); let python_paths = resolve_python_paths(&Path::new(&rule.path), &dist.version, dist.os == "windows"); - let packages_path = python_paths.site_packages; - - for resource in find_python_resources(&packages_path) { - let mut relevant = true; - let full_name = resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecode { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - res + process_resources( + &logger, + &python_paths.site_packages, + &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), + ) } -fn resolve_package_root(rule: &PackagingPackageRoot) -> Vec { - let mut res = Vec::new(); - +fn resolve_package_root( + logger: &slog::Logger, + rule: &PackagingPackageRoot, +) -> Vec { let location = ResourceLocation::new(&rule.install_location); let path = PathBuf::from(&rule.path); - for resource in find_python_resources(&path) { - let mut relevant = false; - let full_name = resource_full_name(&resource); - - for package in &rule.packages { - let prefix = package.clone() + "."; - - if full_name == package || full_name.starts_with(&prefix) { - relevant = true; - } - } - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecode { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - res + process_resources( + &logger, + &path, + &location, + None, + rule.include_source, + rule.optimize_level, + Some(&rule.packages), + None, + ) } fn resolve_pip_install_simple( @@ -497,20 +470,9 @@ fn resolve_pip_install_simple( rule: &PackagingPipInstallSimple, verbose: bool, ) -> Vec { - let mut res = Vec::new(); - let location = ResourceLocation::new(&rule.install_location); - dist.ensure_pip(); - let temp_dir = - tempdir::TempDir::new("pyoxidizer-pip-install").expect("could not creat temp directory"); - - let mut extra_envs = prepare_hacked_distutils(logger, dist, temp_dir.path(), &[]) - .expect("unable to hack distutils"); - - let target_dir_path = temp_dir.path().join("install"); - let target_dir_s = target_dir_path.display().to_string(); - warn!(logger, "pip installing to {}", target_dir_s); + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); let mut pip_args: Vec = vec![ "-m".to_string(), @@ -524,8 +486,6 @@ fn resolve_pip_install_simple( pip_args.extend(vec![ "install".to_string(), - "--target".to_string(), - target_dir_s, "--no-binary".to_string(), ":all:".to_string(), rule.package.clone(), @@ -540,7 +500,7 @@ fn resolve_pip_install_simple( } // TODO send stderr to stdout. - let mut cmd = std::process::Command::new(&dist.python_exe) + let mut cmd = std::process::Command::new(&python_paths.python_exe) .args(&pip_args) .envs(&extra_envs) .stdout(std::process::Stdio::piped()) @@ -560,79 +520,16 @@ fn resolve_pip_install_simple( panic!("error running pip"); } - for resource in find_python_resources(&target_dir_path) { - let mut relevant = true; - let full_name = resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecode { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - resolve_built_extensions( - &PathBuf::from(extra_envs.get("PYOXIDIZER_DISTUTILS_STATE_DIR").unwrap()), - &mut res, + process_resources( + &logger, + &python_paths.site_packages, &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), ) - .unwrap(); - - res } fn resolve_pip_requirements_file( @@ -641,21 +538,9 @@ fn resolve_pip_requirements_file( rule: &PackagingPipRequirementsFile, verbose: bool, ) -> Vec { - let mut res = Vec::new(); - let location = ResourceLocation::new(&rule.install_location); - dist.ensure_pip(); - - let temp_dir = - tempdir::TempDir::new("pyoxidizer-pip-install").expect("could not create temp directory"); - - let mut extra_envs = prepare_hacked_distutils(logger, dist, temp_dir.path(), &[]) - .expect("unable to hack distutils"); - - let target_dir_path = temp_dir.path().join("install"); - let target_dir_s = target_dir_path.display().to_string(); - warn!(logger, "pip installing to {}", target_dir_s); + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); let mut args: Vec = vec![ "-m".to_string(), @@ -669,8 +554,6 @@ fn resolve_pip_requirements_file( args.extend(vec![ "install".to_string(), - "--target".to_string(), - target_dir_s, "--no-binary".to_string(), ":all:".to_string(), "--requirement".to_string(), @@ -685,8 +568,15 @@ fn resolve_pip_requirements_file( extra_envs.insert(key.clone(), value.clone()); } + warn!( + logger, + "Running {} {}", + python_paths.python_exe.display(), + args.join(" ") + ); + // TODO send stderr to stdout. - let mut cmd = std::process::Command::new(&dist.python_exe) + let mut cmd = std::process::Command::new(&python_paths.python_exe) .args(&args) .envs(&extra_envs) .stdout(std::process::Stdio::piped()) @@ -706,64 +596,16 @@ fn resolve_pip_requirements_file( panic!("error running pip"); } - for resource in find_python_resources(&target_dir_path) { - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source file"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecode { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - resolve_built_extensions( - &PathBuf::from(extra_envs.get("PYOXIDIZER_DISTUTILS_STATE_DIR").unwrap()), - &mut res, + process_resources( + &logger, + &python_paths.site_packages, &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + None, ) - .unwrap(); - - res } fn resolve_setup_py_install( @@ -773,8 +615,6 @@ fn resolve_setup_py_install( rule: &PackagingSetupPyInstall, verbose: bool, ) -> Vec { - let mut res = Vec::new(); - // Execution directory is resolved relative to the active configuration // file unless it is absolute. let rule_path = PathBuf::from(&rule.path); @@ -787,31 +627,12 @@ fn resolve_setup_py_install( let location = ResourceLocation::new(&rule.install_location); - let temp_dir = tempdir::TempDir::new("pyoxidizer-setup-py-install") - .expect("could not create temp directory"); - - let target_dir_path = temp_dir.path().join("install"); - let target_dir_s = target_dir_path.display().to_string(); - - let python_paths = resolve_python_paths(&target_dir_path, &dist.version, dist.os == "windows"); - - std::fs::create_dir_all(&python_paths.site_packages) - .expect("unable to create site-packages directory"); - - let mut extra_envs = prepare_hacked_distutils( - logger, - dist, - temp_dir.path(), - &[&python_paths.site_packages, &python_paths.main], - ) - .expect("unable to hack distutils"); + let (python_paths, mut extra_envs) = dist.prepare_venv(&logger, rule.venv_path.as_ref()); for (key, value) in rule.extra_env.iter() { extra_envs.insert(key.clone(), value.clone()); } - warn!(logger, "python setup.py installing to {}", target_dir_s); - let mut args = vec!["setup.py"]; for arg in &rule.extra_global_arguments { @@ -822,10 +643,19 @@ fn resolve_setup_py_install( args.push("--verbose"); } - args.extend(&["install", "--prefix", &target_dir_s, "--no-compile"]); + let prefix_dir_s = python_paths.prefix.display().to_string(); + + args.extend(&["install", "--prefix", &prefix_dir_s, "--no-compile"]); + + warn!( + logger, + "Running {} {}", + python_paths.python_exe.display(), + args.join(" ") + ); // TODO send stderr to stdout. - let mut cmd = std::process::Command::new(&dist.python_exe) + let mut cmd = std::process::Command::new(&python_paths.python_exe) .current_dir(cwd) .args(&args) .envs(&extra_envs) @@ -846,81 +676,16 @@ fn resolve_setup_py_install( panic!("error running setup.py"); } - let packages_path = python_paths.site_packages; - - for resource in find_python_resources(&packages_path) { - let mut relevant = true; - let full_name = resource_full_name(&resource); - - for exclude in &rule.excludes { - let prefix = exclude.clone() + "."; - - if full_name == exclude || full_name.starts_with(&prefix) { - relevant = false; - } - } - - if !relevant { - continue; - } - - match resource { - PythonFileResource::Source { - full_name, path, .. - } => { - let is_package = is_package_from_path(&path); - let source = fs::read(path).expect("error reading source"); - - if rule.include_source { - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleSource { - name: full_name.clone(), - source: source.clone(), - is_package, - }, - }); - } - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::ModuleBytecode { - name: full_name.clone(), - source, - optimize_level: rule.optimize_level as i32, - is_package, - }, - }); - } - - PythonFileResource::Resource(resource) => { - let data = fs::read(resource.path).expect("error reading resource file"); - - res.push(PythonResourceAction { - action: ResourceAction::Add, - location: location.clone(), - resource: PythonResource::Resource { - package: resource.package.clone(), - name: resource.stem.clone(), - data, - }, - }); - } - - _ => {} - } - } - - resolve_built_extensions( - &PathBuf::from(extra_envs.get("PYOXIDIZER_DISTUTILS_STATE_DIR").unwrap()), - &mut res, + process_resources( + &logger, + &python_paths.site_packages, &location, + Some(&python_paths.pyoxidizer_state_dir), + rule.include_source, + rule.optimize_level, + None, + Some(&rule.excludes), ) - .unwrap(); - - res } /// Resolves a Python packaging rule to resources to package. @@ -950,9 +715,9 @@ pub fn resolve_python_packaging( PythonPackaging::Stdlib(rule) => resolve_stdlib(logger, dist, &rule), - PythonPackaging::Virtualenv(rule) => resolve_virtualenv(dist, &rule), + PythonPackaging::Virtualenv(rule) => resolve_virtualenv(logger, dist, &rule), - PythonPackaging::PackageRoot(rule) => resolve_package_root(&rule), + PythonPackaging::PackageRoot(rule) => resolve_package_root(logger, &rule), PythonPackaging::PipInstallSimple(rule) => { resolve_pip_install_simple(logger, dist, &rule, verbose) diff --git a/pyoxidizer/src/distutils/_msvccompiler.py b/pyoxidizer/src/distutils/_msvccompiler.py index 88e482aad..7b9bfa61a 100644 --- a/pyoxidizer/src/distutils/_msvccompiler.py +++ b/pyoxidizer/src/distutils/_msvccompiler.py @@ -582,6 +582,9 @@ def extension_link_shared_object(self, # binary. dest_path = os.environ['PYOXIDIZER_DISTUTILS_STATE_DIR'] + if not os.path.exists(dest_path): + os.makedirs(dest_path) + # We need to copy the object files because they may be in a temp # directory that doesn't outlive this process. object_paths = [] @@ -604,6 +607,7 @@ def extension_link_shared_object(self, } json.dump(data, fh, indent=4, sort_keys=True) + print('Wrote {}'.format(json_path), file=sys.stderr) # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in diff --git a/pyoxidizer/src/distutils/unixccompiler.py b/pyoxidizer/src/distutils/unixccompiler.py index 26f575fe0..0aee5f85b 100644 --- a/pyoxidizer/src/distutils/unixccompiler.py +++ b/pyoxidizer/src/distutils/unixccompiler.py @@ -238,6 +238,9 @@ def extension_link_shared_object(self, # binary. dest_path = os.environ['PYOXIDIZER_DISTUTILS_STATE_DIR'] + if not os.path.exists(dest_path): + os.makedirs(dest_path) + # We need to copy the object files because they may be in a temp # directory that doesn't outlive this process. object_paths = [] @@ -260,6 +263,7 @@ def extension_link_shared_object(self, } json.dump(data, fh, indent=4, sort_keys=True) + print('Wrote {}'.format(json_path), file=sys.stderr) # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in diff --git a/pyoxidizer/src/py_packaging/distribution.rs b/pyoxidizer/src/py_packaging/distribution.rs index 8fdceec15..6ab74779d 100644 --- a/pyoxidizer/src/py_packaging/distribution.rs +++ b/pyoxidizer/src/py_packaging/distribution.rs @@ -2,23 +2,33 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use copy_dir::copy_dir; use itertools::Itertools; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use slog::warn; -use std::collections::BTreeMap; +use slog::{info, warn}; +use std::collections::{BTreeMap, HashMap}; +use std::env; use std::fs; use std::fs::{create_dir_all, File}; -use std::io::{Cursor, Read}; +use std::io::{BufRead, BufReader, Cursor, Read}; use std::path::{Path, PathBuf}; use url::Url; +use super::distutils::prepare_hacked_distutils; use super::fsscan::{ find_python_resources, is_package_from_path, walk_tree_files, PythonFileResource, }; use super::resource::{PythonResource, ResourceData, SourceModule}; + use crate::licensing::NON_GPL_LICENSES; +#[cfg(windows)] +const PYTHON_EXE_BASENAME: &str = "python3.exe"; + +#[cfg(unix)] +const PYTHON_EXE_BASENAME: &str = "python3"; + #[cfg(windows)] const PIP_EXE_BASENAME: &str = "pip3.exe"; @@ -309,6 +319,9 @@ pub struct ParsedPythonDistribution { /// Describes license info for things in this distribution. pub license_infos: BTreeMap>, + + /// Path to copy of hacked dist to use for packaging rules venvs + pub venv_base: PathBuf, } #[derive(Debug)] @@ -331,6 +344,48 @@ pub enum ExtensionModuleFilter { NoGPL, } +pub struct PythonPaths { + pub prefix: PathBuf, + pub bin_dir: PathBuf, + pub python_exe: PathBuf, + pub stdlib: PathBuf, + pub site_packages: PathBuf, + pub pyoxidizer_state_dir: PathBuf, +} + +/// Resolve the location of Python modules given a base install path. +pub fn resolve_python_paths(base: &Path, python_version: &str, is_windows: bool) -> PythonPaths { + let prefix = base.to_path_buf(); + + let mut p = prefix.clone(); + + let bin_dir = p.join("bin"); + + let python_exe = bin_dir.join(PYTHON_EXE_BASENAME); + + let pyoxidizer_state_dir = p.join("state/pyoxidizer"); + + if is_windows { + p.push("Lib"); + } else { + p.push("lib"); + p.push(format!("python{}", &python_version[0..3])); + } + + let stdlib = p.clone(); + + let site_packages = p.join("site-packages"); + + PythonPaths { + prefix, + bin_dir, + python_exe, + stdlib, + site_packages, + pyoxidizer_state_dir, + } +} + impl ParsedPythonDistribution { pub fn from_path( logger: &slog::Logger, @@ -360,7 +415,7 @@ impl ParsedPythonDistribution { } /// Ensure pip is available to run in the distribution. - pub fn ensure_pip(&self) -> PathBuf { + pub fn ensure_pip(&self, logger: &slog::Logger) -> PathBuf { let pip_path = self .python_exe .parent() @@ -369,6 +424,7 @@ impl ParsedPythonDistribution { .join(PIP_EXE_BASENAME); if !pip_path.exists() { + info!(logger, "running {} -m ensurepip", self.python_exe.display()); std::process::Command::new(&self.python_exe) .args(&["-m", "ensurepip"]) .status() @@ -378,6 +434,152 @@ impl ParsedPythonDistribution { pip_path } + /// Duplicate the python distribution, with distutils hacked + pub fn create_hacked_base(&self, logger: &slog::Logger) -> PythonPaths { + let dist_prefix = self.base_dir.join("python").join("install"); + let venv_base = self.venv_base.clone(); + + let dist_prefix_s = dist_prefix.display().to_string(); + let venv_dir_s = self.venv_base.display().to_string(); + + self.ensure_pip(logger); + + if !venv_base.exists() { + warn!( + logger, + "copying {} to create hacked base {}", dist_prefix_s, venv_dir_s + ); + copy_dir(&dist_prefix, &venv_base).unwrap(); + + // Provide a reliable mtime + File::create(&venv_base.join(".timestamp")).unwrap(); //.sync_all(); + } + + let python_paths = resolve_python_paths(&venv_base, &self.version, self.os == "windows"); + + prepare_hacked_distutils(logger, &python_paths); + + python_paths + } + + /// Create a venv from the distribution at path. + pub fn create_venv(&self, logger: &slog::Logger, path: &PathBuf) -> PythonPaths { + let venv_dir_s = path.display().to_string(); + + let venv_python_paths = resolve_python_paths(&path, &self.version, self.os == "windows"); + + // This will recreate it, if it was deleted + let python_paths = self.create_hacked_base(&logger); + + if path.exists() { + warn!(logger, "re-using venv {}", venv_dir_s); + return venv_python_paths; + } + + warn!(logger, "creating venv {}", venv_dir_s); + + // The venv needs to use --copies otherwise setuptools build_ext + // breaks out of the venv and uses the dist build_ext which is not hacked. + let args: Vec = vec![ + "-m".to_string(), + "venv".to_string(), + //"--copies".to_string(), + venv_dir_s.clone(), + ]; + + info!( + logger, + "Running {} {}", + python_paths.python_exe.display(), + args.join(" ") + ); + + let mut cmd = std::process::Command::new(&python_paths.python_exe) + .args(&args) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("error running venv"); + { + let stdout = cmd.stdout.as_mut().unwrap(); + let reader = BufReader::new(stdout); + + for line in reader.lines() { + warn!(logger, "{}", line.unwrap()); + } + } + + venv_python_paths + } + + /// Create or re-use an existing venv + pub fn prepare_venv( + &self, + logger: &slog::Logger, + venv_path: Option<&String>, + ) -> (PythonPaths, HashMap) { + let venv_dir_path = match venv_path { + Some(path_str) => PathBuf::from(path_str), + None => tempdir::TempDir::new("pyoxidizer-temp-venv") + .expect("could not create temp directory") + .path() + .join("venv"), + }; + + let python_paths = self.create_venv(logger, &venv_dir_path); + + let mut extra_envs = HashMap::new(); + + let prefix_s = python_paths + .prefix + .canonicalize() + .unwrap() + .display() + .to_string(); + + let venv_path_bin_s = python_paths + .bin_dir + .canonicalize() + .unwrap() + .display() + .to_string(); + + let path_separator = if cfg!(windows) { ";" } else { ":" }; + + let process_path_s = env::var("PATH").unwrap(); + + extra_envs.insert( + "PATH".to_string(), + format!("{}{}{}", venv_path_bin_s, path_separator, process_path_s), + ); + + extra_envs.insert("VIRTUAL_ENV".to_string(), prefix_s.clone()); + extra_envs.insert( + "PYTHONPATH".to_string(), + python_paths + .site_packages + .canonicalize() + .unwrap() + .display() + .to_string(), + ); + + fs::create_dir_all(&python_paths.pyoxidizer_state_dir).unwrap(); + + extra_envs.insert( + "PYOXIDIZER_DISTUTILS_STATE_DIR".to_string(), + python_paths + .pyoxidizer_state_dir + .canonicalize() + .unwrap() + .display() + .to_string(), + ); + + extra_envs.insert("PYOXIDIZER".to_string(), "1".to_string()); + + (python_paths, extra_envs) + } + /// Obtain resolved `SourceModule` instances for this distribution. /// /// This effectively resolves the raw file content for .py files into @@ -694,6 +896,8 @@ pub fn analyze_python_distribution_data( }; } + let venv_base = dist_dir.parent().unwrap().join("hacked_base"); + Ok(ParsedPythonDistribution { flavor: pi.python_flavor.clone(), version: pi.python_version.clone(), @@ -720,6 +924,7 @@ pub fn analyze_python_distribution_data( py_modules, resources, license_infos, + venv_base, }) } diff --git a/pyoxidizer/src/py_packaging/distutils.rs b/pyoxidizer/src/py_packaging/distutils.rs index 11a932f38..a339fbfc4 100644 --- a/pyoxidizer/src/py_packaging/distutils.rs +++ b/pyoxidizer/src/py_packaging/distutils.rs @@ -5,11 +5,14 @@ use lazy_static::lazy_static; use serde::Deserialize; use slog::warn; -use std::collections::{BTreeMap, HashMap}; -use std::fs::{create_dir_all, read_dir, read_to_string}; +use std::collections::BTreeMap; +use std::fs::{read_dir, read_to_string}; use std::path::{Path, PathBuf}; -use super::distribution::ParsedPythonDistribution; +use filetime::FileTime; +use std::fs; + +use super::distribution::PythonPaths; use super::resource::BuiltExtensionModule; lazy_static! { @@ -46,70 +49,35 @@ lazy_static! { /// modified distutils will survive multiple process invocations, unlike a /// monkeypatch. People do weird things in setup.py scripts and we want to /// support as many as possible. -pub fn prepare_hacked_distutils( - logger: &slog::Logger, - dist: &ParsedPythonDistribution, - dest_dir: &Path, - extra_python_paths: &[&Path], -) -> Result, String> { - let extra_sys_path = dest_dir.join("packages"); - +pub fn prepare_hacked_distutils(logger: &slog::Logger, target: &PythonPaths) { warn!( logger, "installing modified distutils to {}", - extra_sys_path.display() + target.stdlib.display() ); - let orig_distutils_path = dist.stdlib_path.join("distutils"); - let dest_distutils_path = extra_sys_path.join("distutils"); - - for entry in walkdir::WalkDir::new(&orig_distutils_path) { - match entry { - Ok(entry) => { - if entry.path().is_dir() { - continue; - } - - let source_path = entry.path(); - let rel_path = source_path - .strip_prefix(&orig_distutils_path) - .or_else(|_| Err("unable to strip prefix"))?; - let dest_path = dest_distutils_path.join(rel_path); - - let dest_dir = dest_path.parent().unwrap(); - std::fs::create_dir_all(&dest_dir).or_else(|e| Err(e.to_string()))?; - std::fs::copy(&source_path, &dest_path).or_else(|e| Err(e.to_string()))?; - } - Err(e) => return Err(e.to_string()), - } - } + let dest_distutils_path = target.stdlib.join("distutils"); - for (path, data) in MODIFIED_DISTUTILS_FILES.iter() { - let dest_path = dest_distutils_path.join(path); - - warn!(logger, "modifying distutils/{} for oxidation", path); - std::fs::write(dest_path, data).or_else(|e| Err(e.to_string()))?; - } + // The venv "pyvenv.cfg" is used as a proxy for the first hack + let src_metadata = fs::metadata(&target.prefix.join(".timestamp")).unwrap(); - let state_dir = dest_dir.join("pyoxidizer-build-state"); - create_dir_all(&state_dir).or_else(|e| Err(e.to_string()))?; + let src_mtime = FileTime::from_last_modification_time(&src_metadata); - let mut python_paths = vec![extra_sys_path.display().to_string()]; - python_paths.extend(extra_python_paths.iter().map(|p| p.display().to_string())); + for (path, data) in MODIFIED_DISTUTILS_FILES.iter() { + let dest_path = dest_distutils_path.join(path); - let path_separator = if cfg!(windows) { ";" } else { ":" }; + let dest_metadata = fs::metadata(dest_path.clone()).unwrap(); - let python_path = python_paths.join(path_separator); + let dest_mtime = FileTime::from_last_modification_time(&dest_metadata); - let mut res = HashMap::new(); - res.insert("PYTHONPATH".to_string(), python_path); - res.insert( - "PYOXIDIZER_DISTUTILS_STATE_DIR".to_string(), - state_dir.display().to_string(), - ); - res.insert("PYOXIDIZER".to_string(), "1".to_string()); + if src_mtime < dest_mtime { + warn!(logger, "not overwriting newer distutils/{}", path); + continue; + } - Ok(res) + warn!(logger, "modifying distutils/{} for oxidation", path); + std::fs::write(dest_path, data).unwrap(); + } } #[derive(Debug, Deserialize)] diff --git a/pyoxidizer/src/starlark/python_packaging.rs b/pyoxidizer/src/starlark/python_packaging.rs index cb739a573..be8550d78 100644 --- a/pyoxidizer/src/starlark/python_packaging.rs +++ b/pyoxidizer/src/starlark/python_packaging.rs @@ -16,8 +16,8 @@ use std::cmp::Ordering; use std::collections::HashMap; use super::env::{ - optional_dict_arg, optional_list_arg, required_bool_arg, required_list_arg, required_str_arg, - required_type_arg, + optional_dict_arg, optional_list_arg, optional_str_arg, required_bool_arg, required_list_arg, + required_str_arg, required_type_arg, }; use crate::app_packaging::config::{ resolve_install_location, PackagingFilterInclude, PackagingPackageRoot, @@ -519,6 +519,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case)] PipInstallSimple( package, + venv_path=None, extra_env=None, optimize_level=0, excludes=None, @@ -527,6 +528,7 @@ starlark_module! { python_packaging_env => extra_args=None ) { let package = required_str_arg("package", &package)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; required_type_arg("optimize_level", "int", &optimize_level)?; optional_list_arg("excludes", "string", &excludes)?; @@ -565,6 +567,7 @@ starlark_module! { python_packaging_env => let rule = PackagingPipInstallSimple { package, + venv_path, extra_env, optimize_level, excludes, @@ -579,6 +582,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case)] PipRequirementsFile( requirements_path, + venv_path=None, extra_env=None, optimize_level=0, include_source=true, @@ -586,6 +590,7 @@ starlark_module! { python_packaging_env => extra_args=None ) { let requirements_path = required_str_arg("path", &requirements_path)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; required_type_arg("optimize_level", "int", &optimize_level)?; let include_source = required_bool_arg("include_source", &include_source)?; @@ -618,6 +623,7 @@ starlark_module! { python_packaging_env => }; let rule = PackagingPipRequirementsFile { + venv_path, requirements_path, extra_env, optimize_level, @@ -632,6 +638,7 @@ starlark_module! { python_packaging_env => #[allow(non_snake_case)] SetupPyInstall( package_path, + venv_path=None, extra_env=None, extra_global_arguments=None, optimize_level=0, @@ -640,6 +647,7 @@ starlark_module! { python_packaging_env => excludes=None ) { let package_path = required_str_arg("package_path", &package_path)?; + let venv_path = optional_str_arg("venv_path", &venv_path)?; optional_dict_arg("extra_env", "string", "string", &extra_env)?; optional_list_arg("extra_global_arguments", "string", &extra_global_arguments)?; required_type_arg("optimize_level", "int", &optimize_level)?; @@ -676,6 +684,7 @@ starlark_module! { python_packaging_env => let rule = PackagingSetupPyInstall { path: package_path, + venv_path, extra_env, extra_global_arguments, optimize_level: optimize_level.to_int().unwrap(), @@ -890,6 +899,7 @@ mod tests { let v = starlark_ok("PipInstallSimple('foo')"); let wanted = PackagingPipInstallSimple { package: "foo".to_string(), + venv_path: None, extra_env: HashMap::new(), optimize_level: 0, excludes: Vec::new(), @@ -914,6 +924,7 @@ mod tests { let v = starlark_ok("PipRequirementsFile('path')"); let wanted = PackagingPipRequirementsFile { requirements_path: "path".to_string(), + venv_path: None, extra_env: HashMap::new(), optimize_level: 0, include_source: true, @@ -928,6 +939,7 @@ mod tests { fn test_pip_requirements_file_extra_args() { let v = starlark_ok("PipRequirementsFile('path', extra_args=['foo'])"); let wanted = PackagingPipRequirementsFile { + venv_path: None, requirements_path: "path".to_string(), extra_env: HashMap::new(), optimize_level: 0, @@ -950,6 +962,7 @@ mod tests { let v = starlark_ok("SetupPyInstall('foo')"); let wanted = PackagingSetupPyInstall { path: "foo".to_string(), + venv_path: None, extra_env: HashMap::new(), extra_global_arguments: Vec::new(), optimize_level: 0,