From 8070fe10f1ccc346539f87c88fb65ea55d001b9a Mon Sep 17 00:00:00 2001 From: Nulano Date: Sat, 23 Dec 2023 14:41:50 +0100 Subject: [PATCH 01/42] pass build config before setuptools command; add build_editable to custom build backend --- .github/workflows/wheels.yml | 2 ++ _custom_build/backend.py | 57 +++++++++++++----------------------- pyproject.toml | 2 +- setup.py | 34 ++++++++++++++++++--- 4 files changed, 54 insertions(+), 41 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 5adff7ec1c4..0c8a941de30 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -5,6 +5,7 @@ on: paths: - ".ci/requirements-cibw.txt" - ".github/workflows/wheel*" + - "setup.py" - "wheels/*" - "winbuild/build_prepare.py" - "winbuild/fribidi.cmake" @@ -14,6 +15,7 @@ on: paths: - ".ci/requirements-cibw.txt" - ".github/workflows/wheel*" + - "setup.py" - "wheels/*" - "winbuild/build_prepare.py" - "winbuild/fribidi.cmake" diff --git a/_custom_build/backend.py b/_custom_build/backend.py index d1537b80987..0b183a5870b 100644 --- a/_custom_build/backend.py +++ b/_custom_build/backend.py @@ -11,41 +11,16 @@ class _CustomBuildMetaBackend(backend_class): def run_setup(self, setup_script="setup.py"): if self.config_settings: - - def config_has(key, value): - settings = self.config_settings.get(key) - if settings: - if not isinstance(settings, list): - settings = [settings] - return value in settings - - flags = [] - for dependency in ( - "zlib", - "jpeg", - "tiff", - "freetype", - "raqm", - "lcms", - "webp", - "webpmux", - "jpeg2000", - "imagequant", - "xcb", - ): - if config_has(dependency, "enable"): - flags.append("--enable-" + dependency) - elif config_has(dependency, "disable"): - flags.append("--disable-" + dependency) - for dependency in ("raqm", "fribidi"): - if config_has(dependency, "vendor"): - flags.append("--vendor-" + dependency) - if self.config_settings.get("platform-guessing") == "disable": - flags.append("--disable-platform-guessing") - if self.config_settings.get("debug") == "true": - flags.append("--debug") - if flags: - sys.argv = sys.argv[:1] + ["build_ext"] + flags + sys.argv[1:] + params = [] + for k, v in self.config_settings.items(): + if isinstance(v, list): + msg = "Conflicting options: " + ", ".join( + f"'--config-setting {k}={v_}'" for v_ in v + ) + raise ValueError(msg) + params.append(f"--pillow-configuration={k}={v}") + + sys.argv = sys.argv[:1] + params + sys.argv[1:] return super().run_setup(setup_script) def build_wheel( @@ -54,5 +29,15 @@ def build_wheel( self.config_settings = config_settings return super().build_wheel(wheel_directory, config_settings, metadata_directory) + def build_editable( + self, wheel_directory, config_settings=None, metadata_directory=None + ): + self.config_settings = config_settings + return super().build_editable( + wheel_directory, config_settings, metadata_directory + ) + -build_wheel = _CustomBuildMetaBackend().build_wheel +_backend = _CustomBuildMetaBackend() +build_wheel = _backend.build_wheel +build_editable = _backend.build_editable diff --git a/pyproject.toml b/pyproject.toml index da2537b2137..d63e401af1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ version = {attr = "PIL.__version__"} [tool.cibuildwheel] before-all = ".github/workflows/wheels-dependencies.sh" build-verbosity = 1 -config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable" +config-settings = "raqm=vendor fribidi=vendor imagequant=disable" test-command = "cd {project} && .github/workflows/wheels-test.sh" test-extras = "tests" diff --git a/setup.py b/setup.py index 1bf0bcff558..c74165fb73a 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,9 @@ def get_version(): return locals()["__version__"] +configuration = {} + + PILLOW_VERSION = get_version() FREETYPE_ROOT = None HARFBUZZ_ROOT = None @@ -334,15 +337,24 @@ def __iter__(self): + [("add-imaging-libs=", None, "Add libs to _imaging build")] ) + @staticmethod + def check_configuration(option, value): + return True if configuration.get(option) == value else None + def initialize_options(self): - self.disable_platform_guessing = None + self.disable_platform_guessing = self.check_configuration( + "platform-guessing", "disable" + ) self.add_imaging_libs = "" build_ext.initialize_options(self) for x in self.feature: - setattr(self, f"disable_{x}", None) - setattr(self, f"enable_{x}", None) + setattr(self, f"disable_{x}", self.check_configuration(x, "disable")) + setattr(self, f"enable_{x}", self.check_configuration(x, "enable")) for x in ("raqm", "fribidi"): - setattr(self, f"vendor_{x}", None) + setattr(self, f"vendor_{x}", self.check_configuration(x, "vendor")) + if self.check_configuration("debug", "true"): + self.debug = True + self.parallel = configuration.get("parallel") def finalize_options(self): build_ext.finalize_options(self) @@ -390,6 +402,9 @@ def finalize_options(self): raise ValueError(msg) _dbg("Using vendored version of %s", x) self.feature.vendor.add(x) + if x == "raqm": + _dbg("--vendor-raqm implies --enable-raqm") + self.feature.required.add(x) def _update_extension(self, name, libraries, define_macros=None, sources=None): for extension in self.extensions: @@ -985,6 +1000,17 @@ def debug_build(): Extension("PIL._imagingmorph", ["src/_imagingmorph.c"]), ] + +# parse configuration from _custom_build/backend.py +while len(sys.argv[1]) >= 2 and sys.argv[1].startswith("--pillow-configuration="): + _, key, value = sys.argv[1].split("=", 2) + old = configuration.get(key) + if old is not None: + msg = f"Conflicting options: '-C {key}={old}' and '-C {key}={value}'" + raise ValueError(msg) + configuration[key] = value + del sys.argv[1] + try: setup( cmdclass={"build_ext": pil_build_ext}, From b4e690049d81dc2569d6f02c5b0c1b9eb29a07b2 Mon Sep 17 00:00:00 2001 From: Nulano Date: Sun, 31 Dec 2023 02:22:05 +0100 Subject: [PATCH 02/42] document config setting "-C parallel=n" for number of CPUs to use for compilation --- docs/installation.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index fbcfbb90724..03011619ff7 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -390,9 +390,10 @@ After navigating to the Pillow directory, run:: Build Options """"""""""""" -* Environment variable: ``MAX_CONCURRENCY=n``. Pillow can use - multiprocessing to build the extension. Setting ``MAX_CONCURRENCY`` - sets the number of CPUs to use, or can disable parallel building by +* Config setting: ``-C parallel=n``. Can also be given + with environment variable: ``MAX_CONCURRENCY=n``. Pillow can use + multiprocessing to build the extension. Setting ``-C parallel=n`` + sets the number of CPUs to use to ``n``, or can disable parallel building by using a setting of 1. By default, it uses 4 CPUs, or if 4 are not available, as many as are present. @@ -417,14 +418,13 @@ Build Options used to compile the standard Pillow wheels. Compiling libraqm requires a C99-compliant compiler. -* Build flag: ``-C platform-guessing=disable``. Skips all of the +* Config setting: ``-C platform-guessing=disable``. Skips all of the platform dependent guessing of include and library directories for automated build systems that configure the proper paths in the environment variables (e.g. Buildroot). -* Build flag: ``-C debug=true``. Adds a debugging flag to the include and - library search process to dump all paths searched for and found to - stdout. +* Config setting: ``-C debug=true``. Adds a debugging flag to the include and + library search process to dump all paths searched for and found to stdout. Sample usage:: From f27b838a451d0b20befea03da7b2d58dd1da7821 Mon Sep 17 00:00:00 2001 From: Nulano Date: Tue, 2 Jan 2024 15:47:47 +0100 Subject: [PATCH 03/42] support multiple --config-settings --- _custom_build/backend.py | 12 +++++------- pyproject.toml | 2 +- setup.py | 13 +++---------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/_custom_build/backend.py b/_custom_build/backend.py index 0b183a5870b..2c670ff0ac3 100644 --- a/_custom_build/backend.py +++ b/_custom_build/backend.py @@ -12,13 +12,11 @@ class _CustomBuildMetaBackend(backend_class): def run_setup(self, setup_script="setup.py"): if self.config_settings: params = [] - for k, v in self.config_settings.items(): - if isinstance(v, list): - msg = "Conflicting options: " + ", ".join( - f"'--config-setting {k}={v_}'" for v_ in v - ) - raise ValueError(msg) - params.append(f"--pillow-configuration={k}={v}") + for key, values in self.config_settings.items(): + if not isinstance(values, list): + values = [values] + for value in values: + params.append(f"--pillow-configuration={key}={value}") sys.argv = sys.argv[:1] + params + sys.argv[1:] return super().run_setup(setup_script) diff --git a/pyproject.toml b/pyproject.toml index d63e401af1e..da2537b2137 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ version = {attr = "PIL.__version__"} [tool.cibuildwheel] before-all = ".github/workflows/wheels-dependencies.sh" build-verbosity = 1 -config-settings = "raqm=vendor fribidi=vendor imagequant=disable" +config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable" test-command = "cd {project} && .github/workflows/wheels-test.sh" test-extras = "tests" diff --git a/setup.py b/setup.py index c74165fb73a..686e8889fd9 100755 --- a/setup.py +++ b/setup.py @@ -339,7 +339,7 @@ def __iter__(self): @staticmethod def check_configuration(option, value): - return True if configuration.get(option) == value else None + return True if value in configuration.get(option, []) else None def initialize_options(self): self.disable_platform_guessing = self.check_configuration( @@ -354,7 +354,7 @@ def initialize_options(self): setattr(self, f"vendor_{x}", self.check_configuration(x, "vendor")) if self.check_configuration("debug", "true"): self.debug = True - self.parallel = configuration.get("parallel") + self.parallel = configuration.get("parallel", [None])[-1] def finalize_options(self): build_ext.finalize_options(self) @@ -402,9 +402,6 @@ def finalize_options(self): raise ValueError(msg) _dbg("Using vendored version of %s", x) self.feature.vendor.add(x) - if x == "raqm": - _dbg("--vendor-raqm implies --enable-raqm") - self.feature.required.add(x) def _update_extension(self, name, libraries, define_macros=None, sources=None): for extension in self.extensions: @@ -1004,11 +1001,7 @@ def debug_build(): # parse configuration from _custom_build/backend.py while len(sys.argv[1]) >= 2 and sys.argv[1].startswith("--pillow-configuration="): _, key, value = sys.argv[1].split("=", 2) - old = configuration.get(key) - if old is not None: - msg = f"Conflicting options: '-C {key}={old}' and '-C {key}={value}'" - raise ValueError(msg) - configuration[key] = value + configuration.setdefault(key, []).append(value) del sys.argv[1] try: From 01e5f06da055490f70b1294f39a2718297ee8067 Mon Sep 17 00:00:00 2001 From: Nulano Date: Tue, 2 Jan 2024 16:12:37 +0100 Subject: [PATCH 04/42] document editable mode installation in winbuild/build.rst --- winbuild/build.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/winbuild/build.rst b/winbuild/build.rst index a8e4ebaa6cc..26d0da0a35a 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -87,11 +87,18 @@ are set by running ``winbuild\build\build_env.cmd`` and install Pillow with pip: winbuild\build\build_env.cmd python.exe -m pip install -v -C raqm=vendor -C fribidi=vendor . +You can also install Pillow in `editable mode`_:: + + winbuild\build\build_env.cmd + python.exe -m pip install -v -C raqm=vendor -C fribidi=vendor -e . + To build a wheel instead, run:: winbuild\build\build_env.cmd python.exe -m pip wheel -v -C raqm=vendor -C fribidi=vendor . +.. _editable mode: https://setuptools.pypa.io/en/latest/userguide/development_mode.html + Testing Pillow -------------- From 74af933a9f0441e1285c398675dc3eca1de5406c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ondrej=20Baranovi=C4=8D?= Date: Sat, 20 Jan 2024 10:08:14 +0100 Subject: [PATCH 05/42] Link to stable setuptools documentation Co-authored-by: Andrew Murray <3112309+radarhere@users.noreply.github.com> --- winbuild/build.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winbuild/build.rst b/winbuild/build.rst index 26d0da0a35a..f40982cd546 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -97,7 +97,7 @@ To build a wheel instead, run:: winbuild\build\build_env.cmd python.exe -m pip wheel -v -C raqm=vendor -C fribidi=vendor . -.. _editable mode: https://setuptools.pypa.io/en/latest/userguide/development_mode.html +.. _editable mode: https://setuptools.pypa.io/en/stable/userguide/development_mode.html Testing Pillow -------------- From 420150f0e251c9519a5aea1c24546a44d489828c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ondrej=20Baranovi=C4=8D?= Date: Sat, 20 Jan 2024 14:56:20 +0100 Subject: [PATCH 06/42] Update winbuild/build.rst --- winbuild/build.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winbuild/build.rst b/winbuild/build.rst index f40982cd546..c980d9c7567 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -92,7 +92,7 @@ You can also install Pillow in `editable mode`_:: winbuild\build\build_env.cmd python.exe -m pip install -v -C raqm=vendor -C fribidi=vendor -e . -To build a wheel instead, run:: +To build a binary wheel instead, run:: winbuild\build\build_env.cmd python.exe -m pip wheel -v -C raqm=vendor -C fribidi=vendor . From 5b20811cabd4594fd4fafc596e3f94afe12bb361 Mon Sep 17 00:00:00 2001 From: Nulano Date: Tue, 20 Feb 2024 20:36:36 +0100 Subject: [PATCH 07/42] Add `--bugreport` argument to __main__.py to omit supported formats --- src/PIL/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/PIL/__main__.py b/src/PIL/__main__.py index 943789923dc..32de33624db 100644 --- a/src/PIL/__main__.py +++ b/src/PIL/__main__.py @@ -1,5 +1,7 @@ from __future__ import annotations +import sys + from .features import pilinfo -pilinfo() +pilinfo(supported_formats="--bugreport" not in sys.argv) From ab9dfd8181868922abee46dffb11ffcb9a772958 Mon Sep 17 00:00:00 2001 From: Nulano Date: Tue, 20 Feb 2024 20:37:33 +0100 Subject: [PATCH 08/42] Add sys.{executable,base_prefix,prefix} to features.pilinfo --- Tests/test_features.py | 12 +++++++++--- Tests/test_main.py | 12 +++++++++--- src/PIL/features.py | 9 +++++++-- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/Tests/test_features.py b/Tests/test_features.py index 8d2d198ffcd..3fffa032f81 100644 --- a/Tests/test_features.py +++ b/Tests/test_features.py @@ -129,9 +129,15 @@ def test_pilinfo() -> None: while lines[0].startswith(" "): lines = lines[1:] assert lines[0] == "-" * 68 - assert lines[1].startswith("Python modules loaded from ") - assert lines[2].startswith("Binary modules loaded from ") - assert lines[3] == "-" * 68 + assert lines[1].startswith("Python executable is") + lines = lines[2:] + if lines[0].startswith("Environment Python files loaded from"): + lines = lines[1:] + assert lines[0].startswith("System Python files loaded from") + assert lines[1] == "-" * 68 + assert lines[2].startswith("Python Pillow modules loaded from ") + assert lines[3].startswith("Binary Pillow modules loaded from ") + assert lines[4] == "-" * 68 jpeg = ( "\n" + "-" * 68 diff --git a/Tests/test_main.py b/Tests/test_main.py index 46259f1dc52..e13e0c5e3ac 100644 --- a/Tests/test_main.py +++ b/Tests/test_main.py @@ -15,9 +15,15 @@ def test_main() -> None: while lines[0].startswith(" "): lines = lines[1:] assert lines[0] == "-" * 68 - assert lines[1].startswith("Python modules loaded from ") - assert lines[2].startswith("Binary modules loaded from ") - assert lines[3] == "-" * 68 + assert lines[1].startswith("Python executable is") + lines = lines[2:] + if lines[0].startswith("Environment Python files loaded from"): + lines = lines[1:] + assert lines[0].startswith("System Python files loaded from") + assert lines[1] == "-" * 68 + assert lines[2].startswith("Python Pillow modules loaded from ") + assert lines[3].startswith("Binary Pillow modules loaded from ") + assert lines[4] == "-" * 68 jpeg = ( os.linesep + "-" * 68 diff --git a/src/PIL/features.py b/src/PIL/features.py index b14d6df13f2..7c5112ef6e4 100644 --- a/src/PIL/features.py +++ b/src/PIL/features.py @@ -249,12 +249,17 @@ def pilinfo(out=None, supported_formats=True): for py_version in py_version[1:]: print(f" {py_version.strip()}", file=out) print("-" * 68, file=out) + print(f"Python executable is {sys.executable or 'unknown'}", file=out) + if sys.prefix != sys.base_prefix: + print(f"Environment Python files loaded from {sys.prefix}", file=out) + print(f"System Python files loaded from {sys.base_prefix}", file=out) + print("-" * 68, file=out) print( - f"Python modules loaded from {os.path.dirname(Image.__file__)}", + f"Python Pillow modules loaded from {os.path.dirname(Image.__file__)}", file=out, ) print( - f"Binary modules loaded from {os.path.dirname(Image.core.__file__)}", + f"Binary Pillow modules loaded from {os.path.dirname(Image.core.__file__)}", file=out, ) print("-" * 68, file=out) From 89c44be404081c77e68457c1c50a78e96a6d42b9 Mon Sep 17 00:00:00 2001 From: Nulano Date: Tue, 20 Feb 2024 20:39:40 +0100 Subject: [PATCH 09/42] Mention `python -m PIL --bugreport` in the issue template --- .github/ISSUE_TEMPLATE/ISSUE_REPORT.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/ISSUE_REPORT.md b/.github/ISSUE_TEMPLATE/ISSUE_REPORT.md index 115f6135dfb..cfd576f3588 100644 --- a/.github/ISSUE_TEMPLATE/ISSUE_REPORT.md +++ b/.github/ISSUE_TEMPLATE/ISSUE_REPORT.md @@ -48,6 +48,10 @@ Thank you. * Python: * Pillow: +```text +please paste the output of running `python3 -m PIL --bugreport` here +``` +