From 95d4b9da508716100575b6795421884504726f9a Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 1 Apr 2023 08:55:25 +0100 Subject: [PATCH 1/8] Started implementing support for determinstic figure output --- pytest_mpl/plugin.py | 48 +++++++++++++++++++++++++++++++--------- tests/test_pytest_mpl.py | 20 ++--------------- 2 files changed, 39 insertions(+), 29 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 4128fd1..5f72c08 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -39,6 +39,7 @@ import contextlib from pathlib import Path from urllib.request import urlopen +from contextlib import nullcontext import pytest @@ -432,15 +433,13 @@ def generate_baseline_image(self, item, fig): Generate reference figures. """ compare = get_compare(item) - savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) if not os.path.exists(self.generate_dir): os.makedirs(self.generate_dir) baseline_filename = self.generate_filename(item) baseline_path = (self.generate_dir / baseline_filename).absolute() - fig.savefig(str(baseline_path), **savefig_kwargs) - + self.save_figure(item, fig, baseline_path) close_mpl_figure(fig) return baseline_path @@ -450,13 +449,9 @@ def generate_image_hash(self, item, fig): For a `matplotlib.figure.Figure`, returns the SHA256 hash as a hexadecimal string. """ - compare = get_compare(item) - savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) - - ext = self._file_extension(item) imgdata = io.BytesIO() - fig.savefig(imgdata, **savefig_kwargs) + self.save_figure(item, fig, imgdata) out = _hash_file(imgdata) imgdata.close() @@ -476,13 +471,14 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): compare = get_compare(item) tolerance = compare.kwargs.get('tolerance', 2) savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + deterministic = compare.kwargs.get('deterministic', False) ext = self._file_extension(item) baseline_image_ref = self.obtain_baseline_image(item, result_dir) test_image = (result_dir / f"result.{ext}").absolute() - fig.savefig(str(test_image), **savefig_kwargs) + self.save_figure(item, fig, test_image) if ext in ['png', 'svg']: # Use original file summary['result_image'] = test_image.relative_to(self.results_dir).as_posix() @@ -554,13 +550,43 @@ def load_hash_library(self, library_path): with open(str(library_path)) as fp: return json.load(fp) + def save_figure(self, item, fig, filename): + if isinstance(filename, Path): + filename = str(filename) + compare = get_compare(item) + savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + deterministic = compare.kwargs.get('deterministic', False) + + if deterministic: + + # Make sure we don't modify the original dictionary in case is a common + # object used by different tests + savefig_kwargs = savefig_kwargs.copy() + + if 'metadata' not in savefig_kwargs: + savefig_kwargs['metadata'] = {} + + ext = self._file_extension(item) + + if ext == 'png': + extra_metadata = {"Software": None} + elif ext == 'pdf': + extra_metadata = {"Creator": None, "Producer": None, "CreationDate": None} + elif ext == 'eps': + extra_metadata = {"Creator": "test"} + elif ext == 'svg': + extra_metadata = {"Date": None} + + savefig_kwargs['metadata'].update(extra_metadata) + + fig.savefig(filename, **savefig_kwargs) + def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): hash_comparison_pass = False if summary is None: summary = {} compare = get_compare(item) - savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) ext = self._file_extension(item) @@ -601,7 +627,7 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): # Save the figure for later summary (will be removed later if not needed) test_image = (result_dir / f"result.{ext}").absolute() - fig.savefig(str(test_image), **savefig_kwargs) + self.save_figure(item, fig, test_image) summary['result_image'] = test_image.relative_to(self.results_dir).as_posix() # Hybrid mode (hash and image comparison) diff --git a/tests/test_pytest_mpl.py b/tests/test_pytest_mpl.py index 23cfe78..73d959c 100644 --- a/tests/test_pytest_mpl.py +++ b/tests/test_pytest_mpl.py @@ -704,15 +704,6 @@ def test_formats(pytester, use_hash_library, passes, file_format): else: pytest.skip('Comparing EPS and PDF files requires ghostscript to be installed') - if file_format == 'png': - metadata = '{"Software": None}' - elif file_format == 'pdf': - metadata = '{"Creator": None, "Producer": None, "CreationDate": None}' - elif file_format == 'eps': - metadata = '{"Creator": "test"}' - elif file_format == 'svg': - metadata = '{"Date": None}' - pytester.makepyfile( f""" import os @@ -721,16 +712,9 @@ def test_formats(pytester, use_hash_library, passes, file_format): @pytest.mark.mpl_image_compare(baseline_dir=r"{baseline_dir_abs}", {f'hash_library=r"{hash_library}",' if use_hash_library else ''} tolerance={DEFAULT_TOLERANCE}, - savefig_kwargs={{'format': '{file_format}', - 'metadata': {metadata}}}) + deterministic=True, + savefig_kwargs={{'format': '{file_format}'}}) def test_format_{file_format}(): - - # For reproducible EPS output - os.environ['SOURCE_DATE_EPOCH'] = '1680254601' - - # For reproducible SVG output - plt.rcParams['svg.hashsalt'] = 'test' - fig = plt.figure() ax = fig.add_subplot(1, 1, 1) ax.plot([{1 if passes else 3}, 2, 3]) From 9993b5b2c81eb36b2ad1f00994c947892edd66dc Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 1 Apr 2023 09:15:12 +0100 Subject: [PATCH 2/8] Set environment and rc params as needed for reproducible EPS and SVG output --- pytest_mpl/plugin.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 5f72c08..17e670c 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -557,6 +557,8 @@ def save_figure(self, item, fig, filename): savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) deterministic = compare.kwargs.get('deterministic', False) + original_source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH', None) + if deterministic: # Make sure we don't modify the original dictionary in case is a common @@ -568,18 +570,28 @@ def save_figure(self, item, fig, filename): ext = self._file_extension(item) + extra_rcparams = {} + if ext == 'png': extra_metadata = {"Software": None} elif ext == 'pdf': extra_metadata = {"Creator": None, "Producer": None, "CreationDate": None} elif ext == 'eps': extra_metadata = {"Creator": "test"} + os.environ['SOURCE_DATE_EPOCH'] = '1234567890' elif ext == 'svg': extra_metadata = {"Date": None} + extra_rcparams["svg.hashsalt"] = "test" savefig_kwargs['metadata'].update(extra_metadata) - fig.savefig(filename, **savefig_kwargs) + import matplotlib.pyplot as plt + + with plt.rc_context(**extra_rcparams): + fig.savefig(filename, **savefig_kwargs) + + if original_source_date_epoch is not None: + os.environ['SOURCE_DATE_EPOCH'] = original_source_date_epoch def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): hash_comparison_pass = False From 15baf2e8fc04eb4cb3f6674a05941ce0fda6ffc1 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 1 Apr 2023 09:17:23 +0100 Subject: [PATCH 3/8] Update README --- README.rst | 64 +++++------------------------------------------------- 1 file changed, 5 insertions(+), 59 deletions(-) diff --git a/README.rst b/README.rst index 13e413e..a405755 100644 --- a/README.rst +++ b/README.rst @@ -288,69 +288,15 @@ Inkscape is required for SVG comparison. By default, Matplotlib does not produce deterministic output that will have a consistent hash every time it is run, or over different Matplotlib versions. In -order to enforce that the output is deterministic, you will need to set metadata -as described in the following subsections. - -PNG -^^^ - -For PNG files, the output can be made deterministic by setting: - -.. code:: python - - @pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {"Software": None}}) - -PDF -^^^ - -For PDF files, the output can be made deterministic by setting: - -.. code:: python - - @pytest.mark.mpl_image_compare(savefig_kwargs={'format': 'pdf', - 'metadata': {"Creator": None, - "Producer": None, - "CreationDate": None}}) - -Note that deterministic PDF output can only be achieved with Matplotlib 2.1 and above - -EPS -^^^ - -For PDF files, the output can be made deterministic by setting: - -.. code:: python - - @pytest.mark.mpl_image_compare(savefig_kwargs={'format': 'pdf', - 'metadata': {"Creator": "test"}) - -and in addition you will need to set the SOURCE_DATE_EPOCH environment variable to -a constant value (this is a unit timestamp): - -.. code:: python - - os.environ['SOURCE_DATE_EPOCH'] = '1680254601' - -You could do this inside the test. - -Note that deterministic PDF output can only be achieved with Matplotlib 2.1 and above - -SVG -^^^ - -For SVG files, the output can be made deterministic by setting: - -.. code:: python - - @pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': '{"Date": None}}) - -and in addition, you should make sure the following rcParam is set to a constant string: +order to enforce that the output is deterministic, you can set the ``deterministic`` +keyword argument in ``mpl_image_compare``: .. code:: python - plt.rcParams['svg.hashsalt'] = 'test' + @pytest.mark.mpl_image_compare(deterministic=True) -Note that SVG files can only be used in pytest-mpl with Matplotlib 3.3 and above. +This does a number of things such as e.g., setting the creation date in the +metadata to be constant, and avoids hard-coding the Matplotlib in the files. Test failure example -------------------- From fcb54851315dbf44fba2337039b37a54bb2e09a3 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 1 Apr 2023 09:18:08 +0100 Subject: [PATCH 4/8] Remove unused import --- pytest_mpl/plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 17e670c..56e8bae 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -39,7 +39,6 @@ import contextlib from pathlib import Path from urllib.request import urlopen -from contextlib import nullcontext import pytest From 6a4765515c5e8c41670b02da018b7b5bd349742d Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 1 Apr 2023 09:19:11 +0100 Subject: [PATCH 5/8] Remove unused variables --- pytest_mpl/plugin.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 56e8bae..11ce886 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -431,7 +431,6 @@ def generate_baseline_image(self, item, fig): """ Generate reference figures. """ - compare = get_compare(item) if not os.path.exists(self.generate_dir): os.makedirs(self.generate_dir) @@ -469,8 +468,6 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): compare = get_compare(item) tolerance = compare.kwargs.get('tolerance', 2) - savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) - deterministic = compare.kwargs.get('deterministic', False) ext = self._file_extension(item) From 8b83298dd8827a3ddf4581cd8bb1eb571d488974 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 1 Apr 2023 09:38:02 +0100 Subject: [PATCH 6/8] Fix undefined extra_rcparams --- pytest_mpl/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 11ce886..f4d5ed8 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -555,6 +555,8 @@ def save_figure(self, item, fig, filename): original_source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH', None) + extra_rcparams = {} + if deterministic: # Make sure we don't modify the original dictionary in case is a common @@ -566,8 +568,6 @@ def save_figure(self, item, fig, filename): ext = self._file_extension(item) - extra_rcparams = {} - if ext == 'png': extra_metadata = {"Software": None} elif ext == 'pdf': From bd41a5c2e8656a997805e1bd2d49dffe8113a036 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 1 Apr 2023 10:09:43 +0100 Subject: [PATCH 7/8] Change SOURCE_DATE_EPOCH back to previous value --- pytest_mpl/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index f4d5ed8..5be16b3 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -574,7 +574,7 @@ def save_figure(self, item, fig, filename): extra_metadata = {"Creator": None, "Producer": None, "CreationDate": None} elif ext == 'eps': extra_metadata = {"Creator": "test"} - os.environ['SOURCE_DATE_EPOCH'] = '1234567890' + os.environ['SOURCE_DATE_EPOCH'] = '1680254601' elif ext == 'svg': extra_metadata = {"Date": None} extra_rcparams["svg.hashsalt"] = "test" From 21d06537ba8c8da3180d63f994c4714318f50117 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 1 Apr 2023 10:26:27 +0100 Subject: [PATCH 8/8] Fix rc_context call --- pytest_mpl/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 5be16b3..afd38f2 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -583,7 +583,7 @@ def save_figure(self, item, fig, filename): import matplotlib.pyplot as plt - with plt.rc_context(**extra_rcparams): + with plt.rc_context(rc=extra_rcparams): fig.savefig(filename, **savefig_kwargs) if original_source_date_epoch is not None: