Skip to content

Commit

Permalink
Merge pull request #196 from astrofrog/deterministic-option
Browse files Browse the repository at this point in the history 
Started implementing support for deterministic figure output
  • Loading branch information
@astrofrog
astrofrog committed Apr 3, 2023
2 parents d5ed60c + 21d0653 commit e6c1308
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 90 deletions.
64 changes: 5 additions & 59 deletions README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,69 +288,15 @@ Inkscape is required for SVG comparison.

By default, Matplotlib does not produce deterministic output that will have a
consistent hash every time it is run, or over different Matplotlib versions. In
order to enforce that the output is deterministic, you will need to set metadata
as described in the following subsections.

PNG
^^^

For PNG files, the output can be made deterministic by setting:

.. code:: python
@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': {"Software": None}})
PDF
^^^

For PDF files, the output can be made deterministic by setting:

.. code:: python
@pytest.mark.mpl_image_compare(savefig_kwargs={'format': 'pdf',
'metadata': {"Creator": None,
"Producer": None,
"CreationDate": None}})
Note that deterministic PDF output can only be achieved with Matplotlib 2.1 and above

EPS
^^^

For PDF files, the output can be made deterministic by setting:

.. code:: python
@pytest.mark.mpl_image_compare(savefig_kwargs={'format': 'pdf',
'metadata': {"Creator": "test"})
and in addition you will need to set the SOURCE_DATE_EPOCH environment variable to
a constant value (this is a unit timestamp):
.. code:: python
os.environ['SOURCE_DATE_EPOCH'] = '1680254601'
You could do this inside the test.
Note that deterministic PDF output can only be achieved with Matplotlib 2.1 and above
SVG
^^^
For SVG files, the output can be made deterministic by setting:
.. code:: python
@pytest.mark.mpl_image_compare(savefig_kwargs={'metadata': '{"Date": None}})
and in addition, you should make sure the following rcParam is set to a constant string:
order to enforce that the output is deterministic, you can set the ``deterministic``
keyword argument in ``mpl_image_compare``:

.. code:: python
plt.rcParams['svg.hashsalt'] = 'test'
@pytest.mark.mpl_image_compare(deterministic=True)
Note that SVG files can only be used in pytest-mpl with Matplotlib 3.3 and above.
This does a number of things such as e.g., setting the creation date in the
metadata to be constant, and avoids hard-coding the Matplotlib in the files.

Test failure example
--------------------
Expand Down
60 changes: 47 additions & 13 deletions pytest_mpl/plugin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -431,16 +431,13 @@ def generate_baseline_image(self, item, fig):
"""
Generate reference figures.
"""
compare = get_compare(item)
savefig_kwargs = compare.kwargs.get('savefig_kwargs', {})

if not os.path.exists(self.generate_dir):
os.makedirs(self.generate_dir)

baseline_filename = self.generate_filename(item)
baseline_path = (self.generate_dir / baseline_filename).absolute()
fig.savefig(str(baseline_path), **savefig_kwargs)

self.save_figure(item, fig, baseline_path)
close_mpl_figure(fig)

return baseline_path
Expand All @@ -450,13 +447,9 @@ def generate_image_hash(self, item, fig):
For a `matplotlib.figure.Figure`, returns the SHA256 hash as a hexadecimal
string.
"""
compare = get_compare(item)
savefig_kwargs = compare.kwargs.get('savefig_kwargs', {})

ext = self._file_extension(item)

imgdata = io.BytesIO()
fig.savefig(imgdata, **savefig_kwargs)
self.save_figure(item, fig, imgdata)
out = _hash_file(imgdata)
imgdata.close()

Expand All @@ -475,14 +468,13 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None):

compare = get_compare(item)
tolerance = compare.kwargs.get('tolerance', 2)
savefig_kwargs = compare.kwargs.get('savefig_kwargs', {})

ext = self._file_extension(item)

baseline_image_ref = self.obtain_baseline_image(item, result_dir)

test_image = (result_dir / f"result.{ext}").absolute()
fig.savefig(str(test_image), **savefig_kwargs)
self.save_figure(item, fig, test_image)

if ext in ['png', 'svg']: # Use original file
summary['result_image'] = test_image.relative_to(self.results_dir).as_posix()
Expand Down Expand Up @@ -554,13 +546,55 @@ def load_hash_library(self, library_path):
with open(str(library_path)) as fp:
return json.load(fp)

def save_figure(self, item, fig, filename):
if isinstance(filename, Path):
filename = str(filename)
compare = get_compare(item)
savefig_kwargs = compare.kwargs.get('savefig_kwargs', {})
deterministic = compare.kwargs.get('deterministic', False)

original_source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH', None)

extra_rcparams = {}

if deterministic:

# Make sure we don't modify the original dictionary in case is a common
# object used by different tests
savefig_kwargs = savefig_kwargs.copy()

if 'metadata' not in savefig_kwargs:
savefig_kwargs['metadata'] = {}

ext = self._file_extension(item)

if ext == 'png':
extra_metadata = {"Software": None}
elif ext == 'pdf':
extra_metadata = {"Creator": None, "Producer": None, "CreationDate": None}
elif ext == 'eps':
extra_metadata = {"Creator": "test"}
os.environ['SOURCE_DATE_EPOCH'] = '1680254601'
elif ext == 'svg':
extra_metadata = {"Date": None}
extra_rcparams["svg.hashsalt"] = "test"

savefig_kwargs['metadata'].update(extra_metadata)

import matplotlib.pyplot as plt

with plt.rc_context(rc=extra_rcparams):
fig.savefig(filename, **savefig_kwargs)

if original_source_date_epoch is not None:
os.environ['SOURCE_DATE_EPOCH'] = original_source_date_epoch

def compare_image_to_hash_library(self, item, fig, result_dir, summary=None):
hash_comparison_pass = False
if summary is None:
summary = {}

compare = get_compare(item)
savefig_kwargs = compare.kwargs.get('savefig_kwargs', {})

ext = self._file_extension(item)

Expand Down Expand Up @@ -601,7 +635,7 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None):

# Save the figure for later summary (will be removed later if not needed)
test_image = (result_dir / f"result.{ext}").absolute()
fig.savefig(str(test_image), **savefig_kwargs)
self.save_figure(item, fig, test_image)
summary['result_image'] = test_image.relative_to(self.results_dir).as_posix()

# Hybrid mode (hash and image comparison)
Expand Down
20 changes: 2 additions & 18 deletions tests/test_pytest_mpl.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -704,15 +704,6 @@ def test_formats(pytester, use_hash_library, passes, file_format):
else:
pytest.skip('Comparing EPS and PDF files requires ghostscript to be installed')

if file_format == 'png':
metadata = '{"Software": None}'
elif file_format == 'pdf':
metadata = '{"Creator": None, "Producer": None, "CreationDate": None}'
elif file_format == 'eps':
metadata = '{"Creator": "test"}'
elif file_format == 'svg':
metadata = '{"Date": None}'

pytester.makepyfile(
f"""
import os
Expand All @@ -721,16 +712,9 @@ def test_formats(pytester, use_hash_library, passes, file_format):
@pytest.mark.mpl_image_compare(baseline_dir=r"{baseline_dir_abs}",
{f'hash_library=r"{hash_library}",' if use_hash_library else ''}
tolerance={DEFAULT_TOLERANCE},
savefig_kwargs={{'format': '{file_format}',
'metadata': {metadata}}})
deterministic=True,
savefig_kwargs={{'format': '{file_format}'}})
def test_format_{file_format}():
# For reproducible EPS output
os.environ['SOURCE_DATE_EPOCH'] = '1680254601'
# For reproducible SVG output
plt.rcParams['svg.hashsalt'] = 'test'
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot([{1 if passes else 3}, 2, 3])
Expand Down

0 comments on commit e6c1308

Please sign in to comment.