test: add new testutil.assert_jsonschema_error_contains() helper

This commit adds a new helper `assert_jsonschema_error_contains()`
to `testutil` and uses it everywhere where we check errors from
jsonschema.

osbuild/testutil/__init__.py

@@ -3,6 +3,7 @@
 """
 import os
 import pathlib
+import re
 import shutil
 
 
@@ -39,3 +40,20 @@ def make_fake_input_tree(tmpdir: pathlib.Path, fake_content: dict) -> str:
     basedir = tmpdir / "tree"
     make_fake_tree(basedir, fake_content)
     return os.fspath(basedir)
+
+
+def assert_jsonschema_error_contains(res, expected_err, expected_num_errs=None):
+    err_msgs = [e.as_dict()["message"] for e in res.errors]
+    if expected_num_errs is not None:
+        assert len(err_msgs) == expected_num_errs, \
+            f"expected exactly {expected_num_errs} errors in {[e.as_dict() for e in res.errors]}"
+    re_typ = getattr(re, 'Pattern', None)
+    # this can be removed once we no longer support py3.6 (re.Pattern is modern)
+    if not re_typ:
+        re_typ = getattr(re, '_pattern_type')
+    if isinstance(expected_err, re_typ):
+        finder = expected_err.search
+    else:
+        def finder(s): return expected_err in s  # pylint: disable=C0321
+    assert any(finder(err_msg)
+               for err_msg in err_msgs), f"{expected_err} not found in {err_msgs}"

stages/test/test_autotailor.py

@@ -7,6 +7,7 @@
 import pytest
 
 import osbuild.meta
+from osbuild import testutil
 from osbuild.testutil.imports import import_module_from_path
 
 TEST_INPUT = [
@@ -112,6 +113,4 @@ def test_schema_validation_oscap_autotailor(fake_input, test_data, expected_err)
     res = schema_validate_stage_oscap_autotailor(fake_input, test_data)
 
     assert res.valid is False
-    err_msgs = [e.as_dict()["message"] for e in res.errors]
-    assert len(res.errors) == 1, err_msgs
-    assert expected_err in err_msgs[0]
+    testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)

stages/test/test_bootupd.py

@@ -6,6 +6,7 @@
 import pytest
 
 import osbuild.meta
+from osbuild import testutil
 from osbuild.testutil.imports import import_module_from_path
 
 
@@ -54,9 +55,7 @@ def test_bootupd_schema_validation(test_data, expected_err):
         assert res.valid is True, f"err: {[e.as_dict() for e in res.errors]}"
     else:
         assert res.valid is False
-        assert len(res.errors) == 1, [e.as_dict() for e in res.errors]
-        err_msgs = [e.as_dict()["message"] for e in res.errors]
-        assert expected_err in err_msgs[0]
+        testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)
 
 
 @pytest.mark.parametrize("test_options,expected_bootupd_opts", [

stages/test/test_containers_storage_conf.py

@@ -11,7 +11,7 @@
     import pytoml as toml
 
 import osbuild.meta
-from osbuild.testutil import assert_dict_has
+from osbuild import testutil
 from osbuild.testutil.imports import import_module_from_path
 
 TEST_INPUT = [
@@ -56,15 +56,15 @@ def test_containers_storage_conf_integration(tmp_path, test_filename, test_stora
     assert conf is not None
 
     for (key, value) in expected:
-        assert_dict_has(conf, key, value)
+        testutil.assert_dict_has(conf, key, value)
 
 
 @pytest.mark.parametrize(
     "test_data,storage_test_data,expected_err",
     [
         # None, note that starting from jsonschema 4.21.0 the error changes
         # so we need a regexp here
-        ({}, {}, r"does not have enough properties|should be non-empty"),
+        ({}, {}, re.compile("does not have enough properties|should be non-empty")),
         # All options
         ({
             "filename": "/etc/containers/storage.conf",
@@ -120,6 +120,4 @@ def test_schema_validation_containers_storage_conf(test_data, storage_test_data,
         assert res.valid is True, f"err: {[e.as_dict() for e in res.errors]}"
     else:
         assert res.valid is False
-        err_msgs = [e.as_dict()["message"] for e in res.errors]
-        assert any(re.search(expected_err, err_msg)
-                   for err_msg in err_msgs), f"{expected_err} not found in {err_msgs}"
+        testutil.assert_jsonschema_error_contains(res, expected_err)

stages/test/test_erofs.py

@@ -7,6 +7,7 @@
 import pytest
 
 import osbuild.meta
+from osbuild import testutil
 from osbuild.testutil import has_executable, make_fake_input_tree
 from osbuild.testutil.imports import import_module_from_path
 
@@ -109,6 +110,4 @@ def test_schema_validation_erofs(test_data, expected_err):
         assert res.valid is True, f"err: {[e.as_dict() for e in res.errors]}"
     else:
         assert res.valid is False
-        assert len(res.errors) == 1, [e.as_dict() for e in res.errors]
-        err_msgs = [e.as_dict()["message"] for e in res.errors]
-        assert expected_err in err_msgs[0]
+        testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)

stages/test/test_kickstart.py

@@ -6,6 +6,7 @@
 import pytest
 
 import osbuild.meta
+from osbuild import testutil
 from osbuild.testutil import has_executable
 from osbuild.testutil.imports import import_module_from_path
 
@@ -365,6 +366,4 @@ def test_schema_validation_bad_apples(test_data, expected_err):
     res = schema_validate_kickstart_stage(test_data)
 
     assert res.valid is False
-    assert len(res.errors) == 1
-    err_msgs = [e.as_dict()["message"] for e in res.errors]
-    assert expected_err in err_msgs[0]
+    testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)

stages/test/test_machine-id.py

@@ -7,6 +7,7 @@
 import pytest
 
 import osbuild.meta
+from osbuild import testutil
 from osbuild.testutil.imports import import_module_from_path
 
 
@@ -72,9 +73,7 @@ def test_machine_id_schema_validation(test_data, expected_err):
     res = schema.validate(test_input)
 
     assert res.valid is False
-    assert len(res.errors) == 1
-    err_msgs = [e.as_dict()["message"] for e in res.errors]
-    assert expected_err in err_msgs[0]
+    testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)
 
 
 def test_machine_id_first_boot_unknown(tmp_path):

stages/test/test_mkfs_ext4.py

@@ -8,6 +8,7 @@
 import pytest
 
 import osbuild.meta
+from osbuild import testutil
 from osbuild.testutil import has_executable
 from osbuild.testutil.imports import import_module_from_path
 
@@ -48,9 +49,7 @@ def test_schema_validation_mkfs_ext4(test_data, expected_err):
         assert res.valid is True, f"err: {[e.as_dict() for e in res.errors]}"
     else:
         assert res.valid is False
-        assert len(res.errors) == 1, [e.as_dict() for e in res.errors]
-        err_msgs = [e.as_dict()["message"] for e in res.errors]
-        assert expected_err in err_msgs[0]
+        testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)
 
 
 @pytest.mark.skipif(not has_executable("mkfs.ext4"), reason="need mkfs.ext4")

stages/test/test_ostree_post_copy.py

@@ -6,6 +6,7 @@
 import pytest
 
 import osbuild.meta
+from osbuild import testutil
 from osbuild.testutil.imports import import_module_from_path
 
 
@@ -56,6 +57,4 @@ def test_schema_validation_ostree_post_copy(test_data, expected_err):
     res = schema_validate_stage_ostree_post_copy(test_data)
 
     assert res.valid is False
-    err_msgs = [e.as_dict()["message"] for e in res.errors]
-    assert len(res.errors) == 1, err_msgs
-    assert expected_err in err_msgs[0]
+    testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)

stages/test/test_selinux.py

@@ -6,7 +6,7 @@
 import pytest
 
 import osbuild.meta
-import osbuild.testutil
+from osbuild import testutil
 from osbuild.testutil.imports import import_module_from_path
 
 
@@ -42,18 +42,15 @@ def test_schema_validation_selinux(test_data, expected_err):
         assert res.valid is True, f"err: {[e.as_dict() for e in res.errors]}"
     else:
         assert res.valid is False
-        assert len(res.errors) == 1, [e.as_dict() for e in res.errors]
-        err_msgs = [e.as_dict()["message"] for e in res.errors]
-        assert expected_err in err_msgs[0]
+        testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)
 
 
 def test_schema_validation_selinux_file_context_required():
     test_data = {}
     res = schema_validation_selinux(test_data, implicit_file_contexts=False)
     assert res.valid is False
-    assert len(res.errors) == 1, [e.as_dict() for e in res.errors]
-    err_msgs = [e.as_dict()["message"] for e in res.errors]
-    assert "'file_contexts' is a required property" in err_msgs[0]
+    expected_err = "'file_contexts' is a required property"
+    testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)
 
 
 @patch("osbuild.util.selinux.setfiles")

stages/test/test_skopeo.py

@@ -5,6 +5,7 @@
 import pytest
 
 import osbuild.meta
+from osbuild import testutil
 
 
 @pytest.mark.parametrize("test_data,expected_err", [
@@ -37,6 +38,4 @@ def test_schema_validation_skopeo(test_data, expected_err):
         assert res.valid is True, f"err: {[e.as_dict() for e in res.errors]}"
     else:
         assert res.valid is False
-        assert len(res.errors) == 1, [e.as_dict() for e in res.errors]
-        err_msgs = [e.as_dict()["message"] for e in res.errors]
-        assert expected_err in err_msgs[0]
+        testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)

stages/test/test_xz.py

@@ -7,6 +7,7 @@
 import pytest
 
 import osbuild.meta
+from osbuild import testutil
 from osbuild.testutil import has_executable, make_fake_input_tree
 from osbuild.testutil.imports import import_module_from_path
 
@@ -35,9 +36,7 @@ def test_schema_validation_xz(test_data, expected_err):
         assert res.valid is True, f"err: {[e.as_dict() for e in res.errors]}"
     else:
         assert res.valid is False
-        assert len(res.errors) == 1, [e.as_dict() for e in res.errors]
-        err_msgs = [e.as_dict()["message"] for e in res.errors]
-        assert expected_err in err_msgs[0]
+        testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)
 
 
 @pytest.fixture(name="fake_input_tree")

stages/test/test_zstd.py

@@ -7,6 +7,7 @@
 import pytest
 
 import osbuild.meta
+from osbuild import testutil
 from osbuild.testutil import has_executable, make_fake_input_tree
 from osbuild.testutil.imports import import_module_from_path
 
@@ -35,9 +36,7 @@ def test_schema_validation_zstd(test_data, expected_err):
         assert res.valid is True, f"err: {[e.as_dict() for e in res.errors]}"
     else:
         assert res.valid is False
-        assert len(res.errors) == 1, [e.as_dict() for e in res.errors]
-        err_msgs = [e.as_dict()["message"] for e in res.errors]
-        assert expected_err in err_msgs[0]
+        testutil.assert_jsonschema_error_contains(res, expected_err, expected_num_errs=1)
 
 
 @pytest.fixture(name="fake_input_tree")

test/mod/test_testutil_jsonschema.py

@@ -0,0 +1,51 @@
+import re
+
+import pytest
+
+import osbuild.meta
+from osbuild import testutil
+
+fake_schema = {
+    "type": "object",
+    "required": ["name"],
+}
+
+
+@pytest.fixture(name="validation_error")
+def validation_error_fixture():
+    schema = osbuild.meta.Schema(fake_schema, "fake-schema")
+    res = schema.validate({"not": "name"})
+    assert res.valid is False
+    return res
+
+
+def test_assert_jsonschema_error_contains(validation_error):
+    expected_err = "'name' is a required property"
+    testutil.assert_jsonschema_error_contains(validation_error, expected_err)
+
+
+def test_assert_jsonschema_error_regex(validation_error):
+    expected_err = re.compile("'.*' is a required property")
+    testutil.assert_jsonschema_error_contains(validation_error, expected_err)
+
+
+def test_assert_jsonschema_error_not_contains(validation_error):
+    with pytest.raises(AssertionError, match=r'not-in-errs not found in \['):
+        testutil.assert_jsonschema_error_contains(validation_error, "not-in-errs")
+
+
+def test_assert_jsonschema_error_not_found_re(validation_error):
+    expected_err_re = re.compile("not-in-errs")
+    with pytest.raises(AssertionError, match=r"re.*not found in"):
+        testutil.assert_jsonschema_error_contains(validation_error, expected_err_re)
+
+
+def test_assert_jsonschema_error_num_errs(validation_error):
+    expected_err = "'name' is a required property"
+    testutil.assert_jsonschema_error_contains(validation_error, expected_err, expected_num_errs=1)
+
+
+def test_assert_jsonschema_error_num_errs_wrong(validation_error):
+    expected_err = "'name' is a required property"
+    with pytest.raises(AssertionError, match=r'expected exactly 99 errors in'):
+        testutil.assert_jsonschema_error_contains(validation_error, expected_err, expected_num_errs=99)