From ef002f996b82257cfcf00d9bc7357900e2bf6c35 Mon Sep 17 00:00:00 2001 From: "David R. MacIver" Date: Fri, 1 Nov 2024 10:01:04 +0000 Subject: [PATCH 1/3] Automatically reload settings profile if it's redefined --- hypothesis-python/RELEASE.rst | 5 +++ hypothesis-python/docs/settings.rst | 15 ++++++++ hypothesis-python/src/hypothesis/_settings.py | 7 +++- .../tests/cover/test_settings.py | 37 +++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..cd9fd8720b --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,5 @@ +RELEASE_TYPE: minor + +This changes the behaviour of settings profiles so that if you reregister the currently loaded profile it will automatically reload it. Previously you would have had to load it again. + +In particular this means that if you register a "ci" profile, it will automatically be used when Hypothesis detects you are running on CI. diff --git a/hypothesis-python/docs/settings.rst b/hypothesis-python/docs/settings.rst index 7a2a0557d6..c68e301a1d 100644 --- a/hypothesis-python/docs/settings.rst +++ b/hypothesis-python/docs/settings.rst @@ -258,6 +258,21 @@ Hypothesis will automatically detect certain common CI environments and use the when running in them. In particular, if you wish to use the ``ci`` profile, setting the ``CI`` environment variable will do this. +If you want to customise your CI behaviour, registering a new version of the ``ci`` profile will automatically be picked up in CI. For example, if you wanted to run more examples in CI, you might configure it as follows: + + +.. code-block:: python + + settings.register_profile( + "ci", + settings( + settings.get_profile("ci"), + max_examples=1000, + ), + ) + +This will configure your CI to run 1000 examples per test rather than the default of 100, and this change will automatically be picked up when running on a CI server. + .. _healthchecks: ------------- diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py index 6aab3fa188..066e7b9781 100644 --- a/hypothesis-python/src/hypothesis/_settings.py +++ b/hypothesis-python/src/hypothesis/_settings.py @@ -97,7 +97,7 @@ def default(cls): v = default_variable.value if v is not None: return v - if hasattr(settings, "_current_profile"): + if getattr(settings, "_current_profile", None) is not None: settings.load_profile(settings._current_profile) assert default_variable.value is not None return default_variable.value @@ -132,6 +132,7 @@ class settings(metaclass=settingsMeta): __definitions_are_locked = False _profiles: ClassVar[dict[str, "settings"]] = {} __module__ = "hypothesis" + _current_profile = None def __getattr__(self, name): if name in all_settings: @@ -319,6 +320,8 @@ def register_profile( """ check_type(str, name, "name") settings._profiles[name] = settings(parent=parent, **kwargs) + if settings._current_profile == name: + settings.load_profile(name) @staticmethod def get_profile(name: str) -> "settings": @@ -768,7 +771,7 @@ def note_deprecation( # This is tested in a subprocess so the branch doesn't show up in coverage. -if is_in_ci(): # pragma: no cover +if is_in_ci(): # pragma: no settings.load_profile("ci") assert settings.default is not None diff --git a/hypothesis-python/tests/cover/test_settings.py b/hypothesis-python/tests/cover/test_settings.py index 4e58926793..44eace34bf 100644 --- a/hypothesis-python/tests/cover/test_settings.py +++ b/hypothesis-python/tests/cover/test_settings.py @@ -558,3 +558,40 @@ def test_check_defaults_to_randomize_when_not_running_on_ci(): ).strip() == "False" ) + + +def test_reloads_the_loaded_profile_if_registered_again(): + prev_profile = settings._current_profile + try: + test_profile = "some nonsense profile purely for this test" + test_value = 123456 + settings.register_profile(test_profile, settings(max_examples=test_value)) + settings.load_profile(test_profile) + assert settings.default.max_examples == test_value + test_value_2 = 42 + settings.register_profile(test_profile, settings(max_examples=test_value_2)) + assert settings.default.max_examples == test_value_2 + finally: + if prev_profile is not None: + settings.load_profile(prev_profile) + + +CI_TESTING_SCRIPT = """ +from hypothesis import settings + +if __name__ == '__main__': + settings.register_profile("ci", settings(max_examples=42)) + assert settings.default.max_examples == 42 +""" + + +@skipif_emscripten +def test_will_automatically_pick_up_changes_to_ci_profile_in_ci(): + env = dict(os.environ) + env["CI"] = "true" + subprocess.check_call( + [sys.executable, "-c", CI_TESTING_SCRIPT], + env=env, + text=True, + encoding="utf-8", + ) From b2262b885f5b247f6f48be0886afee3f74b21f5e Mon Sep 17 00:00:00 2001 From: "David R. MacIver" Date: Thu, 7 Nov 2024 15:32:33 +0000 Subject: [PATCH 2/3] Update documentation --- hypothesis-python/docs/settings.rst | 7 +++---- hypothesis-python/src/hypothesis/_settings.py | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/hypothesis-python/docs/settings.rst b/hypothesis-python/docs/settings.rst index c68e301a1d..be210d4487 100644 --- a/hypothesis-python/docs/settings.rst +++ b/hypothesis-python/docs/settings.rst @@ -251,15 +251,14 @@ by your conftest you can load one with the command line option ``--hypothesis-pr Hypothesis comes with two built-in profiles, ``ci`` and ``default``. -``ci`` is set up to have good defaults for running in a CI environment, so emphasizes determinism, while the +``ci`` is set up to have good defaults for running in a CI environment, so emphasizes determinism, while the ``default`` settings are picked to be more likely to find bugs and to have a good workflow when used for local development. -Hypothesis will automatically detect certain common CI environments and use the CI profile automatically +Hypothesis will automatically detect certain common CI environments and use the ci profile automatically when running in them. In particular, if you wish to use the ``ci`` profile, setting the ``CI`` environment variable will do this. -If you want to customise your CI behaviour, registering a new version of the ``ci`` profile will automatically be picked up in CI. For example, if you wanted to run more examples in CI, you might configure it as follows: - +This will still be the case if you register your own ci profile. For example, if you wanted to run more examples in CI, you might configure it as follows: .. code-block:: python diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py index 066e7b9781..d01e2907cd 100644 --- a/hypothesis-python/src/hypothesis/_settings.py +++ b/hypothesis-python/src/hypothesis/_settings.py @@ -317,6 +317,10 @@ def register_profile( :class:`~hypothesis.settings`: optional ``parent`` settings, and keyword arguments for each setting that will be set differently to parent (or settings.default, if parent is None). + + If you register a profile that has already been defined and that profile + is the currently loaded profile, the new changes will take effect immediately, + and do not require reloading the profile. """ check_type(str, name, "name") settings._profiles[name] = settings(parent=parent, **kwargs) From 364f309ccddf1ba04561ce0b5b05eb23db09b37c Mon Sep 17 00:00:00 2001 From: "David R. MacIver" Date: Thu, 7 Nov 2024 15:32:38 +0000 Subject: [PATCH 3/3] Remove unneeded pragma --- hypothesis-python/src/hypothesis/_settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py index d01e2907cd..f4c77ca420 100644 --- a/hypothesis-python/src/hypothesis/_settings.py +++ b/hypothesis-python/src/hypothesis/_settings.py @@ -774,8 +774,7 @@ def note_deprecation( settings.register_profile("ci", CI) -# This is tested in a subprocess so the branch doesn't show up in coverage. -if is_in_ci(): # pragma: no +if is_in_ci(): settings.load_profile("ci") assert settings.default is not None