From ffe750e4ebd0416f2061d4d81587537e621d73ae Mon Sep 17 00:00:00 2001 From: DecFox <33030671+DecFox@users.noreply.github.com> Date: Thu, 15 Aug 2024 01:24:21 +0530 Subject: [PATCH] feat(testshift): add oonifindings A/B tests (#863) * feat: add oonifindings A/B tests * feat: A/B test oonifindings with new and legacy deployments --- testshift/LICENSE.txt | 26 ++++++++ testshift/README.md | 21 ++++++ testshift/pyproject.toml | 95 ++++++++++++++++++++++++++++ testshift/src/testshift/__about__.py | 1 + testshift/src/testshift/__init__.py | 0 testshift/tests/__init__.py | 0 testshift/tests/test_oonifindings.py | 50 +++++++++++++++ 7 files changed, 193 insertions(+) create mode 100644 testshift/LICENSE.txt create mode 100644 testshift/README.md create mode 100644 testshift/pyproject.toml create mode 100644 testshift/src/testshift/__about__.py create mode 100644 testshift/src/testshift/__init__.py create mode 100644 testshift/tests/__init__.py create mode 100644 testshift/tests/test_oonifindings.py diff --git a/testshift/LICENSE.txt b/testshift/LICENSE.txt new file mode 100644 index 00000000..3ec29c80 --- /dev/null +++ b/testshift/LICENSE.txt @@ -0,0 +1,26 @@ +Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/testshift/README.md b/testshift/README.md new file mode 100644 index 00000000..ef31d54c --- /dev/null +++ b/testshift/README.md @@ -0,0 +1,21 @@ +# testshift + +[![PyPI - Version](https://img.shields.io/pypi/v/testshift.svg)](https://pypi.org/project/testshift) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/testshift.svg)](https://pypi.org/project/testshift) + +----- + +**Table of Contents** + +- [Installation](#installation) +- [License](#license) + +## Installation + +```console +pip install testshift +``` + +## License + +`testshift` is distributed under the terms of the [OONI] (https://github.com/ooni/license/blob/master/software/LICENSE.md) license. diff --git a/testshift/pyproject.toml b/testshift/pyproject.toml new file mode 100644 index 00000000..13882f4a --- /dev/null +++ b/testshift/pyproject.toml @@ -0,0 +1,95 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "testshift" +dynamic = ["version"] +description = '' + +dependencies = [ + "httpx ~= 0.26.0", +] + +readme = "README.md" +requires-python = ">=3.11" +license = "BSD-3-Clause" +keywords = [] +authors = [ + { name = "OONI", email = "contact@ooni.org" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +[project.urls] +Documentation = "https://docs.ooni.org" +Issues = "https://github.com/ooni/backend/issues" +Source = "https://github.com/ooni/backend" + +[tool.hatch.version] +path = "src/testshift/__about__.py" + +[tool.hatch.build.targets.sdist] +include = ["BUILD_LABEL"] + +[tool.hatch.build.targets.wheel] +packages = ["src/testshift"] +artifacts = ["BUILD_LABEL"] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.envs.default] +dependencies = [ + "pytest", + "pytest-cov", + "click", + "black", + "pytest-asyncio", +] +path = ".venv/" + +[tool.hatch.envs.default.scripts] +test = "pytest {args:tests}" +test-cov = "pytest -s --full-trace --log-level=INFO --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=html --cov-report=term {args:tests}" +cov-report = ["coverage report"] +cov = ["test-cov", "cov-report"] + +[[tool.hatch.envs.all.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12"] + +[tool.hatch.envs.types] +dependencies = [ + "mypy>=1.0.0", +] +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:src/testshift tests}" + +[tool.coverage.run] +source_pkgs = ["testshift", "tests"] +branch = true +parallel = true +omit = [ + "src/testshift/common/*", + "src/testshift/__about__.py" +] + +[tool.coverage.paths] +testshift = ["src/testshift", "*/testshift/src/testshift"] +tests = ["tests", "*/testshift/tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] diff --git a/testshift/src/testshift/__about__.py b/testshift/src/testshift/__about__.py new file mode 100644 index 00000000..f102a9ca --- /dev/null +++ b/testshift/src/testshift/__about__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/testshift/src/testshift/__init__.py b/testshift/src/testshift/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testshift/tests/__init__.py b/testshift/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testshift/tests/test_oonifindings.py b/testshift/tests/test_oonifindings.py new file mode 100644 index 00000000..b095d2d7 --- /dev/null +++ b/testshift/tests/test_oonifindings.py @@ -0,0 +1,50 @@ +from typing import Dict + +import httpx + +LEGACY_HOST = "https://api.ooni.io" +ACTIVE_HOST = "https://api.dev.ooni.io" + + +# NOTE: The new API has been updated to include one new response +# field which we do not use and hence default to an empty value. +# +# `creator_account_id`: "" +def check_search_response_keys(legacy_response: Dict, active_response: Dict): + legacy_keys = list(legacy_response.keys()) + active_keys = list(active_response.keys()) + + assert len(active_keys) == len(legacy_keys) + 1 + + active_keys.remove("creator_account_id") + return sorted(active_keys) == sorted(legacy_keys) + + +def test_oonifindings(): + with httpx.Client(base_url=LEGACY_HOST) as legacy_client, httpx.Client(base_url=ACTIVE_HOST) as active_client: + legacy_response = legacy_client.get("api/v1/incidents/search?only_mine=false") + active_response = active_client.get("api/v1/incidents/search?only_mine=false") + legacy_incidents, active_incidents = legacy_response.json()["incidents"], active_response.json()["incidents"] + + assert len(legacy_incidents) == len(active_incidents) + + for idx in range(len(legacy_incidents)): + assert check_search_response_keys(legacy_incidents[idx], active_incidents[idx]) + + legacy_response = legacy_client.get("api/v1/incidents/search?only_mine=true") + active_response = active_client.get("api/v1/incidents/search?only_mine=true") + legacy_incidents, active_incidents = legacy_response.json()["incidents"], active_response.json()["incidents"] + + assert len(legacy_incidents) == len(active_incidents) + + for idx in range(len(legacy_incidents)): + assert check_search_response_keys(legacy_incidents[idx], active_incidents[idx]) + + incident_id = "330022197701" + legacy_response = legacy_client.get(f"api/v1/incidents/show/{incident_id}") + active_response = active_client.get(f"api/v1/incidents/show/{incident_id}") + + legacy_incident = legacy_response.json()["incident"] + active_incident = active_response.json()["incident"] + + assert check_search_response_keys(legacy_incident, active_incident)