feat(testshift): add oonifindings A/B tests (#863)

* feat: add oonifindings A/B tests

* feat: A/B test oonifindings with new and legacy deployments

testshift/LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright 2022-present Open Observatory of Network Interference Foundation (OONI) ETS
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

testshift/README.md

@@ -0,0 +1,21 @@
+# testshift
+
+[![PyPI - Version](https://img.shields.io/pypi/v/testshift.svg)](https://pypi.org/project/testshift)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/testshift.svg)](https://pypi.org/project/testshift)
+
+-----
+
+**Table of Contents**
+
+- [Installation](#installation)
+- [License](#license)
+
+## Installation
+
+```console
+pip install testshift
+```
+
+## License
+
+`testshift` is distributed under the terms of the [OONI] (https://github.com/ooni/license/blob/master/software/LICENSE.md) license.

testshift/pyproject.toml

@@ -0,0 +1,95 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "testshift"
+dynamic = ["version"]
+description = ''
+
+dependencies = [
+  "httpx ~= 0.26.0",
+]
+
+readme = "README.md"
+requires-python = ">=3.11"
+license = "BSD-3-Clause"
+keywords = []
+authors = [
+  { name = "OONI", email = "[email protected]" },
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: Implementation :: CPython",
+  "Programming Language :: Python :: Implementation :: PyPy",
+]
+
+[project.urls]
+Documentation = "https://docs.ooni.org"
+Issues = "https://github.com/ooni/backend/issues"
+Source = "https://github.com/ooni/backend"
+
+[tool.hatch.version]
+path = "src/testshift/__about__.py"
+
+[tool.hatch.build.targets.sdist]
+include = ["BUILD_LABEL"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/testshift"]
+artifacts = ["BUILD_LABEL"]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.envs.default]
+dependencies = [
+  "pytest",
+  "pytest-cov",
+  "click",
+  "black",
+  "pytest-asyncio",
+]
+path = ".venv/"
+
+[tool.hatch.envs.default.scripts]
+test = "pytest {args:tests}"
+test-cov = "pytest -s --full-trace --log-level=INFO  --log-cli-level=INFO -v --setup-show --cov=./ --cov-report=xml --cov-report=html --cov-report=term {args:tests}"
+cov-report = ["coverage report"]
+cov = ["test-cov", "cov-report"]
+
+[[tool.hatch.envs.all.matrix]]
+python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
+
+[tool.hatch.envs.types]
+dependencies = [
+  "mypy>=1.0.0",
+]
+[tool.hatch.envs.types.scripts]
+check = "mypy --install-types --non-interactive {args:src/testshift tests}"
+
+[tool.coverage.run]
+source_pkgs = ["testshift", "tests"]
+branch = true
+parallel = true
+omit = [
+  "src/testshift/common/*",
+  "src/testshift/__about__.py"
+]
+
+[tool.coverage.paths]
+testshift = ["src/testshift", "*/testshift/src/testshift"]
+tests = ["tests", "*/testshift/tests"]
+
+[tool.coverage.report]
+exclude_lines = [
+  "no cov",
+  "if __name__ == .__main__.:",
+  "if TYPE_CHECKING:",
+]

testshift/src/testshift/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

testshift/tests/test_oonifindings.py

@@ -0,0 +1,50 @@
+from typing import Dict
+
+import httpx
+
+LEGACY_HOST = "https://api.ooni.io"
+ACTIVE_HOST = "https://api.dev.ooni.io"
+
+
+# NOTE: The new API has been updated to include one new response
+# field which we do not use and hence default to an empty value.
+#
+# `creator_account_id`: ""
+def check_search_response_keys(legacy_response: Dict, active_response: Dict):
+    legacy_keys = list(legacy_response.keys())
+    active_keys = list(active_response.keys())
+
+    assert len(active_keys) == len(legacy_keys) + 1
+
+    active_keys.remove("creator_account_id")
+    return sorted(active_keys) == sorted(legacy_keys)
+
+
+def test_oonifindings():
+    with httpx.Client(base_url=LEGACY_HOST) as legacy_client, httpx.Client(base_url=ACTIVE_HOST) as active_client:
+        legacy_response = legacy_client.get("api/v1/incidents/search?only_mine=false")
+        active_response = active_client.get("api/v1/incidents/search?only_mine=false")
+        legacy_incidents, active_incidents = legacy_response.json()["incidents"], active_response.json()["incidents"]
+
+        assert len(legacy_incidents) == len(active_incidents)
+
+        for idx in range(len(legacy_incidents)):
+            assert check_search_response_keys(legacy_incidents[idx], active_incidents[idx])
+
+        legacy_response = legacy_client.get("api/v1/incidents/search?only_mine=true")
+        active_response = active_client.get("api/v1/incidents/search?only_mine=true")
+        legacy_incidents, active_incidents = legacy_response.json()["incidents"], active_response.json()["incidents"]
+
+        assert len(legacy_incidents) == len(active_incidents)
+
+        for idx in range(len(legacy_incidents)):
+            assert check_search_response_keys(legacy_incidents[idx], active_incidents[idx])
+
+        incident_id = "330022197701"
+        legacy_response = legacy_client.get(f"api/v1/incidents/show/{incident_id}")
+        active_response = active_client.get(f"api/v1/incidents/show/{incident_id}")
+
+        legacy_incident = legacy_response.json()["incident"]
+        active_incident = active_response.json()["incident"]
+
+        assert check_search_response_keys(legacy_incident, active_incident)