chore(test_visibility): rename and privatize CI Visibility API to Test Visibility

.github/CODEOWNERS

@@ -59,7 +59,7 @@ tests/contrib/pytest                @DataDog/ci-app-libraries
 tests/contrib/pytest_bdd            @DataDog/ci-app-libraries
 tests/contrib/unittest_plugin       @DataDog/ci-app-libraries
 ddtrace/ext/ci.py                   @DataDog/ci-app-libraries
-ddtrace/ext/ci_visibility           @DataDog/ci-app-libraries
+ddtrace/ext/test_visibility         @DataDog/ci-app-libraries
 ddtrace/ext/test.py                 @DataDog/ci-app-libraries
 ddtrace/internal/ci_visibility      @DataDog/ci-app-libraries
 ddtrace/internal/codeowners.py      @DataDog/apm-core-python @datadog/ci-app-libraries

ddtrace/_trace/trace_handlers.py

@@ -749,6 +749,24 @@ def _on_redis_command_post(ctx: core.ExecutionContext, rowcount):
         ctx[ctx["call_key"]].set_metric(db.ROWCOUNT, rowcount)
 
 
+def _on_test_visibility_enable(config) -> None:
+    from ddtrace.internal.ci_visibility import CIVisibility
+
+    CIVisibility.enable(config=config)
+
+
+def _on_test_visibility_disable() -> None:
+    from ddtrace.internal.ci_visibility import CIVisibility
+
+    CIVisibility.disable()
+
+
+def _on_test_visibility_is_enabled() -> bool:
+    from ddtrace.internal.ci_visibility import CIVisibility
+
+    return CIVisibility.enabled
+
+
 def listen():
     core.on("wsgi.block.started", _wsgi_make_block_content, "status_headers_content")
     core.on("asgi.block.started", _asgi_make_block_content, "status_headers_content")
@@ -802,6 +820,10 @@ def listen():
     core.on("redis.async_command.post", _on_redis_command_post)
     core.on("redis.command.post", _on_redis_command_post)
 
+    core.on("test_visibility.enable", _on_test_visibility_enable)
+    core.on("test_visibility.disable", _on_test_visibility_disable)
+    core.on("test_visibility.is_enabled", _on_test_visibility_is_enabled, "is_enabled")
+
     for context_name in (
         "flask.call",
         "flask.jsonify",

ddtrace/contrib/pytest/_plugin_v2.py

@@ -26,14 +26,15 @@
 from ddtrace.contrib.pytest.plugin import is_enabled
 from ddtrace.contrib.unittest import unpatch as unpatch_unittest
 from ddtrace.ext import test
-from ddtrace.ext.ci_visibility.api import CIExcInfo
-from ddtrace.ext.ci_visibility.api import CIModule
-from ddtrace.ext.ci_visibility.api import CISession
-from ddtrace.ext.ci_visibility.api import CISuite
-from ddtrace.ext.ci_visibility.api import CITest
-from ddtrace.ext.ci_visibility.api import disable_ci_visibility
-from ddtrace.ext.ci_visibility.api import enable_ci_visibility
-from ddtrace.ext.ci_visibility.api import is_ci_visibility_enabled
+from ddtrace.ext.test_visibility.api import Test
+from ddtrace.ext.test_visibility.api import TestExcInfo
+from ddtrace.ext.test_visibility.api import TestModule
+from ddtrace.ext.test_visibility.api import TestSession
+from ddtrace.ext.test_visibility.api import TestSuite
+from ddtrace.ext.test_visibility.api import disable_test_visibility
+from ddtrace.ext.test_visibility.api import enable_test_visibility
+from ddtrace.ext.test_visibility.api import is_test_visibility_enabled
+from ddtrace.ext.test_visibility.coverage_lines import CoverageLines
 from ddtrace.internal.ci_visibility.constants import SKIPPED_BY_ITR_REASON
 from ddtrace.internal.ci_visibility.telemetry.coverage import COVERAGE_LIBRARY
 from ddtrace.internal.ci_visibility.telemetry.coverage import record_code_coverage_empty
@@ -42,7 +43,6 @@
 from ddtrace.internal.ci_visibility.utils import take_over_logger_stream_handler
 from ddtrace.internal.coverage.code import ModuleCodeCollector
 from ddtrace.internal.coverage.installer import install as install_coverage
-from ddtrace.internal.coverage.lines import CoverageLines
 from ddtrace.internal.logger import get_logger
 
 
@@ -57,20 +57,20 @@ def _handle_itr_should_skip(item, test_id) -> bool:
 
     This function has the side effect of marking the test as skipped immediately if it should be skipped.
     """
-    if not CISession.is_test_skipping_enabled():
+    if not TestSession.is_test_skipping_enabled():
         return False
 
     suite_id = test_id.parent_id
 
-    item_is_unskippable = CISuite.is_item_itr_unskippable(suite_id)
+    item_is_unskippable = TestSuite.is_itr_unskippable(suite_id)
 
-    if CISuite.is_item_itr_skippable(suite_id):
+    if TestSuite.is_itr_skippable(suite_id):
         if item_is_unskippable:
             # Marking the test as forced run also applies to its hierarchy
-            CITest.mark_itr_forced_run(test_id)
+            Test.mark_itr_forced_run(test_id)
             return False
         else:
-            CITest.mark_itr_skipped(test_id)
+            Test.mark_itr_skipped(test_id)
             # Marking the test as skipped by ITR so that it appears in pytest's output
             item.add_marker(pytest.mark.skip(reason=SKIPPED_BY_ITR_REASON))  # TODO don't rely on internal for reason
             return True
@@ -105,19 +105,19 @@ def _handle_collected_coverage(test_id, coverage_collector) -> None:
     for path_str, covered_lines in test_covered_lines.items():
         coverage_data[Path(path_str).absolute()] = covered_lines
 
-    CISuite.add_coverage_data(test_id.parent_id, coverage_data)
+    TestSuite.add_coverage_data(test_id.parent_id, coverage_data)
 
 
 def _handle_coverage_dependencies(suite_id) -> None:
-    coverage_data = CISuite.get_coverage_data(suite_id)
+    coverage_data = TestSuite.get_coverage_data(suite_id)
     coverage_paths = coverage_data.keys()
     import_coverage = ModuleCodeCollector.get_import_coverage_for_paths(coverage_paths)
-    CISuite.add_coverage_data(suite_id, import_coverage)
+    TestSuite.add_coverage_data(suite_id, import_coverage)
 
 
 def _disable_ci_visibility():
     try:
-        disable_ci_visibility()
+        disable_test_visibility()
     except:  # noqa: E722
         log.debug("encountered error during disable_ci_visibility", exc_info=True)
 
@@ -133,9 +133,9 @@ def pytest_load_initial_conftests(early_config, parser, args):
 
     try:
         take_over_logger_stream_handler()
-        enable_ci_visibility(config=dd_config.pytest)
-        if CISession.should_collect_coverage():
-            workspace_path = CISession.get_workspace_path()
+        enable_test_visibility(config=dd_config.pytest)
+        if TestSession.should_collect_coverage():
+            workspace_path = TestSession.get_workspace_path()
             if workspace_path is None:
                 workspace_path = Path.cwd().absolute()
             log.warning("Installing ModuleCodeCollector with include_paths=%s", [workspace_path])
@@ -150,7 +150,7 @@ def pytest_configure(config: pytest.Config) -> None:
         if is_enabled(config):
             take_over_logger_stream_handler()
             unpatch_unittest()
-            enable_ci_visibility(config=dd_config.pytest)
+            enable_test_visibility(config=dd_config.pytest)
             if _is_pytest_cov_enabled(config):
                 patch_coverage()
         else:
@@ -163,15 +163,15 @@ def pytest_configure(config: pytest.Config) -> None:
 
 
 def pytest_sessionstart(session: pytest.Session) -> None:
-    if not is_ci_visibility_enabled():
+    if not is_test_visibility_enabled():
         return
 
     log.debug("CI Visibility enabled - starting test session")
 
     try:
         command = _get_session_command(session)
 
-        CISession.discover(
+        TestSession.discover(
             test_command=command,
             test_framework=FRAMEWORK,
             test_framework_version=pytest.__version__,
@@ -182,7 +182,7 @@ def pytest_sessionstart(session: pytest.Session) -> None:
             reject_duplicates=False,
         )
 
-        CISession.start()
+        TestSession.start()
     except:  # noqa: E722
         log.debug("encountered error during session start, disabling Datadog CI Visibility", exc_info=True)
         _disable_ci_visibility()
@@ -194,39 +194,39 @@ def _pytest_collection_finish(session) -> None:
     NOTE: Using pytest_collection_finish instead of pytest_collection_modifyitems allows us to capture only the
     tests that pytest has selection for run (eg: with the use of -k as an argument).
     """
-    codeowners = CISession.get_codeowners()
+    codeowners = TestSession.get_codeowners()
 
     for item in session.items:
         test_id = _get_test_id_from_item(item)
         suite_id = test_id.parent_id
         module_id = suite_id.parent_id
 
         # TODO: don't rediscover modules and suites if already discovered
-        CIModule.discover(module_id, _get_module_path_from_item(item))
-        CISuite.discover(suite_id)
+        TestModule.discover(module_id, _get_module_path_from_item(item))
+        TestSuite.discover(suite_id)
 
         item_path = Path(item.path if hasattr(item, "path") else item.fspath).absolute()
 
         item_codeowners = codeowners.of(str(item_path)) if codeowners is not None else None
 
         source_file_info = _get_source_file_info(item, item_path)
 
-        CITest.discover(test_id, codeowners=item_codeowners, source_file_info=source_file_info)
+        Test.discover(test_id, codeowners=item_codeowners, source_file_info=source_file_info)
 
         markers = [marker.kwargs for marker in item.iter_markers(name="dd_tags")]
         for tags in markers:
-            CITest.set_tags(test_id, tags)
+            Test.set_tags(test_id, tags)
 
         # Pytest markers do not allow us to determine if the test or the suite was marked as unskippable, but any
         # test marked unskippable in a suite makes the entire suite unskippable (since we are in suite skipping
         # mode)
-        if CISession.is_test_skipping_enabled() and _is_test_unskippable(item):
-            CITest.mark_itr_unskippable(test_id)
-            CISuite.mark_itr_unskippable(suite_id)
+        if TestSession.is_test_skipping_enabled() and _is_test_unskippable(item):
+            Test.mark_itr_unskippable(test_id)
+            TestSuite.mark_itr_unskippable(suite_id)
 
 
 def pytest_collection_finish(session) -> None:
-    if not is_ci_visibility_enabled():
+    if not is_test_visibility_enabled():
         return
 
     try:
@@ -242,16 +242,16 @@ def _pytest_runtest_protocol_pre_yield(item) -> t.Optional[ModuleCodeCollector.C
     module_id = suite_id.parent_id
 
     # TODO: don't re-start modules if already started
-    CIModule.start(module_id)
-    CISuite.start(suite_id)
+    TestModule.start(module_id)
+    TestSuite.start(suite_id)
 
-    CITest.start(test_id)
+    Test.start(test_id)
 
     _handle_itr_should_skip(item, test_id)
 
-    item_will_skip = _pytest_marked_to_skip(item) or CITest.was_item_skipped_by_itr(test_id)
+    item_will_skip = _pytest_marked_to_skip(item) or Test.was_skipped_by_itr(test_id)
 
-    collect_test_coverage = CISession.should_collect_coverage() and not item_will_skip
+    collect_test_coverage = TestSession.should_collect_coverage() and not item_will_skip
 
     if collect_test_coverage:
         return _start_collecting_coverage()
@@ -274,19 +274,19 @@ def _pytest_runtest_protocol_post_yield(item, nextitem, coverage_collector):
     # - we trust that the next item is in the same module if it is in the same suite
     next_test_id = _get_test_id_from_item(nextitem) if nextitem else None
     if next_test_id is None or next_test_id.parent_id != suite_id:
-        if CISuite.is_item_itr_skippable(suite_id) and not CISuite.was_forced_run(suite_id):
-            CISuite.mark_itr_skipped(suite_id)
+        if TestSuite.is_itr_skippable(suite_id) and not TestSuite.was_forced_run(suite_id):
+            TestSuite.mark_itr_skipped(suite_id)
         else:
             _handle_coverage_dependencies(suite_id)
-            CISuite.finish(suite_id)
+            TestSuite.finish(suite_id)
         if nextitem is None or (next_test_id is not None and next_test_id.parent_id.parent_id != module_id):
-            CIModule.finish(module_id)
+            TestModule.finish(module_id)
 
 
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
 def pytest_runtest_protocol(item, nextitem) -> None:
     """Discovers tests, and starts tests, suites, and modules, then handles coverage data collection"""
-    if not is_ci_visibility_enabled():
+    if not is_test_visibility_enabled():
         yield
         return
 
@@ -320,15 +320,15 @@ def _pytest_runtest_makereport(item, call, outcome):
     # There are scenarios in which we may have already finished this item in setup or call, eg:
     # - it was skipped by ITR
     # - it was marked with skipif
-    if CITest.is_finished(test_id):
+    if Test.is_finished(test_id):
         return
 
     # In cases where a test was marked as XFAIL, the reason is only available during when call.when == "call", so we
     # add it as a tag immediately:
     if getattr(result, "wasxfail", None):
-        CITest.set_tag(test_id, XFAIL_REASON, result.wasxfail)
+        Test.set_tag(test_id, XFAIL_REASON, result.wasxfail)
     elif "xfail" in getattr(result, "keywords", []) and getattr(result, "longrepr", None):
-        CITest.set_tag(test_id, XFAIL_REASON, result.longrepr)
+        Test.set_tag(test_id, XFAIL_REASON, result.longrepr)
 
     # Only capture result if:
     # - there is an exception
@@ -341,49 +341,49 @@ def _pytest_runtest_makereport(item, call, outcome):
         return
 
     xfail = hasattr(result, "wasxfail") or "xfail" in result.keywords
-    xfail_reason_tag = CITest.get_tag(test_id, XFAIL_REASON) if xfail else None
+    xfail_reason_tag = Test.get_tag(test_id, XFAIL_REASON) if xfail else None
     has_skip_keyword = any(x in result.keywords for x in ["skip", "skipif", "skipped"])
 
     # If run with --runxfail flag, tests behave as if they were not marked with xfail,
     # that's why no XFAIL_REASON or test.RESULT tags will be added.
     if result.skipped:
-        if CITest.was_item_skipped_by_itr(test_id):
+        if Test.was_skipped_by_itr(test_id):
             # Items that were skipped by ITR already have their status set
             return
 
         if xfail and not has_skip_keyword:
             # XFail tests that fail are recorded skipped by pytest, should be passed instead
             if not item.config.option.runxfail:
-                CITest.set_tag(test_id, test.RESULT, test.Status.XFAIL.value)
+                Test.set_tag(test_id, test.RESULT, test.Status.XFAIL.value)
                 if xfail_reason_tag is None:
-                    CITest.set_tag(test_id, XFAIL_REASON, getattr(result, "wasxfail", "XFail"))
-                CITest.mark_pass(test_id)
+                    Test.set_tag(test_id, XFAIL_REASON, getattr(result, "wasxfail", "XFail"))
+                Test.mark_pass(test_id)
                 return
 
-        CITest.mark_skip(test_id, _extract_reason(call))
+        Test.mark_skip(test_id, _extract_reason(call))
         return
 
     if result.passed:
         if xfail and not has_skip_keyword and not item.config.option.runxfail:
             # XPass (strict=False) are recorded passed by pytest
             if xfail_reason_tag is None:
-                CITest.set_tag(test_id, XFAIL_REASON, "XFail")
-            CITest.set_tag(test_id, test.RESULT, test.Status.XPASS.value)
+                Test.set_tag(test_id, XFAIL_REASON, "XFail")
+            Test.set_tag(test_id, test.RESULT, test.Status.XPASS.value)
 
-        CITest.mark_pass(test_id)
+        Test.mark_pass(test_id)
         return
 
     if xfail and not has_skip_keyword and not item.config.option.runxfail:
         # XPass (strict=True) are recorded failed by pytest, longrepr contains reason
         if xfail_reason_tag is None:
-            CITest.set_tag(test_id, XFAIL_REASON, getattr(result, "longrepr", "XFail"))
-        CITest.set_tag(test_id, test.RESULT, test.Status.XPASS.value)
-        CITest.mark_fail(test_id)
+            Test.set_tag(test_id, XFAIL_REASON, getattr(result, "longrepr", "XFail"))
+        Test.set_tag(test_id, test.RESULT, test.Status.XPASS.value)
+        Test.mark_fail(test_id)
         return
 
-    exc_info = CIExcInfo(call.excinfo.type, call.excinfo.value, call.excinfo.tb) if call.excinfo else None
+    exc_info = TestExcInfo(call.excinfo.type, call.excinfo.value, call.excinfo.tb) if call.excinfo else None
 
-    CITest.mark_fail(test_id, exc_info)
+    Test.mark_fail(test_id, exc_info)
 
 
 @pytest.hookimpl(hookwrapper=True)
@@ -392,7 +392,7 @@ def pytest_runtest_makereport(item, call) -> None:
     outcome: pytest.TestReport
     outcome = yield
 
-    if not is_ci_visibility_enabled():
+    if not is_test_visibility_enabled():
         return
 
     try:
@@ -402,7 +402,7 @@ def pytest_runtest_makereport(item, call) -> None:
 
 
 def _pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
-    if not is_ci_visibility_enabled():
+    if not is_test_visibility_enabled():
         return
 
     # TODO: make coverage officially part of test session object so we don't use set_tag() directly.
@@ -416,17 +416,17 @@ def _pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
         if not isinstance(lines_pct_value, float):
             log.warning("Tried to add total covered percentage to session span but the format was unexpected")
             return
-        CISession.set_tag(test.TEST_LINES_PCT, lines_pct_value)
+        TestSession.set_tag(test.TEST_LINES_PCT, lines_pct_value)
 
     if ModuleCodeCollector.is_installed():
         ModuleCodeCollector.uninstall()
 
-    CISession.finish(force_finish_children=True)
-    disable_ci_visibility()
+    TestSession.finish(force_finish_children=True)
+    disable_test_visibility()
 
 
 def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
-    if not is_ci_visibility_enabled():
+    if not is_test_visibility_enabled():
         return
 
     try: