diff --git a/ddtrace/contrib/pytest/_plugin_v2.py b/ddtrace/contrib/pytest/_plugin_v2.py index e763a14caa8..911f93a3426 100644 --- a/ddtrace/contrib/pytest/_plugin_v2.py +++ b/ddtrace/contrib/pytest/_plugin_v2.py @@ -26,11 +26,7 @@ from ddtrace.contrib.pytest.plugin import is_enabled from ddtrace.contrib.unittest import unpatch as unpatch_unittest from ddtrace.ext import test -from ddtrace.ext.test_visibility.api import Test from ddtrace.ext.test_visibility.api import TestExcInfo -from ddtrace.ext.test_visibility.api import TestModule -from ddtrace.ext.test_visibility.api import TestSession -from ddtrace.ext.test_visibility.api import TestSuite from ddtrace.ext.test_visibility.api import disable_test_visibility from ddtrace.ext.test_visibility.api import enable_test_visibility from ddtrace.ext.test_visibility.api import is_test_visibility_enabled @@ -44,6 +40,10 @@ from ddtrace.internal.coverage.code import ModuleCodeCollector from ddtrace.internal.coverage.installer import install as install_coverage from ddtrace.internal.logger import get_logger +from ddtrace.internal.test_visibility.api import InternalTest +from ddtrace.internal.test_visibility.api import InternalTestModule +from ddtrace.internal.test_visibility.api import InternalTestSession +from ddtrace.internal.test_visibility.api import InternalTestSuite log = get_logger(__name__) @@ -57,20 +57,20 @@ def _handle_itr_should_skip(item, test_id) -> bool: This function has the side effect of marking the test as skipped immediately if it should be skipped. """ - if not TestSession.is_test_skipping_enabled(): + if not InternalTestSession.is_test_skipping_enabled(): return False suite_id = test_id.parent_id - item_is_unskippable = TestSuite.is_itr_unskippable(suite_id) + item_is_unskippable = InternalTestSuite.is_itr_unskippable(suite_id) - if TestSuite.is_itr_skippable(suite_id): + if InternalTestSuite.is_itr_skippable(suite_id): if item_is_unskippable: # Marking the test as forced run also applies to its hierarchy - Test.mark_itr_forced_run(test_id) + InternalTest.mark_itr_forced_run(test_id) return False else: - Test.mark_itr_skipped(test_id) + InternalTest.mark_itr_skipped(test_id) # Marking the test as skipped by ITR so that it appears in pytest's output item.add_marker(pytest.mark.skip(reason=SKIPPED_BY_ITR_REASON)) # TODO don't rely on internal for reason return True @@ -105,14 +105,14 @@ def _handle_collected_coverage(test_id, coverage_collector) -> None: for path_str, covered_lines in test_covered_lines.items(): coverage_data[Path(path_str).absolute()] = covered_lines - TestSuite.add_coverage_data(test_id.parent_id, coverage_data) + InternalTestSuite.add_coverage_data(test_id.parent_id, coverage_data) def _handle_coverage_dependencies(suite_id) -> None: - coverage_data = TestSuite.get_coverage_data(suite_id) + coverage_data = InternalTestSuite.get_coverage_data(suite_id) coverage_paths = coverage_data.keys() import_coverage = ModuleCodeCollector.get_import_coverage_for_paths(coverage_paths) - TestSuite.add_coverage_data(suite_id, import_coverage) + InternalTestSuite.add_coverage_data(suite_id, import_coverage) def _disable_ci_visibility(): @@ -134,8 +134,8 @@ def pytest_load_initial_conftests(early_config, parser, args): try: take_over_logger_stream_handler() enable_test_visibility(config=dd_config.pytest) - if TestSession.should_collect_coverage(): - workspace_path = TestSession.get_workspace_path() + if InternalTestSession.should_collect_coverage(): + workspace_path = InternalTestSession.get_workspace_path() if workspace_path is None: workspace_path = Path.cwd().absolute() log.warning("Installing ModuleCodeCollector with include_paths=%s", [workspace_path]) @@ -171,7 +171,7 @@ def pytest_sessionstart(session: pytest.Session) -> None: try: command = _get_session_command(session) - TestSession.discover( + InternalTestSession.discover( test_command=command, test_framework=FRAMEWORK, test_framework_version=pytest.__version__, @@ -182,7 +182,7 @@ def pytest_sessionstart(session: pytest.Session) -> None: reject_duplicates=False, ) - TestSession.start() + InternalTestSession.start() except: # noqa: E722 log.debug("encountered error during session start, disabling Datadog CI Visibility", exc_info=True) _disable_ci_visibility() @@ -194,7 +194,7 @@ def _pytest_collection_finish(session) -> None: NOTE: Using pytest_collection_finish instead of pytest_collection_modifyitems allows us to capture only the tests that pytest has selection for run (eg: with the use of -k as an argument). """ - codeowners = TestSession.get_codeowners() + codeowners = InternalTestSession.get_codeowners() for item in session.items: test_id = _get_test_id_from_item(item) @@ -202,8 +202,8 @@ def _pytest_collection_finish(session) -> None: module_id = suite_id.parent_id # TODO: don't rediscover modules and suites if already discovered - TestModule.discover(module_id, _get_module_path_from_item(item)) - TestSuite.discover(suite_id) + InternalTestModule.discover(module_id, _get_module_path_from_item(item)) + InternalTestSuite.discover(suite_id) item_path = Path(item.path if hasattr(item, "path") else item.fspath).absolute() @@ -211,18 +211,18 @@ def _pytest_collection_finish(session) -> None: source_file_info = _get_source_file_info(item, item_path) - Test.discover(test_id, codeowners=item_codeowners, source_file_info=source_file_info) + InternalTest.discover(test_id, codeowners=item_codeowners, source_file_info=source_file_info) markers = [marker.kwargs for marker in item.iter_markers(name="dd_tags")] for tags in markers: - Test.set_tags(test_id, tags) + InternalTest.set_tags(test_id, tags) # Pytest markers do not allow us to determine if the test or the suite was marked as unskippable, but any # test marked unskippable in a suite makes the entire suite unskippable (since we are in suite skipping # mode) - if TestSession.is_test_skipping_enabled() and _is_test_unskippable(item): - Test.mark_itr_unskippable(test_id) - TestSuite.mark_itr_unskippable(suite_id) + if InternalTestSession.is_test_skipping_enabled() and _is_test_unskippable(item): + InternalTest.mark_itr_unskippable(test_id) + InternalTestSuite.mark_itr_unskippable(suite_id) def pytest_collection_finish(session) -> None: @@ -242,16 +242,16 @@ def _pytest_runtest_protocol_pre_yield(item) -> t.Optional[ModuleCodeCollector.C module_id = suite_id.parent_id # TODO: don't re-start modules if already started - TestModule.start(module_id) - TestSuite.start(suite_id) + InternalTestModule.start(module_id) + InternalTestSuite.start(suite_id) - Test.start(test_id) + InternalTest.start(test_id) _handle_itr_should_skip(item, test_id) - item_will_skip = _pytest_marked_to_skip(item) or Test.was_skipped_by_itr(test_id) + item_will_skip = _pytest_marked_to_skip(item) or InternalTest.was_skipped_by_itr(test_id) - collect_test_coverage = TestSession.should_collect_coverage() and not item_will_skip + collect_test_coverage = InternalTestSession.should_collect_coverage() and not item_will_skip if collect_test_coverage: return _start_collecting_coverage() @@ -274,13 +274,13 @@ def _pytest_runtest_protocol_post_yield(item, nextitem, coverage_collector): # - we trust that the next item is in the same module if it is in the same suite next_test_id = _get_test_id_from_item(nextitem) if nextitem else None if next_test_id is None or next_test_id.parent_id != suite_id: - if TestSuite.is_itr_skippable(suite_id) and not TestSuite.was_forced_run(suite_id): - TestSuite.mark_itr_skipped(suite_id) + if InternalTestSuite.is_itr_skippable(suite_id) and not InternalTestSuite.was_forced_run(suite_id): + InternalTestSuite.mark_itr_skipped(suite_id) else: _handle_coverage_dependencies(suite_id) - TestSuite.finish(suite_id) + InternalTestSuite.finish(suite_id) if nextitem is None or (next_test_id is not None and next_test_id.parent_id.parent_id != module_id): - TestModule.finish(module_id) + InternalTestModule.finish(module_id) @pytest.hookimpl(tryfirst=True, hookwrapper=True) @@ -320,15 +320,15 @@ def _pytest_runtest_makereport(item, call, outcome): # There are scenarios in which we may have already finished this item in setup or call, eg: # - it was skipped by ITR # - it was marked with skipif - if Test.is_finished(test_id): + if InternalTest.is_finished(test_id): return # In cases where a test was marked as XFAIL, the reason is only available during when call.when == "call", so we # add it as a tag immediately: if getattr(result, "wasxfail", None): - Test.set_tag(test_id, XFAIL_REASON, result.wasxfail) + InternalTest.set_tag(test_id, XFAIL_REASON, result.wasxfail) elif "xfail" in getattr(result, "keywords", []) and getattr(result, "longrepr", None): - Test.set_tag(test_id, XFAIL_REASON, result.longrepr) + InternalTest.set_tag(test_id, XFAIL_REASON, result.longrepr) # Only capture result if: # - there is an exception @@ -341,49 +341,49 @@ def _pytest_runtest_makereport(item, call, outcome): return xfail = hasattr(result, "wasxfail") or "xfail" in result.keywords - xfail_reason_tag = Test.get_tag(test_id, XFAIL_REASON) if xfail else None + xfail_reason_tag = InternalTest.get_tag(test_id, XFAIL_REASON) if xfail else None has_skip_keyword = any(x in result.keywords for x in ["skip", "skipif", "skipped"]) # If run with --runxfail flag, tests behave as if they were not marked with xfail, # that's why no XFAIL_REASON or test.RESULT tags will be added. if result.skipped: - if Test.was_skipped_by_itr(test_id): + if InternalTest.was_skipped_by_itr(test_id): # Items that were skipped by ITR already have their status set return if xfail and not has_skip_keyword: # XFail tests that fail are recorded skipped by pytest, should be passed instead if not item.config.option.runxfail: - Test.set_tag(test_id, test.RESULT, test.Status.XFAIL.value) + InternalTest.set_tag(test_id, test.RESULT, test.Status.XFAIL.value) if xfail_reason_tag is None: - Test.set_tag(test_id, XFAIL_REASON, getattr(result, "wasxfail", "XFail")) - Test.mark_pass(test_id) + InternalTest.set_tag(test_id, XFAIL_REASON, getattr(result, "wasxfail", "XFail")) + InternalTest.mark_pass(test_id) return - Test.mark_skip(test_id, _extract_reason(call)) + InternalTest.mark_skip(test_id, _extract_reason(call)) return if result.passed: if xfail and not has_skip_keyword and not item.config.option.runxfail: # XPass (strict=False) are recorded passed by pytest if xfail_reason_tag is None: - Test.set_tag(test_id, XFAIL_REASON, "XFail") - Test.set_tag(test_id, test.RESULT, test.Status.XPASS.value) + InternalTest.set_tag(test_id, XFAIL_REASON, "XFail") + InternalTest.set_tag(test_id, test.RESULT, test.Status.XPASS.value) - Test.mark_pass(test_id) + InternalTest.mark_pass(test_id) return if xfail and not has_skip_keyword and not item.config.option.runxfail: # XPass (strict=True) are recorded failed by pytest, longrepr contains reason if xfail_reason_tag is None: - Test.set_tag(test_id, XFAIL_REASON, getattr(result, "longrepr", "XFail")) - Test.set_tag(test_id, test.RESULT, test.Status.XPASS.value) - Test.mark_fail(test_id) + InternalTest.set_tag(test_id, XFAIL_REASON, getattr(result, "longrepr", "XFail")) + InternalTest.set_tag(test_id, test.RESULT, test.Status.XPASS.value) + InternalTest.mark_fail(test_id) return exc_info = TestExcInfo(call.excinfo.type, call.excinfo.value, call.excinfo.tb) if call.excinfo else None - Test.mark_fail(test_id, exc_info) + InternalTest.mark_fail(test_id, exc_info) @pytest.hookimpl(hookwrapper=True) @@ -416,12 +416,12 @@ def _pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: if not isinstance(lines_pct_value, float): log.warning("Tried to add total covered percentage to session span but the format was unexpected") return - TestSession.set_tag(test.TEST_LINES_PCT, lines_pct_value) + InternalTestSession.set_tag(test.TEST_LINES_PCT, lines_pct_value) if ModuleCodeCollector.is_installed(): ModuleCodeCollector.uninstall() - TestSession.finish(force_finish_children=True) + InternalTestSession.finish(force_finish_children=True) disable_test_visibility() diff --git a/ddtrace/ext/test_visibility/_utils.py b/ddtrace/ext/test_visibility/_utils.py index a136f3d93f4..e5ea3ac5fc8 100644 --- a/ddtrace/ext/test_visibility/_utils.py +++ b/ddtrace/ext/test_visibility/_utils.py @@ -3,7 +3,6 @@ from typing import Dict from typing import List -from ddtrace import Span from ddtrace.ext.test_visibility._test_visibility_base import TestVisibilityItemId from ddtrace.ext.test_visibility._test_visibility_base import _TestVisibilityAPIBase from ddtrace.internal import core @@ -59,12 +58,6 @@ def _delete_item_tags(item_id: TestVisibilityItemId, tag_names: List[str], recur core.dispatch("test_visibility.item.delete_tags", (_TestVisibilityAPIBase.DeleteTagsArgs(item_id, tag_names),)) -def _get_item_span(item_id: TestVisibilityItemId) -> Span: - log.debug("Getting span for item %s", item_id) - span: Span = core.dispatch_with_results("test_visibility.item.get_span", (item_id,)).span.value - return span - - def _is_item_finished(item_id: TestVisibilityItemId) -> bool: log.debug("Checking if item %s is finished", item_id) _is_finished = bool(core.dispatch_with_results("test_visibility.item.is_finished", (item_id,)).is_finished.value) diff --git a/ddtrace/ext/test_visibility/api.py b/ddtrace/ext/test_visibility/api.py index aac3c5ef957..4b79050751d 100644 --- a/ddtrace/ext/test_visibility/api.py +++ b/ddtrace/ext/test_visibility/api.py @@ -22,30 +22,23 @@ from typing import NamedTuple from typing import Optional from typing import Type -from typing import Union -from ddtrace import Span from ddtrace.ext.test import Status as _TestStatus from ddtrace.ext.test_visibility._test_visibility_base import TestSessionId from ddtrace.ext.test_visibility._test_visibility_base import TestSourceFileInfoBase from ddtrace.ext.test_visibility._test_visibility_base import TestVisibilityItemId from ddtrace.ext.test_visibility._test_visibility_base import _TestVisibilityAPIBase -from ddtrace.ext.test_visibility._test_visibility_base import _TestVisibilityChildItemIdBase -from ddtrace.ext.test_visibility._test_visibility_base import _TestVisibilityRootItemIdBase from ddtrace.ext.test_visibility._utils import _catch_and_log_exceptions from ddtrace.ext.test_visibility._utils import _delete_item_tag from ddtrace.ext.test_visibility._utils import _delete_item_tags -from ddtrace.ext.test_visibility._utils import _get_item_span from ddtrace.ext.test_visibility._utils import _get_item_tag from ddtrace.ext.test_visibility._utils import _is_item_finished from ddtrace.ext.test_visibility._utils import _set_item_tag from ddtrace.ext.test_visibility._utils import _set_item_tags -from ddtrace.ext.test_visibility.coverage_lines import CoverageLines from ddtrace.ext.test_visibility.item_ids import TestId from ddtrace.ext.test_visibility.item_ids import TestModuleId from ddtrace.ext.test_visibility.item_ids import TestSuiteId from ddtrace.internal import core -from ddtrace.internal.codeowners import Codeowners as _Codeowners from ddtrace.internal.logger import get_logger as _get_logger import ddtrace.tracing # noqa: F401 -- this triggers the registration of trace handlers @@ -128,10 +121,6 @@ def delete_tag(item_id: TestVisibilityItemId, tag_name: str, recurse: bool = Fal def delete_tags(item_id: TestVisibilityItemId, tag_names: List[str], recurse: bool = False): _delete_item_tags(item_id, tag_names, recurse) - @staticmethod - def get_span(item_id: TestVisibilityItemId) -> Span: - return _get_item_span(item_id) - @staticmethod def is_finished(item_id: TestVisibilityItemId) -> bool: return _is_item_finished(item_id) @@ -226,60 +215,6 @@ def delete_tag(tag_name: str, recurse: bool = False): def delete_tags(tag_names: List[str], recurse: bool = False): _delete_item_tags(TestSessionId(), tag_names, recurse) - @staticmethod - def get_span() -> Span: - return _get_item_span(TestSessionId()) - - @staticmethod - def is_finished() -> bool: - return _is_item_finished(TestSessionId()) - - @staticmethod - @_catch_and_log_exceptions - def get_codeowners() -> Optional[_Codeowners]: - log.debug("Getting codeowners object") - - codeowners: Optional[_Codeowners] = core.dispatch_with_results( - "test_visibility.session.get_codeowners", - ).codeowners.value - return codeowners - - @staticmethod - @_catch_and_log_exceptions - def get_workspace_path() -> Path: - log.debug("Getting session workspace path") - - workspace_path: Path = core.dispatch_with_results( - "test_visibility.session.get_workspace_path" - ).workspace_path.value - return workspace_path - - @staticmethod - @_catch_and_log_exceptions - def should_collect_coverage() -> bool: - log.debug("Checking if coverage should be collected for session") - - _should_collect_coverage = bool( - core.dispatch_with_results("test_visibility.session.should_collect_coverage").should_collect_coverage.value - ) - log.debug("Coverage should be collected: %s", _should_collect_coverage) - - return _should_collect_coverage - - @staticmethod - @_catch_and_log_exceptions - def is_test_skipping_enabled() -> bool: - log.debug("Checking if test skipping is enabled") - - _is_test_skipping_enabled = bool( - core.dispatch_with_results( - "test_visibility.session.is_test_skipping_enabled" - ).is_test_skipping_enabled.value - ) - log.debug("Test skipping is enabled: %s", _is_test_skipping_enabled) - - return _is_test_skipping_enabled - class TestModule(TestBase): class DiscoverArgs(NamedTuple): @@ -321,95 +256,7 @@ def finish( ) -class ITRMixin(TestBase): - """Mixin class for ITR-related functionality.""" - - @staticmethod - @_catch_and_log_exceptions - def mark_itr_skipped(item_id: Union[TestSuiteId, TestId]): - log.debug("Marking item %s as skipped by ITR", item_id) - core.dispatch("test_visibility.itr.finish_skipped_by_itr", (item_id,)) - - @staticmethod - @_catch_and_log_exceptions - def mark_itr_unskippable(item_id: Union[TestSuiteId, TestId]): - log.debug("Marking item %s as unskippable by ITR", item_id) - core.dispatch("test_visibility.itr.mark_unskippable", (item_id,)) - - @staticmethod - @_catch_and_log_exceptions - def mark_itr_forced_run(item_id: Union[TestSuiteId, TestId]): - log.debug("Marking item %s as unskippable by ITR", item_id) - core.dispatch("test_visibility.itr.mark_forced_run", (item_id,)) - - @staticmethod - @_catch_and_log_exceptions - def was_forced_run(item_id: Union[TestSuiteId, TestId]) -> bool: - """Skippable items are not currently tied to a test session, so no session ID is passed""" - log.debug("Checking if item %s was forced to run", item_id) - _was_forced_run = bool( - core.dispatch_with_results("test_visibility.itr.was_forced_run", (item_id,)).was_forced_run.value - ) - log.debug("Item %s was forced run: %s", item_id, _was_forced_run) - return _was_forced_run - - @staticmethod - @_catch_and_log_exceptions - def is_itr_skippable(item_id: Union[TestSuiteId, TestId]) -> bool: - """Skippable items are not currently tied to a test session, so no session ID is passed""" - log.debug("Checking if item %s is skippable", item_id) - is_item_skippable = bool( - core.dispatch_with_results("test_visibility.itr.is_item_skippable", (item_id,)).is_item_skippable.value - ) - log.debug("Item %s is skippable: %s", item_id, is_item_skippable) - - return is_item_skippable - - @staticmethod - @_catch_and_log_exceptions - def is_itr_unskippable(item_id: Union[TestSuiteId, TestId]) -> bool: - """Skippable items are not currently tied to a test session, so no session ID is passed""" - log.debug("Checking if item %s is unskippable", item_id) - is_item_unskippable = bool( - core.dispatch_with_results("test_visibility.itr.is_item_unskippable", (item_id,)).is_item_unskippable.value - ) - log.debug("Item %s is unskippable: %s", item_id, is_item_unskippable) - - return is_item_unskippable - - @staticmethod - @_catch_and_log_exceptions - def was_skipped_by_itr(item_id: Union[TestSuiteId, TestId]) -> bool: - """Skippable items are not currently tied to a test session, so no session ID is passed""" - log.debug("Checking if item %s was skipped by ITR", item_id) - was_item_skipped = bool( - core.dispatch_with_results("test_visibility.itr.was_item_skipped", (item_id,)).was_item_skipped.value - ) - log.debug("Item %s was skipped by ITR: %s", item_id, was_item_skipped) - return was_item_skipped - - class AddCoverageArgs(NamedTuple): - item_id: Union[_TestVisibilityChildItemIdBase, _TestVisibilityRootItemIdBase] - coverage_data: Dict[Path, CoverageLines] - - @staticmethod - @_catch_and_log_exceptions - def add_coverage_data(item_id: Union[TestSuiteId, TestId], coverage_data: Dict[Path, CoverageLines]): - log.debug("Adding coverage data for item %s: %s", item_id, coverage_data) - core.dispatch("test_visibility.item.add_coverage_data", (ITRMixin.AddCoverageArgs(item_id, coverage_data),)) - - @staticmethod - @_catch_and_log_exceptions - def get_coverage_data(item_id: Union[TestSuiteId, TestId]) -> Optional[Dict[Path, CoverageLines]]: - log.debug("Getting coverage data for item %s", item_id) - coverage_data = core.dispatch_with_results( - "test_visibility.item.get_coverage_data", (item_id,) - ).coverage_data.value - log.debug("Coverage data for item %s: %s", item_id, coverage_data) - return coverage_data - - -class TestSuite(ITRMixin, TestBase): +class TestSuite(TestBase): class DiscoverArgs(NamedTuple): suite_id: TestSuiteId codeowners: Optional[List[str]] = None @@ -458,7 +305,7 @@ def finish( ) -class Test(ITRMixin, TestBase): +class Test(TestBase): class DiscoverArgs(NamedTuple): test_id: TestId codeowners: Optional[List[str]] = None @@ -486,24 +333,6 @@ def discover( "test_visibility.test.discover", (Test.DiscoverArgs(item_id, codeowners, source_file_info, resource),) ) - class DiscoverEarlyFlakeRetryArgs(NamedTuple): - test_id: TestId - retry_number: int - - @staticmethod - @_catch_and_log_exceptions - def discover_early_flake_retry(item_id: TestId): - if item_id.retry_number <= 0: - log.warning( - "Cannot register early flake retry of test %s with retry number %s", item_id, item_id.retry_number - ) - log.debug("Registered early flake retry for test %s, retry number: %s", item_id, item_id.retry_number) - original_test_id = TestId(item_id.parent_id, item_id.name, item_id.parameters) - core.dispatch( - "test_visibility.test.discover_early_flake_retry", - (Test.DiscoverEarlyFlakeRetryArgs(original_test_id, item_id.retry_number),), - ) - @staticmethod @_catch_and_log_exceptions def start(item_id: TestId): @@ -536,11 +365,6 @@ def finish( (Test.FinishArgs(item_id, status, skip_reason=skip_reason, exc_info=exc_info),), ) - @staticmethod - @_catch_and_log_exceptions - def mark_unskippable(): - log.debug("Marking test as unskippable") - @staticmethod @_catch_and_log_exceptions def mark_pass(item_id: TestId): diff --git a/ddtrace/ext/test_visibility/item_ids.py b/ddtrace/ext/test_visibility/item_ids.py index 64c2e0e3422..aec7bd21ab6 100644 --- a/ddtrace/ext/test_visibility/item_ids.py +++ b/ddtrace/ext/test_visibility/item_ids.py @@ -28,13 +28,11 @@ def __repr__(self): @dataclasses.dataclass(frozen=True) class TestId(_TestVisibilityChildItemIdBase[TestSuiteId]): parameters: Optional[str] = None # For hashability, a JSON string of a dictionary of parameters - retry_number: int = 0 def __repr__(self): - return "TestId(module={}, suite={}, test={}, parameters={}, retry_number={})".format( + return "TestId(module={}, suite={}, test={}, parameters={})".format( self.parent_id.parent_id.name, self.parent_id.name, self.name, self.parameters, - self.retry_number, ) diff --git a/ddtrace/internal/ci_visibility/api/_test.py b/ddtrace/internal/ci_visibility/api/_test.py index eaf47c48589..5b195f8ad2a 100644 --- a/ddtrace/internal/ci_visibility/api/_test.py +++ b/ddtrace/internal/ci_visibility/api/_test.py @@ -2,6 +2,7 @@ from typing import Dict from typing import List from typing import Optional +from typing import Union from ddtrace.ext import test from ddtrace.ext.test_visibility.api import TestExcInfo @@ -18,12 +19,15 @@ from ddtrace.internal.ci_visibility.telemetry.events import record_event_created from ddtrace.internal.ci_visibility.telemetry.events import record_event_finished from ddtrace.internal.logger import get_logger +from ddtrace.internal.test_visibility.api import InternalTestId log = get_logger(__name__) +TID = Union[TestId, InternalTestId] -class TestVisibilityTest(TestVisibilityChildItem[TestId], TestVisibilityItemBase): + +class TestVisibilityTest(TestVisibilityChildItem[TID], TestVisibilityItemBase): _event_type = TEST _event_type_metric_name = EVENT_TYPES.TEST @@ -123,11 +127,11 @@ def finish_itr_skipped(self) -> None: self.mark_itr_skipped() self.finish_test(TestStatus.SKIP) - def make_early_flake_retry_from_test(self, original_test_id: TestId, retry_number: int) -> None: + def make_early_flake_retry_from_test(self, original_test_id: InternalTestId, retry_number: int) -> None: if self.parent is None: raise ValueError("Cannot make early flake retry from test without a parent") - new_test_id = TestId( + new_test_id = InternalTestId( original_test_id.parent_id, original_test_id.name, original_test_id.parameters, retry_number ) self.parent.add_child( diff --git a/ddtrace/internal/ci_visibility/recorder.py b/ddtrace/internal/ci_visibility/recorder.py index 259ae07470f..c6d6ce72375 100644 --- a/ddtrace/internal/ci_visibility/recorder.py +++ b/ddtrace/internal/ci_visibility/recorder.py @@ -20,7 +20,6 @@ from ddtrace.ext import test from ddtrace.ext.test_visibility._test_visibility_base import TestSessionId from ddtrace.ext.test_visibility._test_visibility_base import TestVisibilityItemId -from ddtrace.ext.test_visibility.api import ITRMixin from ddtrace.ext.test_visibility.api import Test from ddtrace.ext.test_visibility.api import TestBase from ddtrace.ext.test_visibility.api import TestId @@ -43,6 +42,8 @@ from ddtrace.internal.compat import parse from ddtrace.internal.logger import get_logger from ddtrace.internal.service import Service +from ddtrace.internal.test_visibility.api import InternalTest +from ddtrace.internal.test_visibility.api import ITRMixin from ddtrace.internal.utils.formats import asbool from ddtrace.internal.writer.writer import Response @@ -1080,7 +1081,7 @@ def _on_discover_test(discover_args: Test.DiscoverArgs): @_requires_civisibility_enabled -def _on_discover_test_early_flake_retry(args: Test.DiscoverEarlyFlakeRetryArgs): +def _on_discover_test_early_flake_retry(args: InternalTest.DiscoverEarlyFlakeRetryArgs): log.debug("Handling early flake discovery for test %s", args.test_id) try: original_test = CIVisibility.get_test_by_id(args.test_id) diff --git a/ddtrace/internal/test_visibility/_utils.py b/ddtrace/internal/test_visibility/_utils.py new file mode 100644 index 00000000000..14cbd946aeb --- /dev/null +++ b/ddtrace/internal/test_visibility/_utils.py @@ -0,0 +1,13 @@ +from ddtrace import Span +from ddtrace.ext.test_visibility._test_visibility_base import TestVisibilityItemId +from ddtrace.internal import core +from ddtrace.internal.logger import get_logger + + +log = get_logger(__name__) + + +def _get_item_span(item_id: TestVisibilityItemId) -> Span: + log.debug("Getting span for item %s", item_id) + span: Span = core.dispatch_with_results("test_visibility.item.get_span", (item_id,)).span.value + return span diff --git a/ddtrace/internal/test_visibility/api.py b/ddtrace/internal/test_visibility/api.py new file mode 100644 index 00000000000..90d0446607e --- /dev/null +++ b/ddtrace/internal/test_visibility/api.py @@ -0,0 +1,212 @@ +import dataclasses +from pathlib import Path +import typing as t + +from ddtrace import Span +from ddtrace.ext.test_visibility import api as ext_api +from ddtrace.ext.test_visibility._test_visibility_base import TestSessionId +from ddtrace.ext.test_visibility._utils import _catch_and_log_exceptions +from ddtrace.ext.test_visibility._utils import _is_item_finished +from ddtrace.ext.test_visibility.coverage_lines import CoverageLines +from ddtrace.internal import core +from ddtrace.internal.codeowners import Codeowners as _Codeowners +from ddtrace.internal.logger import get_logger +from ddtrace.internal.test_visibility._utils import _get_item_span + + +log = get_logger(__name__) + + +@dataclasses.dataclass(frozen=True) +class InternalTestId(ext_api.TestId): + retry_number: int = 0 + + def __repr__(self): + return "TestId(module={}, suite={}, test={}, parameters={}, retry_number={})".format( + self.parent_id.parent_id.name, + self.parent_id.name, + self.name, + self.parameters, + self.retry_number, + ) + + +class ITRMixin(ext_api.TestBase): + """Mixin class for ITR-related functionality.""" + + @staticmethod + @_catch_and_log_exceptions + def mark_itr_skipped(item_id: t.Union[ext_api.TestSuiteId, ext_api.TestId]): + log.debug("Marking item %s as skipped by ITR", item_id) + core.dispatch("test_visibility.itr.finish_skipped_by_itr", (item_id,)) + + @staticmethod + @_catch_and_log_exceptions + def mark_itr_unskippable(item_id: t.Union[ext_api.TestSuiteId, ext_api.TestId]): + log.debug("Marking item %s as unskippable by ITR", item_id) + core.dispatch("test_visibility.itr.mark_unskippable", (item_id,)) + + @staticmethod + @_catch_and_log_exceptions + def mark_itr_forced_run(item_id: t.Union[ext_api.TestSuiteId, ext_api.TestId]): + log.debug("Marking item %s as unskippable by ITR", item_id) + core.dispatch("test_visibility.itr.mark_forced_run", (item_id,)) + + @staticmethod + @_catch_and_log_exceptions + def was_forced_run(item_id: t.Union[ext_api.TestSuiteId, ext_api.TestId]) -> bool: + """Skippable items are not currently tied to a test session, so no session ID is passed""" + log.debug("Checking if item %s was forced to run", item_id) + _was_forced_run = bool( + core.dispatch_with_results("test_visibility.itr.was_forced_run", (item_id,)).was_forced_run.value + ) + log.debug("Item %s was forced run: %s", item_id, _was_forced_run) + return _was_forced_run + + @staticmethod + @_catch_and_log_exceptions + def is_itr_skippable(item_id: t.Union[ext_api.TestSuiteId, ext_api.TestId]) -> bool: + """Skippable items are not currently tied to a test session, so no session ID is passed""" + log.debug("Checking if item %s is skippable", item_id) + is_item_skippable = bool( + core.dispatch_with_results("test_visibility.itr.is_item_skippable", (item_id,)).is_item_skippable.value + ) + log.debug("Item %s is skippable: %s", item_id, is_item_skippable) + + return is_item_skippable + + @staticmethod + @_catch_and_log_exceptions + def is_itr_unskippable(item_id: t.Union[ext_api.TestSuiteId, ext_api.TestId]) -> bool: + """Skippable items are not currently tied to a test session, so no session ID is passed""" + log.debug("Checking if item %s is unskippable", item_id) + is_item_unskippable = bool( + core.dispatch_with_results("test_visibility.itr.is_item_unskippable", (item_id,)).is_item_unskippable.value + ) + log.debug("Item %s is unskippable: %s", item_id, is_item_unskippable) + + return is_item_unskippable + + @staticmethod + @_catch_and_log_exceptions + def was_skipped_by_itr(item_id: t.Union[ext_api.TestSuiteId, ext_api.TestId]) -> bool: + """Skippable items are not currently tied to a test session, so no session ID is passed""" + log.debug("Checking if item %s was skipped by ITR", item_id) + was_item_skipped = bool( + core.dispatch_with_results("test_visibility.itr.was_item_skipped", (item_id,)).was_item_skipped.value + ) + log.debug("Item %s was skipped by ITR: %s", item_id, was_item_skipped) + return was_item_skipped + + class AddCoverageArgs(t.NamedTuple): + item_id: t.Union[ext_api.TestSuiteId, ext_api.TestId] + coverage_data: t.Dict[Path, CoverageLines] + + @staticmethod + @_catch_and_log_exceptions + def add_coverage_data( + item_id: t.Union[ext_api.TestSuiteId, ext_api.TestId], coverage_data: t.Dict[Path, CoverageLines] + ): + log.debug("Adding coverage data for item %s: %s", item_id, coverage_data) + core.dispatch("test_visibility.item.add_coverage_data", (ITRMixin.AddCoverageArgs(item_id, coverage_data),)) + + @staticmethod + @_catch_and_log_exceptions + def get_coverage_data( + item_id: t.Union[ext_api.TestSuiteId, ext_api.TestId] + ) -> t.Optional[t.Dict[Path, CoverageLines]]: + log.debug("Getting coverage data for item %s", item_id) + coverage_data = core.dispatch_with_results( + "test_visibility.item.get_coverage_data", (item_id,) + ).coverage_data.value + log.debug("Coverage data for item %s: %s", item_id, coverage_data) + return coverage_data + + +class InternalTestSession(ext_api.TestSession): + @staticmethod + def get_span() -> Span: + return _get_item_span(TestSessionId()) + + @staticmethod + def is_finished() -> bool: + return _is_item_finished(TestSessionId()) + + @staticmethod + @_catch_and_log_exceptions + def get_codeowners() -> t.Optional[_Codeowners]: + log.debug("Getting codeowners object") + + codeowners: t.Optional[_Codeowners] = core.dispatch_with_results( + "test_visibility.session.get_codeowners", + ).codeowners.value + return codeowners + + @staticmethod + @_catch_and_log_exceptions + def get_workspace_path() -> Path: + log.debug("Getting session workspace path") + + workspace_path: Path = core.dispatch_with_results( + "test_visibility.session.get_workspace_path" + ).workspace_path.value + return workspace_path + + @staticmethod + @_catch_and_log_exceptions + def should_collect_coverage() -> bool: + log.debug("Checking if coverage should be collected for session") + + _should_collect_coverage = bool( + core.dispatch_with_results("test_visibility.session.should_collect_coverage").should_collect_coverage.value + ) + log.debug("Coverage should be collected: %s", _should_collect_coverage) + + return _should_collect_coverage + + @staticmethod + @_catch_and_log_exceptions + def is_test_skipping_enabled() -> bool: + log.debug("Checking if test skipping is enabled") + + _is_test_skipping_enabled = bool( + core.dispatch_with_results( + "test_visibility.session.is_test_skipping_enabled" + ).is_test_skipping_enabled.value + ) + log.debug("Test skipping is enabled: %s", _is_test_skipping_enabled) + + return _is_test_skipping_enabled + + +class InternalTestModule(ext_api.TestModule): + pass + + +class InternalTestSuite(ext_api.TestSuite, ITRMixin): + pass + + +class InternalTest(ext_api.Test, ITRMixin): + class DiscoverEarlyFlakeRetryArgs(t.NamedTuple): + test_id: InternalTestId + retry_number: int + + @staticmethod + @_catch_and_log_exceptions + def discover_early_flake_retry(item_id: InternalTestId): + if item_id.retry_number <= 0: + log.warning( + "Cannot register early flake retry of test %s with retry number %s", item_id, item_id.retry_number + ) + log.debug("Registered early flake retry for test %s, retry number: %s", item_id, item_id.retry_number) + original_test_id = InternalTestId(item_id.parent_id, item_id.name, item_id.parameters) + core.dispatch( + "test_visibility.test.discover_early_flake_retry", + (InternalTest.DiscoverEarlyFlakeRetryArgs(original_test_id, item_id.retry_number),), + ) + + @staticmethod + @_catch_and_log_exceptions + def mark_unskippable(): + log.debug("Marking test as unskippable") diff --git a/tests/ci_visibility/api/fake_runner_all_fail.py b/tests/ci_visibility/api/fake_runner_all_fail.py index c128a546dcf..685fcb6fd8c 100644 --- a/tests/ci_visibility/api/fake_runner_all_fail.py +++ b/tests/ci_visibility/api/fake_runner_all_fail.py @@ -31,9 +31,6 @@ def main(): suite_1_test_1_id = api.TestId(suite_1_id, "test_1") suite_1_test_2_id = api.TestId(suite_1_id, "test_2") suite_1_test_3_id = api.TestId(suite_1_id, "test_3") - suite_1_test_3_retry_1_id = api.TestId(suite_1_id, "test_3", retry_number=1) - suite_1_test_3_retry_2_id = api.TestId(suite_1_id, "test_3", retry_number=2) - suite_1_test_3_retry_3_id = api.TestId(suite_1_id, "test_3", retry_number=3) api.Test.discover(suite_1_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) api.Test.discover(suite_1_test_2_id, source_file_info=None) @@ -43,9 +40,6 @@ def main(): source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_3_id) module_2_id = api.TestModuleId("module_2") suite_2_id = api.TestSuiteId(module_2_id, "suite_2") @@ -53,10 +47,6 @@ def main(): suite_2_test_2_id = api.TestId(suite_2_id, "test_2") suite_2_test_3_id = api.TestId(suite_2_id, "test_3") - suite_2_test_3_retry_1_id = api.TestId(suite_2_id, "test_3", retry_number=1) - suite_2_test_3_retry_2_id = api.TestId(suite_2_id, "test_3", retry_number=2) - suite_2_test_3_retry_3_id = api.TestId(suite_2_id, "test_3", retry_number=3) - api.TestModule.discover(module_2_id) api.TestSuite.discover(suite_2_id) api.Test.discover(suite_2_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) @@ -67,9 +57,6 @@ def main(): source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_3_id) # END DISCOVERY @@ -85,12 +72,6 @@ def main(): api.Test.finish(suite_1_test_2_id, api.TestStatus.FAIL, exc_info=_make_excinfo()) api.Test.start(suite_1_test_3_id) api.Test.finish(suite_1_test_3_id, api.TestStatus.FAIL, exc_info=_make_excinfo()) - api.Test.start(suite_1_test_3_retry_1_id) - api.Test.finish(suite_1_test_3_retry_1_id, api.TestStatus.FAIL, exc_info=_make_excinfo()) - api.Test.start(suite_1_test_3_retry_2_id) - api.Test.finish(suite_1_test_3_retry_2_id, api.TestStatus.FAIL, exc_info=_make_excinfo()) - api.Test.start(suite_1_test_3_retry_3_id) - api.Test.finish(suite_1_test_3_retry_3_id, api.TestStatus.FAIL, exc_info=_make_excinfo()) api.TestSuite.finish(suite_1_id) @@ -106,12 +87,6 @@ def main(): api.Test.mark_fail(suite_2_test_2_id, exc_info=_make_excinfo()) api.Test.start(suite_2_test_3_id) api.Test.mark_fail(suite_2_test_3_id, exc_info=_make_excinfo()) - api.Test.start(suite_2_test_3_retry_1_id) - api.Test.mark_fail(suite_2_test_3_retry_1_id, exc_info=_make_excinfo()) - api.Test.start(suite_2_test_3_retry_2_id) - api.Test.mark_fail(suite_2_test_3_retry_2_id, exc_info=_make_excinfo()) - api.Test.start(suite_2_test_3_retry_3_id) - api.Test.mark_fail(suite_2_test_3_retry_3_id, exc_info=_make_excinfo()) api.TestSuite.finish(suite_2_id) diff --git a/tests/ci_visibility/api/fake_runner_all_itr_skip_suite_level.py b/tests/ci_visibility/api/fake_runner_all_itr_skip_suite_level.py index 2c365965469..7e8945953f8 100644 --- a/tests/ci_visibility/api/fake_runner_all_itr_skip_suite_level.py +++ b/tests/ci_visibility/api/fake_runner_all_itr_skip_suite_level.py @@ -4,113 +4,118 @@ from pathlib import Path from unittest import mock -from ddtrace.ext.test_visibility import api +from ddtrace.ext.test_visibility import api as ext_api +from ddtrace.internal.test_visibility import api def main(): - api.enable_test_visibility() + ext_api.enable_test_visibility() # START DISCOVERY - api.TestSession.discover("manual_test_all_itr_skip_suite_level", "dd_manual_test_fw", "1.0.0") + api.InternalTestSession.discover("manual_test_all_itr_skip_suite_level", "dd_manual_test_fw", "1.0.0") - module_1_id = api.TestModuleId("module_1") + module_1_id = ext_api.TestModuleId("module_1") - api.TestModule.discover(module_1_id) + api.InternalTestModule.discover(module_1_id) - suite_1_id = api.TestSuiteId(module_1_id, "suite_1") - api.TestSuite.discover(suite_1_id) + suite_1_id = ext_api.TestSuiteId(module_1_id, "suite_1") + api.InternalTestSuite.discover(suite_1_id) - suite_1_test_1_id = api.TestId(suite_1_id, "test_1") - suite_1_test_2_id = api.TestId(suite_1_id, "test_2") - suite_1_test_3_id = api.TestId(suite_1_id, "test_3") - suite_1_test_3_retry_1_id = api.TestId(suite_1_id, "test_3", retry_number=1) - suite_1_test_3_retry_2_id = api.TestId(suite_1_id, "test_3", retry_number=2) - suite_1_test_3_retry_3_id = api.TestId(suite_1_id, "test_3", retry_number=3) + suite_1_test_1_id = api.InternalTestId(suite_1_id, "test_1") + suite_1_test_2_id = api.InternalTestId(suite_1_id, "test_2") + suite_1_test_3_id = api.InternalTestId(suite_1_id, "test_3") + suite_1_test_3_retry_1_id = api.InternalTestId(suite_1_id, "test_3", retry_number=1) + suite_1_test_3_retry_2_id = api.InternalTestId(suite_1_id, "test_3", retry_number=2) + suite_1_test_3_retry_3_id = api.InternalTestId(suite_1_id, "test_3", retry_number=3) - api.Test.discover(suite_1_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) - api.Test.discover(suite_1_test_2_id, source_file_info=None) - api.Test.discover( + api.InternalTest.discover( + suite_1_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2) + ) + api.InternalTest.discover(suite_1_test_2_id, source_file_info=None) + api.InternalTest.discover( suite_1_test_3_id, codeowners=["@romain", "@romain2"], - source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), + source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_3_id) - - module_2_id = api.TestModuleId("module_2") - suite_2_id = api.TestSuiteId(module_2_id, "suite_2") - suite_2_test_1_id = api.TestId(suite_2_id, "test_1") - suite_2_test_2_id = api.TestId(suite_2_id, "test_2") - suite_2_test_3_id = api.TestId(suite_2_id, "test_3") - - suite_2_test_3_retry_1_id = api.TestId(suite_2_id, "test_3", retry_number=1) - suite_2_test_3_retry_2_id = api.TestId(suite_2_id, "test_3", retry_number=2) - suite_2_test_3_retry_3_id = api.TestId(suite_2_id, "test_3", retry_number=3) - - api.TestModule.discover(module_2_id) - api.TestSuite.discover(suite_2_id) - api.Test.discover(suite_2_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) - api.Test.discover(suite_2_test_2_id, source_file_info=None) - api.Test.discover( + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_1_id) + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_2_id) + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_3_id) + + module_2_id = ext_api.TestModuleId("module_2") + suite_2_id = ext_api.TestSuiteId(module_2_id, "suite_2") + suite_2_test_1_id = api.InternalTestId(suite_2_id, "test_1") + suite_2_test_2_id = api.InternalTestId(suite_2_id, "test_2") + suite_2_test_3_id = api.InternalTestId(suite_2_id, "test_3") + + suite_2_test_3_retry_1_id = api.InternalTestId(suite_2_id, "test_3", retry_number=1) + suite_2_test_3_retry_2_id = api.InternalTestId(suite_2_id, "test_3", retry_number=2) + suite_2_test_3_retry_3_id = api.InternalTestId(suite_2_id, "test_3", retry_number=3) + + api.InternalTestModule.discover(module_2_id) + api.InternalTestSuite.discover(suite_2_id) + api.InternalTest.discover( + suite_2_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2) + ) + api.InternalTest.discover(suite_2_test_2_id, source_file_info=None) + api.InternalTest.discover( suite_2_test_3_id, codeowners=["@romain"], - source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), + source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_3_id) + api.InternalTest.discover_early_flake_retry(suite_2_test_3_retry_1_id) + api.InternalTest.discover_early_flake_retry(suite_2_test_3_retry_2_id) + api.InternalTest.discover_early_flake_retry(suite_2_test_3_retry_3_id) # END DISCOVERY - api.TestSession.start() + api.InternalTestSession.start() - api.TestModule.start(module_1_id) + api.InternalTestModule.start(module_1_id) - api.TestSuite.start(suite_1_id) + api.InternalTestSuite.start(suite_1_id) - api.Test.start(suite_1_test_1_id) - api.Test.mark_itr_skipped(suite_1_test_1_id) - api.Test.start(suite_1_test_2_id) - api.Test.mark_itr_skipped(suite_1_test_2_id) - api.Test.start(suite_1_test_3_id) - api.Test.mark_itr_skipped(suite_1_test_3_id) - api.Test.start(suite_1_test_3_retry_1_id) - api.Test.mark_itr_skipped(suite_1_test_3_retry_1_id) - api.Test.start(suite_1_test_3_retry_2_id) - api.Test.mark_itr_skipped(suite_1_test_3_retry_2_id) - api.Test.start(suite_1_test_3_retry_3_id) - api.Test.mark_itr_skipped(suite_1_test_3_retry_3_id) + api.InternalTest.start(suite_1_test_1_id) + api.InternalTest.mark_itr_skipped(suite_1_test_1_id) + api.InternalTest.start(suite_1_test_2_id) + api.InternalTest.mark_itr_skipped(suite_1_test_2_id) + api.InternalTest.start(suite_1_test_3_id) + api.InternalTest.mark_itr_skipped(suite_1_test_3_id) + api.InternalTest.start(suite_1_test_3_retry_1_id) + api.InternalTest.mark_itr_skipped(suite_1_test_3_retry_1_id) + api.InternalTest.start(suite_1_test_3_retry_2_id) + api.InternalTest.mark_itr_skipped(suite_1_test_3_retry_2_id) + api.InternalTest.start(suite_1_test_3_retry_3_id) + api.InternalTest.mark_itr_skipped(suite_1_test_3_retry_3_id) - api.TestSuite.mark_itr_skipped(suite_1_id) + api.InternalTestSuite.mark_itr_skipped(suite_1_id) - api.TestModule.finish(module_1_id) + api.InternalTestModule.finish(module_1_id) - api.TestModule.start(module_2_id) + api.InternalTestModule.start(module_2_id) - api.TestSuite.start(suite_2_id) + api.InternalTestSuite.start(suite_2_id) - api.Test.start(suite_2_test_1_id) - api.Test.mark_itr_skipped(suite_2_test_1_id) - api.Test.start(suite_2_test_2_id) - api.Test.mark_itr_skipped(suite_2_test_2_id) - api.Test.start(suite_2_test_3_id) - api.Test.mark_itr_skipped(suite_2_test_3_id) - api.Test.start(suite_2_test_3_retry_1_id) - api.Test.mark_itr_skipped(suite_2_test_3_retry_1_id) - api.Test.start(suite_2_test_3_retry_2_id) - api.Test.mark_itr_skipped(suite_2_test_3_retry_2_id) - api.Test.start(suite_2_test_3_retry_3_id) - api.Test.mark_itr_skipped(suite_2_test_3_retry_3_id) + api.InternalTest.start(suite_2_test_1_id) + api.InternalTest.mark_itr_skipped(suite_2_test_1_id) + api.InternalTest.start(suite_2_test_2_id) + api.InternalTest.mark_itr_skipped(suite_2_test_2_id) + api.InternalTest.start(suite_2_test_3_id) + api.InternalTest.mark_itr_skipped(suite_2_test_3_id) + api.InternalTest.start(suite_2_test_3_retry_1_id) + api.InternalTest.mark_itr_skipped(suite_2_test_3_retry_1_id) + api.InternalTest.start(suite_2_test_3_retry_2_id) + api.InternalTest.mark_itr_skipped(suite_2_test_3_retry_2_id) + api.InternalTest.start(suite_2_test_3_retry_3_id) + api.InternalTest.mark_itr_skipped(suite_2_test_3_retry_3_id) - api.TestSuite.mark_itr_skipped(suite_2_id) + api.InternalTestSuite.mark_itr_skipped(suite_2_id) - api.TestModule.finish(module_2_id) + api.InternalTestModule.finish(module_2_id) - api.TestSession.finish() + api.InternalTestSession.finish() if __name__ == "__main__": diff --git a/tests/ci_visibility/api/fake_runner_all_itr_skip_test_level.py b/tests/ci_visibility/api/fake_runner_all_itr_skip_test_level.py index 44f5228d482..c93879134d4 100644 --- a/tests/ci_visibility/api/fake_runner_all_itr_skip_test_level.py +++ b/tests/ci_visibility/api/fake_runner_all_itr_skip_test_level.py @@ -4,113 +4,118 @@ from pathlib import Path from unittest import mock -from ddtrace.ext.test_visibility import api +from ddtrace.ext.test_visibility import api as ext_api +from ddtrace.internal.test_visibility import api def main(): - api.enable_test_visibility() + ext_api.enable_test_visibility() # START DISCOVERY - api.TestSession.discover("manual_test_all_itr_skip", "dd_manual_test_fw", "1.0.0") + api.InternalTestSession.discover("manual_test_all_itr_skip", "dd_manual_test_fw", "1.0.0") - module_1_id = api.TestModuleId("module_1") + module_1_id = ext_api.TestModuleId("module_1") - api.TestModule.discover(module_1_id) + api.InternalTestModule.discover(module_1_id) - suite_1_id = api.TestSuiteId(module_1_id, "suite_1") - api.TestSuite.discover(suite_1_id) + suite_1_id = ext_api.TestSuiteId(module_1_id, "suite_1") + api.InternalTestSuite.discover(suite_1_id) - suite_1_test_1_id = api.TestId(suite_1_id, "test_1") - suite_1_test_2_id = api.TestId(suite_1_id, "test_2") - suite_1_test_3_id = api.TestId(suite_1_id, "test_3") - suite_1_test_3_retry_1_id = api.TestId(suite_1_id, "test_3", retry_number=1) - suite_1_test_3_retry_2_id = api.TestId(suite_1_id, "test_3", retry_number=2) - suite_1_test_3_retry_3_id = api.TestId(suite_1_id, "test_3", retry_number=3) + suite_1_test_1_id = api.InternalTestId(suite_1_id, "test_1") + suite_1_test_2_id = api.InternalTestId(suite_1_id, "test_2") + suite_1_test_3_id = api.InternalTestId(suite_1_id, "test_3") + suite_1_test_3_retry_1_id = api.InternalTestId(suite_1_id, "test_3", retry_number=1) + suite_1_test_3_retry_2_id = api.InternalTestId(suite_1_id, "test_3", retry_number=2) + suite_1_test_3_retry_3_id = api.InternalTestId(suite_1_id, "test_3", retry_number=3) - api.Test.discover(suite_1_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) - api.Test.discover(suite_1_test_2_id, source_file_info=None) - api.Test.discover( + api.InternalTest.discover( + suite_1_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2) + ) + api.InternalTest.discover(suite_1_test_2_id, source_file_info=None) + api.InternalTest.discover( suite_1_test_3_id, codeowners=["@romain", "@romain2"], - source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), + source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_3_id) - - module_2_id = api.TestModuleId("module_2") - suite_2_id = api.TestSuiteId(module_2_id, "suite_2") - suite_2_test_1_id = api.TestId(suite_2_id, "test_1") - suite_2_test_2_id = api.TestId(suite_2_id, "test_2") - suite_2_test_3_id = api.TestId(suite_2_id, "test_3") - - suite_2_test_3_retry_1_id = api.TestId(suite_2_id, "test_3", retry_number=1) - suite_2_test_3_retry_2_id = api.TestId(suite_2_id, "test_3", retry_number=2) - suite_2_test_3_retry_3_id = api.TestId(suite_2_id, "test_3", retry_number=3) - - api.TestModule.discover(module_2_id) - api.TestSuite.discover(suite_2_id) - api.Test.discover(suite_2_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) - api.Test.discover(suite_2_test_2_id, source_file_info=None) - api.Test.discover( + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_1_id) + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_2_id) + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_3_id) + + module_2_id = ext_api.TestModuleId("module_2") + suite_2_id = ext_api.TestSuiteId(module_2_id, "suite_2") + suite_2_test_1_id = api.InternalTestId(suite_2_id, "test_1") + suite_2_test_2_id = api.InternalTestId(suite_2_id, "test_2") + suite_2_test_3_id = api.InternalTestId(suite_2_id, "test_3") + + suite_2_test_3_retry_1_id = api.InternalTestId(suite_2_id, "test_3", retry_number=1) + suite_2_test_3_retry_2_id = api.InternalTestId(suite_2_id, "test_3", retry_number=2) + suite_2_test_3_retry_3_id = api.InternalTestId(suite_2_id, "test_3", retry_number=3) + + api.InternalTestModule.discover(module_2_id) + api.InternalTestSuite.discover(suite_2_id) + api.InternalTest.discover( + suite_2_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2) + ) + api.InternalTest.discover(suite_2_test_2_id, source_file_info=None) + api.InternalTest.discover( suite_2_test_3_id, codeowners=["@romain"], - source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), + source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_3_id) + api.InternalTest.discover_early_flake_retry(suite_2_test_3_retry_1_id) + api.InternalTest.discover_early_flake_retry(suite_2_test_3_retry_2_id) + api.InternalTest.discover_early_flake_retry(suite_2_test_3_retry_3_id) # END DISCOVERY - api.TestSession.start() + api.InternalTestSession.start() - api.TestModule.start(module_1_id) + api.InternalTestModule.start(module_1_id) - api.TestSuite.start(suite_1_id) + api.InternalTestSuite.start(suite_1_id) - api.Test.start(suite_1_test_1_id) - api.Test.mark_itr_skipped(suite_1_test_1_id) - api.Test.start(suite_1_test_2_id) - api.Test.mark_itr_skipped(suite_1_test_2_id) - api.Test.start(suite_1_test_3_id) - api.Test.mark_itr_skipped(suite_1_test_3_id) - api.Test.start(suite_1_test_3_retry_1_id) - api.Test.mark_itr_skipped(suite_1_test_3_retry_1_id) - api.Test.start(suite_1_test_3_retry_2_id) - api.Test.mark_itr_skipped(suite_1_test_3_retry_2_id) - api.Test.start(suite_1_test_3_retry_3_id) - api.Test.mark_itr_skipped(suite_1_test_3_retry_3_id) + api.InternalTest.start(suite_1_test_1_id) + api.InternalTest.mark_itr_skipped(suite_1_test_1_id) + api.InternalTest.start(suite_1_test_2_id) + api.InternalTest.mark_itr_skipped(suite_1_test_2_id) + api.InternalTest.start(suite_1_test_3_id) + api.InternalTest.mark_itr_skipped(suite_1_test_3_id) + api.InternalTest.start(suite_1_test_3_retry_1_id) + api.InternalTest.mark_itr_skipped(suite_1_test_3_retry_1_id) + api.InternalTest.start(suite_1_test_3_retry_2_id) + api.InternalTest.mark_itr_skipped(suite_1_test_3_retry_2_id) + api.InternalTest.start(suite_1_test_3_retry_3_id) + api.InternalTest.mark_itr_skipped(suite_1_test_3_retry_3_id) - api.TestSuite.finish(suite_1_id) + api.InternalTestSuite.finish(suite_1_id) - api.TestModule.finish(module_1_id) + api.InternalTestModule.finish(module_1_id) - api.TestModule.start(module_2_id) + api.InternalTestModule.start(module_2_id) - api.TestSuite.start(suite_2_id) + api.InternalTestSuite.start(suite_2_id) - api.Test.start(suite_2_test_1_id) - api.Test.mark_itr_skipped(suite_2_test_1_id) - api.Test.start(suite_2_test_2_id) - api.Test.mark_itr_skipped(suite_2_test_2_id) - api.Test.start(suite_2_test_3_id) - api.Test.mark_itr_skipped(suite_2_test_3_id) - api.Test.start(suite_2_test_3_retry_1_id) - api.Test.mark_itr_skipped(suite_2_test_3_retry_1_id) - api.Test.start(suite_2_test_3_retry_2_id) - api.Test.mark_itr_skipped(suite_2_test_3_retry_2_id) - api.Test.start(suite_2_test_3_retry_3_id) - api.Test.mark_itr_skipped(suite_2_test_3_retry_3_id) + api.InternalTest.start(suite_2_test_1_id) + api.InternalTest.mark_itr_skipped(suite_2_test_1_id) + api.InternalTest.start(suite_2_test_2_id) + api.InternalTest.mark_itr_skipped(suite_2_test_2_id) + api.InternalTest.start(suite_2_test_3_id) + api.InternalTest.mark_itr_skipped(suite_2_test_3_id) + api.InternalTest.start(suite_2_test_3_retry_1_id) + api.InternalTest.mark_itr_skipped(suite_2_test_3_retry_1_id) + api.InternalTest.start(suite_2_test_3_retry_2_id) + api.InternalTest.mark_itr_skipped(suite_2_test_3_retry_2_id) + api.InternalTest.start(suite_2_test_3_retry_3_id) + api.InternalTest.mark_itr_skipped(suite_2_test_3_retry_3_id) - api.TestSuite.finish(suite_2_id) + api.InternalTestSuite.finish(suite_2_id) - api.TestModule.finish(module_2_id) + api.InternalTestModule.finish(module_2_id) - api.TestSession.finish() + api.InternalTestSession.finish() if __name__ == "__main__": diff --git a/tests/ci_visibility/api/fake_runner_all_pass.py b/tests/ci_visibility/api/fake_runner_all_pass.py index 0a1a810f1c3..af1eebfc9c1 100644 --- a/tests/ci_visibility/api/fake_runner_all_pass.py +++ b/tests/ci_visibility/api/fake_runner_all_pass.py @@ -23,9 +23,6 @@ def main(): suite_1_test_1_id = api.TestId(suite_1_id, "test_1") suite_1_test_2_id = api.TestId(suite_1_id, "test_2") suite_1_test_3_id = api.TestId(suite_1_id, "test_3") - suite_1_test_3_retry_1_id = api.TestId(suite_1_id, "test_3", retry_number=1) - suite_1_test_3_retry_2_id = api.TestId(suite_1_id, "test_3", retry_number=2) - suite_1_test_3_retry_3_id = api.TestId(suite_1_id, "test_3", retry_number=3) api.Test.discover(suite_1_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) api.Test.discover(suite_1_test_2_id, source_file_info=None) @@ -35,9 +32,6 @@ def main(): source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_3_id) module_2_id = api.TestModuleId("module_2") suite_2_id = api.TestSuiteId(module_2_id, "suite_2") @@ -45,10 +39,6 @@ def main(): suite_2_test_2_id = api.TestId(suite_2_id, "test_2") suite_2_test_3_id = api.TestId(suite_2_id, "test_3") - suite_2_test_3_retry_1_id = api.TestId(suite_2_id, "test_3", retry_number=1) - suite_2_test_3_retry_2_id = api.TestId(suite_2_id, "test_3", retry_number=2) - suite_2_test_3_retry_3_id = api.TestId(suite_2_id, "test_3", retry_number=3) - api.TestModule.discover(module_2_id) api.TestSuite.discover(suite_2_id) api.Test.discover(suite_2_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) @@ -59,9 +49,6 @@ def main(): source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_3_id) # END DISCOVERY @@ -77,12 +64,6 @@ def main(): api.Test.finish(suite_1_test_2_id, api.TestStatus.PASS) api.Test.start(suite_1_test_3_id) api.Test.finish(suite_1_test_3_id, api.TestStatus.PASS) - api.Test.start(suite_1_test_3_retry_1_id) - api.Test.finish(suite_1_test_3_retry_1_id, api.TestStatus.PASS) - api.Test.start(suite_1_test_3_retry_2_id) - api.Test.finish(suite_1_test_3_retry_2_id, api.TestStatus.PASS) - api.Test.start(suite_1_test_3_retry_3_id) - api.Test.finish(suite_1_test_3_retry_3_id, api.TestStatus.PASS) api.TestSuite.finish(suite_1_id) @@ -98,12 +79,6 @@ def main(): api.Test.mark_pass(suite_2_test_2_id) api.Test.start(suite_2_test_3_id) api.Test.mark_pass(suite_2_test_3_id) - api.Test.start(suite_2_test_3_retry_1_id) - api.Test.mark_pass(suite_2_test_3_retry_1_id) - api.Test.start(suite_2_test_3_retry_2_id) - api.Test.mark_pass(suite_2_test_3_retry_2_id) - api.Test.start(suite_2_test_3_retry_3_id) - api.Test.mark_pass(suite_2_test_3_retry_3_id) api.TestSuite.finish(suite_2_id) diff --git a/tests/ci_visibility/api/fake_runner_all_skip.py b/tests/ci_visibility/api/fake_runner_all_skip.py index a99294b2d5f..9f31e953246 100644 --- a/tests/ci_visibility/api/fake_runner_all_skip.py +++ b/tests/ci_visibility/api/fake_runner_all_skip.py @@ -23,9 +23,6 @@ def main(): suite_1_test_1_id = api.TestId(suite_1_id, "test_1") suite_1_test_2_id = api.TestId(suite_1_id, "test_2") suite_1_test_3_id = api.TestId(suite_1_id, "test_3") - suite_1_test_3_retry_1_id = api.TestId(suite_1_id, "test_3", retry_number=1) - suite_1_test_3_retry_2_id = api.TestId(suite_1_id, "test_3", retry_number=2) - suite_1_test_3_retry_3_id = api.TestId(suite_1_id, "test_3", retry_number=3) api.Test.discover(suite_1_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) api.Test.discover(suite_1_test_2_id, source_file_info=None) @@ -35,9 +32,6 @@ def main(): source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_3_id) module_2_id = api.TestModuleId("module_2") suite_2_id = api.TestSuiteId(module_2_id, "suite_2") @@ -45,10 +39,6 @@ def main(): suite_2_test_2_id = api.TestId(suite_2_id, "test_2") suite_2_test_3_id = api.TestId(suite_2_id, "test_3") - suite_2_test_3_retry_1_id = api.TestId(suite_2_id, "test_3", retry_number=1) - suite_2_test_3_retry_2_id = api.TestId(suite_2_id, "test_3", retry_number=2) - suite_2_test_3_retry_3_id = api.TestId(suite_2_id, "test_3", retry_number=3) - api.TestModule.discover(module_2_id) api.TestSuite.discover(suite_2_id) api.Test.discover(suite_2_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) @@ -59,9 +49,6 @@ def main(): source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_2_test_3_retry_3_id) # END DISCOVERY @@ -77,24 +64,6 @@ def main(): api.Test.finish(suite_1_test_2_id, api.TestStatus.SKIP, skip_reason="This test was skipped") api.Test.start(suite_1_test_3_id) api.Test.finish(suite_1_test_3_id, api.TestStatus.SKIP, skip_reason="This test was skipped") - api.Test.start(suite_1_test_3_retry_1_id) - api.Test.finish( - suite_1_test_3_retry_1_id, - api.TestStatus.SKIP, - skip_reason="This test was skipped", - ) - api.Test.start(suite_1_test_3_retry_2_id) - api.Test.finish( - suite_1_test_3_retry_2_id, - api.TestStatus.SKIP, - skip_reason="This test was skipped", - ) - api.Test.start(suite_1_test_3_retry_3_id) - api.Test.finish( - suite_1_test_3_retry_3_id, - api.TestStatus.SKIP, - skip_reason="This test was skipped", - ) api.TestSuite.finish(suite_1_id) @@ -110,12 +79,6 @@ def main(): api.Test.mark_skip(suite_2_test_2_id, skip_reason="This test was skipped with mark_skip") api.Test.start(suite_2_test_3_id) api.Test.mark_skip(suite_2_test_3_id, skip_reason="This test was skipped with mark_skip") - api.Test.start(suite_2_test_3_retry_1_id) - api.Test.mark_skip(suite_2_test_3_retry_1_id, skip_reason="This test was skipped with mark_skip") - api.Test.start(suite_2_test_3_retry_2_id) - api.Test.mark_skip(suite_2_test_3_retry_2_id, skip_reason="This test was skipped with mark_skip") - api.Test.start(suite_2_test_3_retry_3_id) - api.Test.mark_skip(suite_2_test_3_retry_3_id, skip_reason="This test was skipped with mark_skip") api.TestSuite.finish(suite_2_id) diff --git a/tests/ci_visibility/api/fake_runner_mix_fail.py b/tests/ci_visibility/api/fake_runner_mix_fail.py index b96ed91185c..71fb3e028d2 100644 --- a/tests/ci_visibility/api/fake_runner_mix_fail.py +++ b/tests/ci_visibility/api/fake_runner_mix_fail.py @@ -42,9 +42,6 @@ def main(): suite_1_test_1_id = api.TestId(suite_1_id, "test_1") suite_1_test_2_id = api.TestId(suite_1_id, "test_2") suite_1_test_3_id = api.TestId(suite_1_id, "test_3") - suite_1_test_3_retry_1_id = api.TestId(suite_1_id, "test_3", retry_number=1) - suite_1_test_3_retry_2_id = api.TestId(suite_1_id, "test_3", retry_number=2) - suite_1_test_3_retry_3_id = api.TestId(suite_1_id, "test_3", retry_number=3) suite_1_test_4_parametrized_1_id = api.TestId(suite_1_id, "test_4", parameters=json.dumps({"param1": "value1"})) suite_1_test_4_parametrized_2_id = api.TestId(suite_1_id, "test_4", parameters=json.dumps({"param1": "value2"})) @@ -58,9 +55,6 @@ def main(): source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_3_id) api.Test.discover(suite_1_test_4_parametrized_1_id) api.Test.discover(suite_1_test_4_parametrized_2_id) @@ -140,16 +134,8 @@ def main(): # # suite_1_test_3 test and EFD retries api.Test.start(suite_1_test_3_id) - api.Test.mark_itr_skipped(suite_1_test_3_id) + api.Test.mark_skip(suite_1_test_3_id) # - api.Test.start(suite_1_test_3_retry_1_id) - api.Test.mark_pass(suite_1_test_3_retry_1_id) - # - api.Test.start(suite_1_test_3_retry_2_id) - api.Test.mark_pass(suite_1_test_3_retry_2_id) - # - api.Test.start(suite_1_test_3_retry_3_id) - api.Test.mark_pass(suite_1_test_3_retry_3_id) # # suite1_test_4 parametrized tests @@ -208,7 +194,7 @@ def main(): # api.Test.start(suite_2_test_2_parametrized_3_id) - api.Test.mark_itr_skipped(suite_2_test_2_parametrized_3_id) + api.Test.mark_skip(suite_2_test_2_parametrized_3_id) # api.Test.start(suite_2_test_2_parametrized_4_id) api.Test.mark_pass(suite_2_test_2_parametrized_4_id) diff --git a/tests/ci_visibility/api/fake_runner_mix_fail_itr_suite_level.py b/tests/ci_visibility/api/fake_runner_mix_fail_itr_suite_level.py index 1a8bcd2c94b..1279a69ff0c 100644 --- a/tests/ci_visibility/api/fake_runner_mix_fail_itr_suite_level.py +++ b/tests/ci_visibility/api/fake_runner_mix_fail_itr_suite_level.py @@ -22,180 +22,223 @@ import sys from unittest import mock -from ddtrace.ext.test_visibility import api +from ddtrace.ext.test_visibility import api as ext_api from ddtrace.ext.test_visibility.coverage_lines import CoverageLines +from ddtrace.internal.test_visibility import api def _make_excinfo(): try: raise ValueError("This is a fake exception") except ValueError: - return api.TestExcInfo(*sys.exc_info()) + return ext_api.TestExcInfo(*sys.exc_info()) def main(): - api.enable_test_visibility() + ext_api.enable_test_visibility() # START DISCOVERY - api.TestSession.discover("manual_test_mix_fail_itr_suite_level", "dd_manual_test_fw", "1.0.0") - api.TestSession.start() + api.InternalTestSession.discover("manual_test_mix_fail_itr_suite_level", "dd_manual_test_fw", "1.0.0") + api.InternalTestSession.start() - module_1_id = api.TestModuleId("module_1") + module_1_id = ext_api.TestModuleId("module_1") - api.TestModule.discover(module_1_id) + api.InternalTestModule.discover(module_1_id) - suite_1_id = api.TestSuiteId(module_1_id, "suite_1") - api.TestSuite.discover(suite_1_id) + suite_1_id = ext_api.TestSuiteId(module_1_id, "suite_1") + api.InternalTestSuite.discover(suite_1_id) - suite_1_test_1_id = api.TestId(suite_1_id, "test_1") - suite_1_test_2_id = api.TestId(suite_1_id, "test_2") - suite_1_test_3_id = api.TestId(suite_1_id, "test_3") - suite_1_test_3_retry_1_id = api.TestId(suite_1_id, "test_3", retry_number=1) - suite_1_test_3_retry_2_id = api.TestId(suite_1_id, "test_3", retry_number=2) - suite_1_test_3_retry_3_id = api.TestId(suite_1_id, "test_3", retry_number=3) + suite_1_test_1_id = api.InternalTestId(suite_1_id, "test_1") + suite_1_test_2_id = api.InternalTestId(suite_1_id, "test_2") + suite_1_test_3_id = api.InternalTestId(suite_1_id, "test_3") + suite_1_test_3_retry_1_id = api.InternalTestId(suite_1_id, "test_3", retry_number=1) + suite_1_test_3_retry_2_id = api.InternalTestId(suite_1_id, "test_3", retry_number=2) + suite_1_test_3_retry_3_id = api.InternalTestId(suite_1_id, "test_3", retry_number=3) - suite_1_test_4_parametrized_1_id = api.TestId(suite_1_id, "test_4", parameters=json.dumps({"param1": "value1"})) - suite_1_test_4_parametrized_2_id = api.TestId(suite_1_id, "test_4", parameters=json.dumps({"param1": "value2"})) - suite_1_test_4_parametrized_3_id = api.TestId(suite_1_id, "test_4", parameters=json.dumps({"param1": "value3"})) + suite_1_test_4_parametrized_1_id = api.InternalTestId( + suite_1_id, "test_4", parameters=json.dumps({"param1": "value1"}) + ) + suite_1_test_4_parametrized_2_id = api.InternalTestId( + suite_1_id, "test_4", parameters=json.dumps({"param1": "value2"}) + ) + suite_1_test_4_parametrized_3_id = api.InternalTestId( + suite_1_id, "test_4", parameters=json.dumps({"param1": "value3"}) + ) - api.Test.discover(suite_1_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) - api.Test.discover(suite_1_test_2_id, source_file_info=None) - api.Test.discover( + api.InternalTest.discover( + suite_1_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2) + ) + api.InternalTest.discover(suite_1_test_2_id, source_file_info=None) + api.InternalTest.discover( suite_1_test_3_id, codeowners=["@romain", "@romain2"], - source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), + source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_3_id) - - api.Test.discover(suite_1_test_4_parametrized_1_id) - api.Test.discover(suite_1_test_4_parametrized_2_id) - api.Test.discover(suite_1_test_4_parametrized_3_id) - - module_2_id = api.TestModuleId("module_2") - suite_2_id = api.TestSuiteId(module_2_id, "suite_2") - suite_2_test_1_id = api.TestId(suite_2_id, "test_1") - suite_2_test_2_parametrized_1_id = api.TestId(suite_2_id, "test_2", parameters=json.dumps({"param1": "value1"})) - suite_2_test_2_parametrized_2_id = api.TestId(suite_2_id, "test_2", parameters=json.dumps({"param1": "value2"})) - suite_2_test_2_parametrized_3_id = api.TestId(suite_2_id, "test_2", parameters=json.dumps({"param1": "value3"})) - suite_2_test_2_parametrized_4_id = api.TestId(suite_2_id, "test_2", parameters=json.dumps({"param1": "value4"})) - suite_2_test_2_parametrized_5_id = api.TestId(suite_2_id, "test_2", parameters=json.dumps({"param1": "value5"})) - suite_2_test_2_source_file_info = api.TestSourceFileInfo(Path("test_file_2.py"), 8, 9) - suite_2_test_3_id = api.TestId(suite_2_id, "test_3") - - api.TestModule.discover(module_2_id) - api.TestSuite.discover(suite_2_id) - api.Test.discover(suite_2_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) - api.Test.discover(suite_2_test_2_parametrized_1_id, source_file_info=suite_2_test_2_source_file_info) - api.Test.discover(suite_2_test_2_parametrized_2_id, source_file_info=suite_2_test_2_source_file_info) - api.Test.discover(suite_2_test_2_parametrized_3_id, source_file_info=suite_2_test_2_source_file_info) - api.Test.discover(suite_2_test_2_parametrized_4_id, source_file_info=suite_2_test_2_source_file_info) - api.Test.discover(suite_2_test_2_parametrized_5_id, source_file_info=suite_2_test_2_source_file_info) - - api.Test.discover( + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_1_id) + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_2_id) + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_3_id) + + api.InternalTest.discover(suite_1_test_4_parametrized_1_id) + api.InternalTest.discover(suite_1_test_4_parametrized_2_id) + api.InternalTest.discover(suite_1_test_4_parametrized_3_id) + + module_2_id = ext_api.TestModuleId("module_2") + suite_2_id = ext_api.TestSuiteId(module_2_id, "suite_2") + suite_2_test_1_id = api.InternalTestId(suite_2_id, "test_1") + suite_2_test_2_parametrized_1_id = api.InternalTestId( + suite_2_id, "test_2", parameters=json.dumps({"param1": "value1"}) + ) + suite_2_test_2_parametrized_2_id = api.InternalTestId( + suite_2_id, "test_2", parameters=json.dumps({"param1": "value2"}) + ) + suite_2_test_2_parametrized_3_id = api.InternalTestId( + suite_2_id, "test_2", parameters=json.dumps({"param1": "value3"}) + ) + suite_2_test_2_parametrized_4_id = api.InternalTestId( + suite_2_id, "test_2", parameters=json.dumps({"param1": "value4"}) + ) + suite_2_test_2_parametrized_5_id = api.InternalTestId( + suite_2_id, "test_2", parameters=json.dumps({"param1": "value5"}) + ) + suite_2_test_2_source_file_info = ext_api.TestSourceFileInfo(Path("test_file_2.py"), 8, 9) + suite_2_test_3_id = api.InternalTestId(suite_2_id, "test_3") + + api.InternalTestModule.discover(module_2_id) + api.InternalTestSuite.discover(suite_2_id) + api.InternalTest.discover( + suite_2_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2) + ) + api.InternalTest.discover(suite_2_test_2_parametrized_1_id, source_file_info=suite_2_test_2_source_file_info) + api.InternalTest.discover(suite_2_test_2_parametrized_2_id, source_file_info=suite_2_test_2_source_file_info) + api.InternalTest.discover(suite_2_test_2_parametrized_3_id, source_file_info=suite_2_test_2_source_file_info) + api.InternalTest.discover(suite_2_test_2_parametrized_4_id, source_file_info=suite_2_test_2_source_file_info) + api.InternalTest.discover(suite_2_test_2_parametrized_5_id, source_file_info=suite_2_test_2_source_file_info) + + api.InternalTest.discover( suite_2_test_3_id, codeowners=["@romain"], - source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), + source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - module_3_id = api.TestModuleId("module_3") - suite_3_id = api.TestSuiteId(module_3_id, "suite_3") - suite_4_id = api.TestSuiteId(module_3_id, "suite_4") + module_3_id = ext_api.TestModuleId("module_3") + suite_3_id = ext_api.TestSuiteId(module_3_id, "suite_3") + suite_4_id = ext_api.TestSuiteId(module_3_id, "suite_4") - suite_3_test_1_id = api.TestId(suite_3_id, "test_1") - suite_3_test_2_id = api.TestId(suite_3_id, "test_2") - suite_3_test_3_id = api.TestId(suite_3_id, "test_3") + suite_3_test_1_id = api.InternalTestId(suite_3_id, "test_1") + suite_3_test_2_id = api.InternalTestId(suite_3_id, "test_2") + suite_3_test_3_id = api.InternalTestId(suite_3_id, "test_3") - suite_4_test_1_id = api.TestId(suite_4_id, "test_1") - suite_4_test_2_id = api.TestId(suite_4_id, "test_2") - suite_4_test_3_id = api.TestId(suite_4_id, "test_3") + suite_4_test_1_id = api.InternalTestId(suite_4_id, "test_1") + suite_4_test_2_id = api.InternalTestId(suite_4_id, "test_2") + suite_4_test_3_id = api.InternalTestId(suite_4_id, "test_3") - api.TestModule.discover(module_3_id) + api.InternalTestModule.discover(module_3_id) - api.TestSuite.discover(suite_3_id) + api.InternalTestSuite.discover(suite_3_id) - api.Test.discover(suite_3_test_1_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_3.py"), 4, 6)) - api.Test.discover(suite_3_test_2_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_3.py"), 9, 12)) - api.Test.discover(suite_3_test_3_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_3.py"), 16, 48)) + api.InternalTest.discover( + suite_3_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_3.py"), 4, 6) + ) + api.InternalTest.discover( + suite_3_test_2_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_3.py"), 9, 12) + ) + api.InternalTest.discover( + suite_3_test_3_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_3.py"), 16, 48) + ) - api.TestSuite.discover(suite_4_id) - api.Test.discover(suite_4_test_1_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_4.py"), 4, 6)) - api.Test.discover(suite_4_test_2_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_4.py"), 9, 12)) - api.Test.discover(suite_4_test_3_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_4.py"), 16, 48)) + api.InternalTestSuite.discover(suite_4_id) + api.InternalTest.discover( + suite_4_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_4.py"), 4, 6) + ) + api.InternalTest.discover( + suite_4_test_2_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_4.py"), 9, 12) + ) + api.InternalTest.discover( + suite_4_test_3_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_4.py"), 16, 48) + ) - module_4_id = api.TestModuleId("module_4") - suite_5_id = api.TestSuiteId(module_4_id, "suite_5") - suite_6_id = api.TestSuiteId(module_4_id, "suite_6") + module_4_id = ext_api.TestModuleId("module_4") + suite_5_id = ext_api.TestSuiteId(module_4_id, "suite_5") + suite_6_id = ext_api.TestSuiteId(module_4_id, "suite_6") - suite_5_test_1_id = api.TestId(suite_5_id, "test_1") - suite_5_test_2_id = api.TestId(suite_5_id, "test_2") - suite_5_test_3_id = api.TestId(suite_5_id, "test_3") + suite_5_test_1_id = api.InternalTestId(suite_5_id, "test_1") + suite_5_test_2_id = api.InternalTestId(suite_5_id, "test_2") + suite_5_test_3_id = api.InternalTestId(suite_5_id, "test_3") - suite_6_test_1_id = api.TestId(suite_6_id, "test_1") - suite_6_test_2_id = api.TestId(suite_6_id, "test_2") - suite_6_test_3_id = api.TestId(suite_6_id, "test_3") + suite_6_test_1_id = api.InternalTestId(suite_6_id, "test_1") + suite_6_test_2_id = api.InternalTestId(suite_6_id, "test_2") + suite_6_test_3_id = api.InternalTestId(suite_6_id, "test_3") - api.TestModule.discover(module_4_id) + api.InternalTestModule.discover(module_4_id) - api.TestSuite.discover(suite_5_id) + api.InternalTestSuite.discover(suite_5_id) - api.Test.discover(suite_5_test_1_id, source_file_info=api.TestSourceFileInfo(Path("module_5/suite_5.py"), 4, 6)) - api.Test.discover(suite_5_test_2_id, source_file_info=api.TestSourceFileInfo(Path("module_5/suite_5.py"), 9, 12)) - api.Test.discover(suite_5_test_3_id, source_file_info=api.TestSourceFileInfo(Path("module_5/suite_5.py"), 16, 48)) + api.InternalTest.discover( + suite_5_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_5/suite_5.py"), 4, 6) + ) + api.InternalTest.discover( + suite_5_test_2_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_5/suite_5.py"), 9, 12) + ) + api.InternalTest.discover( + suite_5_test_3_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_5/suite_5.py"), 16, 48) + ) - api.TestSuite.discover(suite_6_id) - api.TestSuite.mark_itr_unskippable(suite_6_id) - api.Test.discover(suite_6_test_1_id) - api.Test.discover(suite_6_test_2_id) - api.Test.discover(suite_6_test_3_id) + api.InternalTestSuite.discover(suite_6_id) + api.InternalTestSuite.mark_itr_unskippable(suite_6_id) + api.InternalTest.discover(suite_6_test_1_id) + api.InternalTest.discover(suite_6_test_2_id) + api.InternalTest.discover(suite_6_test_3_id) # END DISCOVERY # START TESTS - api.TestModule.start(module_1_id) + api.InternalTestModule.start(module_1_id) - api.TestSuite.start(suite_1_id) + api.InternalTestSuite.start(suite_1_id) # # suite_1_test_1 test - api.Test.start(suite_1_test_1_id) - api.Test.add_coverage_data(suite_1_test_1_id, {Path("my_file_1.py"): CoverageLines.from_list([1, 2, 3, 4, 8, 9])}) - api.Test.mark_pass(suite_1_test_1_id) + api.InternalTest.start(suite_1_test_1_id) + api.InternalTest.add_coverage_data( + suite_1_test_1_id, {Path("my_file_1.py"): CoverageLines.from_list([1, 2, 3, 4, 8, 9])} + ) + api.InternalTest.mark_pass(suite_1_test_1_id) # # suite_1_test_2 test - api.Test.start(suite_1_test_2_id) - api.Test.set_tag(suite_1_test_2_id, "test.tag1", "suite_1_test_2_id") - api.Test.add_coverage_data( + api.InternalTest.start(suite_1_test_2_id) + api.InternalTest.set_tag(suite_1_test_2_id, "test.tag1", "suite_1_test_2_id") + api.InternalTest.add_coverage_data( suite_1_id, { Path("my_file_1.py"): CoverageLines.from_list([1, 2, 3, 4, 8, 9]), Path("my/other/path/my_file_2.py"): CoverageLines.from_list([1] + [9] + list(range(10, 101))), }, ) - api.Test.mark_skip(suite_1_test_2_id) + api.InternalTest.mark_skip(suite_1_test_2_id) # # suite_1_test_3 test and EFD retries - api.Test.start(suite_1_test_3_id) + api.InternalTest.start(suite_1_test_3_id) suite_1_test_3_abs_path_1 = Path("my_abs_file_3.py").absolute() - api.Test.add_coverage_data(suite_1_id, {suite_1_test_3_abs_path_1: CoverageLines.from_list([1, 2, 3, 4, 8, 9])}) + api.InternalTest.add_coverage_data( + suite_1_id, {suite_1_test_3_abs_path_1: CoverageLines.from_list([1, 2, 3, 4, 8, 9])} + ) suite_1_test_3_rel_path_1 = Path("my_rel_file_3.py") - api.Test.add_coverage_data(suite_1_id, {suite_1_test_3_rel_path_1: CoverageLines.from_list([2, 2])}) - api.Test.mark_itr_skipped(suite_1_test_3_id) + api.InternalTest.add_coverage_data(suite_1_id, {suite_1_test_3_rel_path_1: CoverageLines.from_list([2, 2])}) + api.InternalTest.mark_itr_skipped(suite_1_test_3_id) # - api.Test.start(suite_1_test_3_retry_1_id) - api.Test.mark_pass(suite_1_test_3_retry_1_id) + api.InternalTest.start(suite_1_test_3_retry_1_id) + api.InternalTest.mark_pass(suite_1_test_3_retry_1_id) # - api.Test.start(suite_1_test_3_retry_2_id) - api.Test.mark_pass(suite_1_test_3_retry_2_id) + api.InternalTest.start(suite_1_test_3_retry_2_id) + api.InternalTest.mark_pass(suite_1_test_3_retry_2_id) # - api.Test.start(suite_1_test_3_retry_3_id) - api.Test.add_coverage_data( + api.InternalTest.start(suite_1_test_3_retry_3_id) + api.InternalTest.add_coverage_data( suite_1_id, { Path("my_file_1.py"): CoverageLines.from_list([1, 2, 3, 4, 8, 9]), @@ -204,12 +247,12 @@ def main(): Path("my_rel_file_3.py"): CoverageLines.from_list([1, 3, 4, 5, 6] + list(range(79, 98))), }, ) - api.Test.mark_pass(suite_1_test_3_retry_3_id) + api.InternalTest.mark_pass(suite_1_test_3_retry_3_id) # # suite1_test_4 parametrized tests - api.Test.start(suite_1_test_4_parametrized_1_id) - api.Test.set_tags( + api.InternalTest.start(suite_1_test_4_parametrized_1_id) + api.InternalTest.set_tags( suite_1_test_4_parametrized_1_id, { "test.tag1": "suite_1_test_4_parametrized_1_id", @@ -219,65 +262,65 @@ def main(): "test.tag5": "this should also be deleted", }, ) - api.Test.delete_tag(suite_1_test_4_parametrized_1_id, "test.tag3") - api.Test.delete_tag(suite_1_test_4_parametrized_1_id, "test.tag5") - api.Test.mark_skip(suite_1_test_4_parametrized_1_id) + api.InternalTest.delete_tag(suite_1_test_4_parametrized_1_id, "test.tag3") + api.InternalTest.delete_tag(suite_1_test_4_parametrized_1_id, "test.tag5") + api.InternalTest.mark_skip(suite_1_test_4_parametrized_1_id) # - api.Test.start(suite_1_test_4_parametrized_2_id) - api.Test.mark_pass(suite_1_test_4_parametrized_2_id) + api.InternalTest.start(suite_1_test_4_parametrized_2_id) + api.InternalTest.mark_pass(suite_1_test_4_parametrized_2_id) # - api.Test.set_tag(suite_1_test_4_parametrized_3_id, "test.tag1", "suite_1_test_4_parametrized_3_id") - api.Test.set_tag(suite_1_test_4_parametrized_3_id, "test.tag2", "this will be deleted") - api.Test.set_tag(suite_1_test_4_parametrized_3_id, "test.tag3", 12333333) - api.Test.set_tag(suite_1_test_4_parametrized_3_id, "test.tag4", "this will also be deleted") - api.Test.delete_tags(suite_1_test_4_parametrized_3_id, ["test.tag2", "test.tag4"]) - api.Test.start(suite_1_test_4_parametrized_3_id) - api.Test.mark_fail(suite_1_test_4_parametrized_3_id, exc_info=_make_excinfo()) + api.InternalTest.set_tag(suite_1_test_4_parametrized_3_id, "test.tag1", "suite_1_test_4_parametrized_3_id") + api.InternalTest.set_tag(suite_1_test_4_parametrized_3_id, "test.tag2", "this will be deleted") + api.InternalTest.set_tag(suite_1_test_4_parametrized_3_id, "test.tag3", 12333333) + api.InternalTest.set_tag(suite_1_test_4_parametrized_3_id, "test.tag4", "this will also be deleted") + api.InternalTest.delete_tags(suite_1_test_4_parametrized_3_id, ["test.tag2", "test.tag4"]) + api.InternalTest.start(suite_1_test_4_parametrized_3_id) + api.InternalTest.mark_fail(suite_1_test_4_parametrized_3_id, exc_info=_make_excinfo()) # - api.TestSuite.finish(suite_1_id) + api.InternalTestSuite.finish(suite_1_id) - api.TestModule.finish(module_1_id) + api.InternalTestModule.finish(module_1_id) - api.TestModule.start(module_2_id) + api.InternalTestModule.start(module_2_id) - api.TestSuite.start(suite_2_id) + api.InternalTestSuite.start(suite_2_id) # # suite_2_test_1 test - api.Test.start(suite_2_test_1_id) - api.Test.mark_skip(suite_2_test_1_id) + api.InternalTest.start(suite_2_test_1_id) + api.InternalTest.mark_skip(suite_2_test_1_id) # # suite_2_test_2 parametrized tests - api.Test.set_tags( + api.InternalTest.set_tags( suite_2_test_2_parametrized_1_id, {"test.tag1": "suite_2_test_2_parametrized_1_id", "test.tag2": "two", "test.tag3": 3}, ) - api.Test.start(suite_2_test_2_parametrized_1_id) - api.Test.mark_pass(suite_2_test_2_parametrized_1_id) + api.InternalTest.start(suite_2_test_2_parametrized_1_id) + api.InternalTest.mark_pass(suite_2_test_2_parametrized_1_id) # - api.Test.start(suite_2_test_2_parametrized_2_id) - api.Test.set_tag(suite_2_test_2_parametrized_2_id, "test.tag1", "suite_2_test_2_parametrized_2_id") - api.Test.delete_tag(suite_2_test_2_parametrized_2_id, "test.tag1") - api.Test.mark_pass(suite_2_test_2_parametrized_2_id) + api.InternalTest.start(suite_2_test_2_parametrized_2_id) + api.InternalTest.set_tag(suite_2_test_2_parametrized_2_id, "test.tag1", "suite_2_test_2_parametrized_2_id") + api.InternalTest.delete_tag(suite_2_test_2_parametrized_2_id, "test.tag1") + api.InternalTest.mark_pass(suite_2_test_2_parametrized_2_id) # - api.Test.start(suite_2_test_2_parametrized_3_id) + api.InternalTest.start(suite_2_test_2_parametrized_3_id) - api.Test.mark_itr_skipped(suite_2_test_2_parametrized_3_id) + api.InternalTest.mark_itr_skipped(suite_2_test_2_parametrized_3_id) # - api.Test.start(suite_2_test_2_parametrized_4_id) - api.Test.mark_pass(suite_2_test_2_parametrized_4_id) + api.InternalTest.start(suite_2_test_2_parametrized_4_id) + api.InternalTest.mark_pass(suite_2_test_2_parametrized_4_id) # - api.Test.start(suite_2_test_2_parametrized_5_id) - api.Test.mark_pass(suite_2_test_2_parametrized_5_id) + api.InternalTest.start(suite_2_test_2_parametrized_5_id) + api.InternalTest.mark_pass(suite_2_test_2_parametrized_5_id) # # suite_2_test_3 test - api.Test.start(suite_2_test_3_id) - api.Test.set_tags( + api.InternalTest.start(suite_2_test_3_id) + api.InternalTest.set_tags( suite_2_test_3_id, {"test.tag1": "suite_2_test_3_id", "test.tag2": 2, "test.tag3": "this tag stays"} ) - api.Test.mark_skip(suite_2_test_3_id) + api.InternalTest.mark_skip(suite_2_test_3_id) # suite_2 coverage suite_2_test_2_parametrized_5_abs_path_1 = Path("my_abs_file_suite_2_5_1.py").absolute() @@ -285,7 +328,7 @@ def main(): # The two paths below should merge into a single file suite_2_test_2_parametrized_5_rel_path_1 = Path("my_rel_file_2_5.py").absolute() suite_2_test_2_parametrized_5_rel_path_2 = Path("my_rel_file_2_5.py") - api.Test.add_coverage_data( + api.InternalTest.add_coverage_data( suite_2_id, { suite_2_test_2_parametrized_5_abs_path_1: CoverageLines.from_list([1, 2, 3, 4, 5, 6, 10, 11, 12]), @@ -295,18 +338,18 @@ def main(): }, ) - api.TestSuite.finish(suite_2_id) + api.InternalTestSuite.finish(suite_2_id) - api.TestModule.finish(module_2_id) + api.InternalTestModule.finish(module_2_id) - api.TestModule.start(module_3_id) + api.InternalTestModule.start(module_3_id) - api.TestSuite.start(suite_3_id) + api.InternalTestSuite.start(suite_3_id) # # suite_3_test_1 test - api.Test.start(suite_3_test_1_id) - api.Test.add_coverage_data( + api.InternalTest.start(suite_3_test_1_id) + api.InternalTest.add_coverage_data( suite_3_id, { Path("my_file_suite_3_1.py"): CoverageLines.from_list([1, 2, 3, 4, 5, 6, 7, 8, 9]), @@ -317,22 +360,22 @@ def main(): Path("my_rel_file_suite_3_3.py"): CoverageLines.from_list([1, 3, 4, 5, 6] + list(range(79, 98))), }, ) - api.Test.mark_pass(suite_3_test_1_id) + api.InternalTest.mark_pass(suite_3_test_1_id) # # suite_3_test_2 test - api.Test.start(suite_3_test_2_id) - api.Test.mark_fail(suite_3_test_2_id) + api.InternalTest.start(suite_3_test_2_id) + api.InternalTest.mark_fail(suite_3_test_2_id) # # suite_3_test_3 test - api.Test.start(suite_3_test_3_id) - api.Test.mark_pass(suite_3_test_3_id) + api.InternalTest.start(suite_3_test_3_id) + api.InternalTest.mark_pass(suite_3_test_3_id) suite_2_test_2_parametrized_5_abs_path_1 = Path("my_abs_file_suite_3_1.py").absolute() suite_2_test_2_parametrized_5_abs_path_2 = Path("my_abs_file_suite_3_2.py").absolute() # The two paths below should merge into a single file suite_2_test_2_parametrized_5_rel_path_1 = Path("my_rel_file_suite3_5.py").absolute() suite_2_test_2_parametrized_5_rel_path_2 = Path("my_rel_file_suite3_5.py") - api.Test.add_coverage_data( + api.InternalTest.add_coverage_data( suite_3_id, { suite_2_test_2_parametrized_5_abs_path_1: CoverageLines.from_list([1, 2, 3, 4, 5, 6, 10, 11, 12]), @@ -342,67 +385,67 @@ def main(): }, ) - api.TestSuite.finish(suite_3_id) + api.InternalTestSuite.finish(suite_3_id) - api.TestSuite.start(suite_4_id) + api.InternalTestSuite.start(suite_4_id) # # suite_4_test_1 test - api.Test.start(suite_4_test_1_id) - api.Test.mark_pass(suite_4_test_1_id) + api.InternalTest.start(suite_4_test_1_id) + api.InternalTest.mark_pass(suite_4_test_1_id) # # suite_4_test_2 test - api.Test.start(suite_4_test_2_id) - api.Test.mark_pass(suite_4_test_2_id) + api.InternalTest.start(suite_4_test_2_id) + api.InternalTest.mark_pass(suite_4_test_2_id) # # suite_4_test_3 test - api.Test.start(suite_4_test_3_id) - api.Test.mark_pass(suite_4_test_3_id) + api.InternalTest.start(suite_4_test_3_id) + api.InternalTest.mark_pass(suite_4_test_3_id) - api.TestSuite.finish(suite_4_id) + api.InternalTestSuite.finish(suite_4_id) - api.TestModule.finish(module_3_id) + api.InternalTestModule.finish(module_3_id) - api.TestModule.start(module_4_id) + api.InternalTestModule.start(module_4_id) - api.TestSuite.start(suite_5_id) + api.InternalTestSuite.start(suite_5_id) # # suite_5_test_1 test - api.Test.start(suite_5_test_1_id) - api.Test.mark_itr_skipped(suite_5_test_1_id) + api.InternalTest.start(suite_5_test_1_id) + api.InternalTest.mark_itr_skipped(suite_5_test_1_id) # # suite_5_test_2 test - api.Test.start(suite_5_test_2_id) - api.Test.mark_itr_skipped(suite_5_test_2_id) + api.InternalTest.start(suite_5_test_2_id) + api.InternalTest.mark_itr_skipped(suite_5_test_2_id) # # suite_5_test_3 test - api.Test.start(suite_5_test_3_id) - api.Test.mark_itr_skipped(suite_5_test_3_id) + api.InternalTest.start(suite_5_test_3_id) + api.InternalTest.mark_itr_skipped(suite_5_test_3_id) - api.TestSuite.mark_itr_skipped(suite_5_id) + api.InternalTestSuite.mark_itr_skipped(suite_5_id) - api.TestSuite.start(suite_6_id) + api.InternalTestSuite.start(suite_6_id) # # suite_6_test_1 test - api.Test.start(suite_6_test_1_id) - api.Test.mark_itr_skipped(suite_6_test_1_id) + api.InternalTest.start(suite_6_test_1_id) + api.InternalTest.mark_itr_skipped(suite_6_test_1_id) # # suite_6_test_2 test - api.Test.start(suite_6_test_2_id) - api.Test.mark_pass(suite_6_test_2_id) + api.InternalTest.start(suite_6_test_2_id) + api.InternalTest.mark_pass(suite_6_test_2_id) # # suite_6_test_3 test - api.Test.start(suite_6_test_3_id) - api.Test.mark_itr_skipped(suite_6_test_3_id) + api.InternalTest.start(suite_6_test_3_id) + api.InternalTest.mark_itr_skipped(suite_6_test_3_id) - api.Test.mark_itr_forced_run(suite_6_id) - api.TestSuite.finish(suite_6_id) + api.InternalTest.mark_itr_forced_run(suite_6_id) + api.InternalTestSuite.finish(suite_6_id) - api.TestModule.finish(module_4_id) + api.InternalTestModule.finish(module_4_id) - api.TestSession.finish() + api.InternalTestSession.finish() # FINISH TESTS diff --git a/tests/ci_visibility/api/fake_runner_mix_fail_itr_test_level.py b/tests/ci_visibility/api/fake_runner_mix_fail_itr_test_level.py index c8be9a0315b..6c10b378e30 100644 --- a/tests/ci_visibility/api/fake_runner_mix_fail_itr_test_level.py +++ b/tests/ci_visibility/api/fake_runner_mix_fail_itr_test_level.py @@ -21,185 +21,224 @@ import sys from unittest import mock -from ddtrace.ext.test_visibility import api +from ddtrace.ext.test_visibility import api as ext_api from ddtrace.ext.test_visibility.coverage_lines import CoverageLines +from ddtrace.internal.test_visibility import api def _make_excinfo(): try: raise ValueError("This is a fake exception") except ValueError: - return api.TestExcInfo(*sys.exc_info()) + return ext_api.TestExcInfo(*sys.exc_info()) def main(): - api.enable_test_visibility() + ext_api.enable_test_visibility() # START DISCOVERY - api.TestSession.discover("manual_test_mix_fail_itr_test_level", "dd_manual_test_fw", "1.0.0") - api.TestSession.start() + api.InternalTestSession.discover("manual_test_mix_fail_itr_test_level", "dd_manual_test_fw", "1.0.0") + api.InternalTestSession.start() - module_1_id = api.TestModuleId("module_1") + module_1_id = ext_api.TestModuleId("module_1") - api.TestModule.discover(module_1_id) + api.InternalTestModule.discover(module_1_id) - suite_1_id = api.TestSuiteId(module_1_id, "suite_1") - api.TestSuite.discover(suite_1_id) + suite_1_id = ext_api.TestSuiteId(module_1_id, "suite_1") + api.InternalTestSuite.discover(suite_1_id) - suite_1_test_1_id = api.TestId(suite_1_id, "test_1") - suite_1_test_2_id = api.TestId(suite_1_id, "test_2") - suite_1_test_3_id = api.TestId(suite_1_id, "test_3") - suite_1_test_3_retry_1_id = api.TestId(suite_1_id, "test_3", retry_number=1) - suite_1_test_3_retry_2_id = api.TestId(suite_1_id, "test_3", retry_number=2) - suite_1_test_3_retry_3_id = api.TestId(suite_1_id, "test_3", retry_number=3) + suite_1_test_1_id = api.InternalTestId(suite_1_id, "test_1") + suite_1_test_2_id = api.InternalTestId(suite_1_id, "test_2") + suite_1_test_3_id = api.InternalTestId(suite_1_id, "test_3") + suite_1_test_3_retry_1_id = api.InternalTestId(suite_1_id, "test_3", retry_number=1) + suite_1_test_3_retry_2_id = api.InternalTestId(suite_1_id, "test_3", retry_number=2) + suite_1_test_3_retry_3_id = api.InternalTestId(suite_1_id, "test_3", retry_number=3) - suite_1_test_4_parametrized_1_id = api.TestId(suite_1_id, "test_4", parameters=json.dumps({"param1": "value1"})) - suite_1_test_4_parametrized_2_id = api.TestId(suite_1_id, "test_4", parameters=json.dumps({"param1": "value2"})) - suite_1_test_4_parametrized_3_id = api.TestId(suite_1_id, "test_4", parameters=json.dumps({"param1": "value3"})) + suite_1_test_4_parametrized_1_id = api.InternalTestId( + suite_1_id, "test_4", parameters=json.dumps({"param1": "value1"}) + ) + suite_1_test_4_parametrized_2_id = api.InternalTestId( + suite_1_id, "test_4", parameters=json.dumps({"param1": "value2"}) + ) + suite_1_test_4_parametrized_3_id = api.InternalTestId( + suite_1_id, "test_4", parameters=json.dumps({"param1": "value3"}) + ) - api.Test.discover(suite_1_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) - api.Test.discover(suite_1_test_2_id, source_file_info=None) - api.Test.discover( + api.InternalTest.discover( + suite_1_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2) + ) + api.InternalTest.discover(suite_1_test_2_id, source_file_info=None) + api.InternalTest.discover( suite_1_test_3_id, codeowners=["@romain", "@romain2"], - source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), + source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_3_id) - - api.Test.discover(suite_1_test_4_parametrized_1_id) - api.Test.discover(suite_1_test_4_parametrized_2_id) - api.Test.discover(suite_1_test_4_parametrized_3_id) - - module_2_id = api.TestModuleId("module_2") - suite_2_id = api.TestSuiteId(module_2_id, "suite_2") - suite_2_test_1_id = api.TestId(suite_2_id, "test_1") - suite_2_test_2_parametrized_1_id = api.TestId(suite_2_id, "test_2", parameters=json.dumps({"param1": "value1"})) - suite_2_test_2_parametrized_2_id = api.TestId(suite_2_id, "test_2", parameters=json.dumps({"param1": "value2"})) - suite_2_test_2_parametrized_3_id = api.TestId(suite_2_id, "test_2", parameters=json.dumps({"param1": "value3"})) - suite_2_test_2_parametrized_4_id = api.TestId(suite_2_id, "test_2", parameters=json.dumps({"param1": "value4"})) - suite_2_test_2_parametrized_5_id = api.TestId(suite_2_id, "test_2", parameters=json.dumps({"param1": "value5"})) - suite_2_test_2_source_file_info = api.TestSourceFileInfo(Path("test_file_2.py"), 8, 9) - suite_2_test_3_id = api.TestId(suite_2_id, "test_3") - - api.TestModule.discover(module_2_id) - api.TestSuite.discover(suite_2_id) - api.Test.discover(suite_2_test_1_id, source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2)) - api.Test.discover(suite_2_test_2_parametrized_1_id, source_file_info=suite_2_test_2_source_file_info) - api.Test.discover(suite_2_test_2_parametrized_2_id, source_file_info=suite_2_test_2_source_file_info) - api.Test.discover(suite_2_test_2_parametrized_3_id, source_file_info=suite_2_test_2_source_file_info) - api.Test.discover(suite_2_test_2_parametrized_4_id, source_file_info=suite_2_test_2_source_file_info) - api.Test.discover(suite_2_test_2_parametrized_5_id, source_file_info=suite_2_test_2_source_file_info) - - api.Test.discover( + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_1_id) + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_2_id) + api.InternalTest.discover_early_flake_retry(suite_1_test_3_retry_3_id) + + api.InternalTest.discover(suite_1_test_4_parametrized_1_id) + api.InternalTest.discover(suite_1_test_4_parametrized_2_id) + api.InternalTest.discover(suite_1_test_4_parametrized_3_id) + + module_2_id = ext_api.TestModuleId("module_2") + suite_2_id = ext_api.TestSuiteId(module_2_id, "suite_2") + suite_2_test_1_id = api.InternalTestId(suite_2_id, "test_1") + suite_2_test_2_parametrized_1_id = api.InternalTestId( + suite_2_id, "test_2", parameters=json.dumps({"param1": "value1"}) + ) + suite_2_test_2_parametrized_2_id = api.InternalTestId( + suite_2_id, "test_2", parameters=json.dumps({"param1": "value2"}) + ) + suite_2_test_2_parametrized_3_id = api.InternalTestId( + suite_2_id, "test_2", parameters=json.dumps({"param1": "value3"}) + ) + suite_2_test_2_parametrized_4_id = api.InternalTestId( + suite_2_id, "test_2", parameters=json.dumps({"param1": "value4"}) + ) + suite_2_test_2_parametrized_5_id = api.InternalTestId( + suite_2_id, "test_2", parameters=json.dumps({"param1": "value5"}) + ) + suite_2_test_2_source_file_info = ext_api.TestSourceFileInfo(Path("test_file_2.py"), 8, 9) + suite_2_test_3_id = api.InternalTestId(suite_2_id, "test_3") + + api.InternalTestModule.discover(module_2_id) + api.InternalTestSuite.discover(suite_2_id) + api.InternalTest.discover( + suite_2_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 1, 2) + ) + api.InternalTest.discover(suite_2_test_2_parametrized_1_id, source_file_info=suite_2_test_2_source_file_info) + api.InternalTest.discover(suite_2_test_2_parametrized_2_id, source_file_info=suite_2_test_2_source_file_info) + api.InternalTest.discover(suite_2_test_2_parametrized_3_id, source_file_info=suite_2_test_2_source_file_info) + api.InternalTest.discover(suite_2_test_2_parametrized_4_id, source_file_info=suite_2_test_2_source_file_info) + api.InternalTest.discover(suite_2_test_2_parametrized_5_id, source_file_info=suite_2_test_2_source_file_info) + + api.InternalTest.discover( suite_2_test_3_id, codeowners=["@romain"], - source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), + source_file_info=ext_api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - module_3_id = api.TestModuleId("module_3") - suite_3_id = api.TestSuiteId(module_3_id, "suite_3") - suite_4_id = api.TestSuiteId(module_3_id, "suite_4") + module_3_id = ext_api.TestModuleId("module_3") + suite_3_id = ext_api.TestSuiteId(module_3_id, "suite_3") + suite_4_id = ext_api.TestSuiteId(module_3_id, "suite_4") - suite_3_test_1_id = api.TestId(suite_3_id, "test_1") - suite_3_test_2_id = api.TestId(suite_3_id, "test_2") - suite_3_test_3_id = api.TestId(suite_3_id, "test_3") + suite_3_test_1_id = api.InternalTestId(suite_3_id, "test_1") + suite_3_test_2_id = api.InternalTestId(suite_3_id, "test_2") + suite_3_test_3_id = api.InternalTestId(suite_3_id, "test_3") - suite_4_test_1_id = api.TestId(suite_4_id, "test_1") - suite_4_test_2_id = api.TestId(suite_4_id, "test_2") - suite_4_test_3_id = api.TestId(suite_4_id, "test_3") + suite_4_test_1_id = api.InternalTestId(suite_4_id, "test_1") + suite_4_test_2_id = api.InternalTestId(suite_4_id, "test_2") + suite_4_test_3_id = api.InternalTestId(suite_4_id, "test_3") - api.TestModule.discover(module_3_id) + api.InternalTestModule.discover(module_3_id) - api.TestSuite.discover(suite_3_id) + api.InternalTestSuite.discover(suite_3_id) - api.Test.discover(suite_3_test_1_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_3.py"), 4, 6)) - api.Test.discover(suite_3_test_2_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_3.py"), 9, 12)) - api.Test.discover(suite_3_test_3_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_3.py"), 16, 48)) + api.InternalTest.discover( + suite_3_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_3.py"), 4, 6) + ) + api.InternalTest.discover( + suite_3_test_2_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_3.py"), 9, 12) + ) + api.InternalTest.discover( + suite_3_test_3_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_3.py"), 16, 48) + ) - api.TestSuite.discover(suite_4_id) - api.Test.discover(suite_4_test_1_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_4.py"), 4, 6)) - api.Test.discover(suite_4_test_2_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_4.py"), 9, 12)) - api.Test.discover(suite_4_test_3_id, source_file_info=api.TestSourceFileInfo(Path("module_3/suite_4.py"), 16, 48)) + api.InternalTestSuite.discover(suite_4_id) + api.InternalTest.discover( + suite_4_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_4.py"), 4, 6) + ) + api.InternalTest.discover( + suite_4_test_2_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_4.py"), 9, 12) + ) + api.InternalTest.discover( + suite_4_test_3_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_3/suite_4.py"), 16, 48) + ) - module_4_id = api.TestModuleId("module_4") - suite_5_id = api.TestSuiteId(module_4_id, "suite_5") - suite_6_id = api.TestSuiteId(module_4_id, "suite_6") + module_4_id = ext_api.TestModuleId("module_4") + suite_5_id = ext_api.TestSuiteId(module_4_id, "suite_5") + suite_6_id = ext_api.TestSuiteId(module_4_id, "suite_6") - suite_5_test_1_id = api.TestId(suite_5_id, "test_1") - suite_5_test_2_id = api.TestId(suite_5_id, "test_2") - suite_5_test_3_id = api.TestId(suite_5_id, "test_3") + suite_5_test_1_id = api.InternalTestId(suite_5_id, "test_1") + suite_5_test_2_id = api.InternalTestId(suite_5_id, "test_2") + suite_5_test_3_id = api.InternalTestId(suite_5_id, "test_3") - suite_6_test_1_id = api.TestId(suite_6_id, "test_1") - suite_6_test_2_id = api.TestId(suite_6_id, "test_2") - suite_6_test_3_id = api.TestId(suite_6_id, "test_3") + suite_6_test_1_id = api.InternalTestId(suite_6_id, "test_1") + suite_6_test_2_id = api.InternalTestId(suite_6_id, "test_2") + suite_6_test_3_id = api.InternalTestId(suite_6_id, "test_3") - api.TestModule.discover(module_4_id) + api.InternalTestModule.discover(module_4_id) - api.TestSuite.discover(suite_5_id) + api.InternalTestSuite.discover(suite_5_id) - api.Test.discover(suite_5_test_1_id, source_file_info=api.TestSourceFileInfo(Path("module_5/suite_5.py"), 4, 6)) - api.Test.discover(suite_5_test_2_id, source_file_info=api.TestSourceFileInfo(Path("module_5/suite_5.py"), 9, 12)) - api.Test.discover(suite_5_test_3_id, source_file_info=api.TestSourceFileInfo(Path("module_5/suite_5.py"), 16, 48)) + api.InternalTest.discover( + suite_5_test_1_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_5/suite_5.py"), 4, 6) + ) + api.InternalTest.discover( + suite_5_test_2_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_5/suite_5.py"), 9, 12) + ) + api.InternalTest.discover( + suite_5_test_3_id, source_file_info=ext_api.TestSourceFileInfo(Path("module_5/suite_5.py"), 16, 48) + ) - api.TestSuite.discover(suite_6_id) - api.Test.discover(suite_6_test_1_id) - api.Test.discover(suite_6_test_2_id) - api.Test.mark_itr_unskippable(suite_6_test_2_id) - api.Test.discover(suite_6_test_3_id) + api.InternalTestSuite.discover(suite_6_id) + api.InternalTest.discover(suite_6_test_1_id) + api.InternalTest.discover(suite_6_test_2_id) + api.InternalTest.mark_itr_unskippable(suite_6_test_2_id) + api.InternalTest.discover(suite_6_test_3_id) # END DISCOVERY # START TESTS - api.TestModule.start(module_1_id) + api.InternalTestModule.start(module_1_id) - api.TestSuite.start(suite_1_id) + api.InternalTestSuite.start(suite_1_id) # # suite_1_test_1 test - api.Test.start(suite_1_test_1_id) - api.Test.add_coverage_data( + api.InternalTest.start(suite_1_test_1_id) + api.InternalTest.add_coverage_data( suite_1_test_1_id, {Path("my_file_1.py"): CoverageLines.from_list([1, 2, 3, 4, 5, 6, 7, 8, 9])} ) - api.Test.mark_pass(suite_1_test_1_id) + api.InternalTest.mark_pass(suite_1_test_1_id) # # suite_1_test_2 test - api.Test.start(suite_1_test_2_id) - api.Test.set_tag(suite_1_test_2_id, "test.tag1", "suite_1_test_2_id") - api.Test.add_coverage_data( + api.InternalTest.start(suite_1_test_2_id) + api.InternalTest.set_tag(suite_1_test_2_id, "test.tag1", "suite_1_test_2_id") + api.InternalTest.add_coverage_data( suite_1_test_2_id, { Path("my_file_1.py"): CoverageLines.from_list([1, 2, 3, 4, 5, 6, 7, 8, 9]), Path("my/other/path/my_file_2.py"): CoverageLines.from_list(list(range(1, 10)) + list(range(10, 101))), }, ) - api.Test.mark_skip(suite_1_test_2_id) + api.InternalTest.mark_skip(suite_1_test_2_id) # # suite_1_test_3 test and EFD retries - api.Test.start(suite_1_test_3_id) + api.InternalTest.start(suite_1_test_3_id) suite_1_test_3_abs_path_1 = Path("my_abs_file_3.py").absolute() - api.Test.add_coverage_data( + api.InternalTest.add_coverage_data( suite_1_test_3_id, {suite_1_test_3_abs_path_1: CoverageLines.from_list([1, 2, 3, 4, 8, 9])} ) suite_1_test_3_rel_path_1 = Path("my_rel_file_3.py") - api.Test.add_coverage_data(suite_1_test_3_id, {suite_1_test_3_rel_path_1: CoverageLines.from_list([2])}) - api.Test.mark_itr_skipped(suite_1_test_3_id) + api.InternalTest.add_coverage_data(suite_1_test_3_id, {suite_1_test_3_rel_path_1: CoverageLines.from_list([2])}) + api.InternalTest.mark_itr_skipped(suite_1_test_3_id) # - api.Test.start(suite_1_test_3_retry_1_id) - api.Test.mark_pass(suite_1_test_3_retry_1_id) + api.InternalTest.start(suite_1_test_3_retry_1_id) + api.InternalTest.mark_pass(suite_1_test_3_retry_1_id) # - api.Test.start(suite_1_test_3_retry_2_id) - api.Test.mark_pass(suite_1_test_3_retry_2_id) + api.InternalTest.start(suite_1_test_3_retry_2_id) + api.InternalTest.mark_pass(suite_1_test_3_retry_2_id) # - api.Test.start(suite_1_test_3_retry_3_id) - api.Test.add_coverage_data( + api.InternalTest.start(suite_1_test_3_retry_3_id) + api.InternalTest.add_coverage_data( suite_1_test_2_id, { Path("my_file_1.py"): CoverageLines.from_list([1, 2, 3, 4, 5, 6, 7, 8, 9]), @@ -208,12 +247,12 @@ def main(): Path("my_rel_file_3.py"): CoverageLines.from_list([1, 3, 4, 5, 6] + list(range(79, 98))), }, ) - api.Test.mark_pass(suite_1_test_3_retry_3_id) + api.InternalTest.mark_pass(suite_1_test_3_retry_3_id) # # suite1_test_4 parametrized tests - api.Test.start(suite_1_test_4_parametrized_1_id) - api.Test.set_tags( + api.InternalTest.start(suite_1_test_4_parametrized_1_id) + api.InternalTest.set_tags( suite_1_test_4_parametrized_1_id, { "test.tag1": "suite_1_test_4_parametrized_1_id", @@ -223,62 +262,62 @@ def main(): "test.tag5": "this should also be deleted", }, ) - api.Test.delete_tag(suite_1_test_4_parametrized_1_id, "test.tag3") - api.Test.delete_tag(suite_1_test_4_parametrized_1_id, "test.tag5") - api.Test.mark_skip(suite_1_test_4_parametrized_1_id) + api.InternalTest.delete_tag(suite_1_test_4_parametrized_1_id, "test.tag3") + api.InternalTest.delete_tag(suite_1_test_4_parametrized_1_id, "test.tag5") + api.InternalTest.mark_skip(suite_1_test_4_parametrized_1_id) # - api.Test.start(suite_1_test_4_parametrized_2_id) - api.Test.mark_pass(suite_1_test_4_parametrized_2_id) + api.InternalTest.start(suite_1_test_4_parametrized_2_id) + api.InternalTest.mark_pass(suite_1_test_4_parametrized_2_id) # - api.Test.set_tag(suite_1_test_4_parametrized_3_id, "test.tag1", "suite_1_test_4_parametrized_3_id") - api.Test.set_tag(suite_1_test_4_parametrized_3_id, "test.tag2", "this will be deleted") - api.Test.set_tag(suite_1_test_4_parametrized_3_id, "test.tag3", 12333333) - api.Test.set_tag(suite_1_test_4_parametrized_3_id, "test.tag4", "this will also be deleted") - api.Test.delete_tags(suite_1_test_4_parametrized_3_id, ["test.tag2", "test.tag4"]) - api.Test.start(suite_1_test_4_parametrized_3_id) - api.Test.mark_fail(suite_1_test_4_parametrized_3_id, exc_info=_make_excinfo()) + api.InternalTest.set_tag(suite_1_test_4_parametrized_3_id, "test.tag1", "suite_1_test_4_parametrized_3_id") + api.InternalTest.set_tag(suite_1_test_4_parametrized_3_id, "test.tag2", "this will be deleted") + api.InternalTest.set_tag(suite_1_test_4_parametrized_3_id, "test.tag3", 12333333) + api.InternalTest.set_tag(suite_1_test_4_parametrized_3_id, "test.tag4", "this will also be deleted") + api.InternalTest.delete_tags(suite_1_test_4_parametrized_3_id, ["test.tag2", "test.tag4"]) + api.InternalTest.start(suite_1_test_4_parametrized_3_id) + api.InternalTest.mark_fail(suite_1_test_4_parametrized_3_id, exc_info=_make_excinfo()) # - api.TestSuite.finish(suite_1_id) + api.InternalTestSuite.finish(suite_1_id) - api.TestModule.finish(module_1_id) + api.InternalTestModule.finish(module_1_id) - api.TestModule.start(module_2_id) + api.InternalTestModule.start(module_2_id) - api.TestSuite.start(suite_2_id) + api.InternalTestSuite.start(suite_2_id) # # suite_2_test_1 test - api.Test.start(suite_2_test_1_id) - api.Test.mark_skip(suite_2_test_1_id) + api.InternalTest.start(suite_2_test_1_id) + api.InternalTest.mark_skip(suite_2_test_1_id) # # suite_2_test_2 parametrized tests - api.Test.set_tags( + api.InternalTest.set_tags( suite_2_test_2_parametrized_1_id, {"test.tag1": "suite_2_test_2_parametrized_1_id", "test.tag2": "two", "test.tag3": 3}, ) - api.Test.start(suite_2_test_2_parametrized_1_id) - api.Test.mark_pass(suite_2_test_2_parametrized_1_id) + api.InternalTest.start(suite_2_test_2_parametrized_1_id) + api.InternalTest.mark_pass(suite_2_test_2_parametrized_1_id) # - api.Test.start(suite_2_test_2_parametrized_2_id) - api.Test.set_tag(suite_2_test_2_parametrized_2_id, "test.tag1", "suite_2_test_2_parametrized_2_id") - api.Test.delete_tag(suite_2_test_2_parametrized_2_id, "test.tag1") - api.Test.mark_pass(suite_2_test_2_parametrized_2_id) + api.InternalTest.start(suite_2_test_2_parametrized_2_id) + api.InternalTest.set_tag(suite_2_test_2_parametrized_2_id, "test.tag1", "suite_2_test_2_parametrized_2_id") + api.InternalTest.delete_tag(suite_2_test_2_parametrized_2_id, "test.tag1") + api.InternalTest.mark_pass(suite_2_test_2_parametrized_2_id) # - api.Test.start(suite_2_test_2_parametrized_3_id) + api.InternalTest.start(suite_2_test_2_parametrized_3_id) - api.Test.mark_itr_skipped(suite_2_test_2_parametrized_3_id) + api.InternalTest.mark_itr_skipped(suite_2_test_2_parametrized_3_id) # - api.Test.start(suite_2_test_2_parametrized_4_id) - api.Test.mark_pass(suite_2_test_2_parametrized_4_id) + api.InternalTest.start(suite_2_test_2_parametrized_4_id) + api.InternalTest.mark_pass(suite_2_test_2_parametrized_4_id) # - api.Test.start(suite_2_test_2_parametrized_5_id) + api.InternalTest.start(suite_2_test_2_parametrized_5_id) suite_2_test_2_parametrized_5_abs_path_1 = Path("my_abs_file_5_1.py").absolute() suite_2_test_2_parametrized_5_abs_path_2 = Path("my_abs_file_5_2.py").absolute() # The two paths below should merge into a single file suite_2_test_2_parametrized_5_rel_path_1 = Path("my_rel_file_5.py").absolute() suite_2_test_2_parametrized_5_rel_path_2 = Path("my_rel_file_5.py") - api.Test.add_coverage_data( + api.InternalTest.add_coverage_data( suite_2_test_2_parametrized_5_id, { suite_2_test_2_parametrized_5_abs_path_1: CoverageLines.from_list([1, 2, 3, 4, 5, 6, 10, 11, 12]), @@ -287,98 +326,98 @@ def main(): suite_2_test_2_parametrized_5_rel_path_2: CoverageLines.from_list([3]), }, ) - api.Test.mark_pass(suite_2_test_2_parametrized_5_id) + api.InternalTest.mark_pass(suite_2_test_2_parametrized_5_id) # # suite_2_test_3 test - api.Test.start(suite_2_test_3_id) - api.Test.set_tags( + api.InternalTest.start(suite_2_test_3_id) + api.InternalTest.set_tags( suite_2_test_3_id, {"test.tag1": "suite_2_test_3_id", "test.tag2": 2, "test.tag3": "this tag stays"} ) - api.Test.mark_skip(suite_2_test_3_id) + api.InternalTest.mark_skip(suite_2_test_3_id) - api.TestSuite.finish(suite_2_id) + api.InternalTestSuite.finish(suite_2_id) - api.TestModule.finish(module_2_id) + api.InternalTestModule.finish(module_2_id) - api.TestModule.start(module_3_id) + api.InternalTestModule.start(module_3_id) - api.TestSuite.start(suite_3_id) + api.InternalTestSuite.start(suite_3_id) # # suite_3_test_1 test - api.Test.start(suite_3_test_1_id) - api.Test.mark_pass(suite_3_test_1_id) + api.InternalTest.start(suite_3_test_1_id) + api.InternalTest.mark_pass(suite_3_test_1_id) # # suite_3_test_2 test - api.Test.start(suite_3_test_2_id) - api.Test.mark_fail(suite_3_test_2_id) + api.InternalTest.start(suite_3_test_2_id) + api.InternalTest.mark_fail(suite_3_test_2_id) # # suite_3_test_3 test - api.Test.start(suite_3_test_3_id) - api.Test.mark_pass(suite_3_test_3_id) + api.InternalTest.start(suite_3_test_3_id) + api.InternalTest.mark_pass(suite_3_test_3_id) - api.TestSuite.finish(suite_3_id) + api.InternalTestSuite.finish(suite_3_id) - api.TestSuite.start(suite_4_id) + api.InternalTestSuite.start(suite_4_id) # # suite_4_test_1 test - api.Test.start(suite_4_test_1_id) - api.Test.mark_pass(suite_4_test_1_id) + api.InternalTest.start(suite_4_test_1_id) + api.InternalTest.mark_pass(suite_4_test_1_id) # # suite_4_test_2 test - api.Test.start(suite_4_test_2_id) - api.Test.mark_pass(suite_4_test_2_id) + api.InternalTest.start(suite_4_test_2_id) + api.InternalTest.mark_pass(suite_4_test_2_id) # # suite_4_test_3 test - api.Test.start(suite_4_test_3_id) - api.Test.mark_pass(suite_4_test_3_id) + api.InternalTest.start(suite_4_test_3_id) + api.InternalTest.mark_pass(suite_4_test_3_id) - api.TestSuite.finish(suite_4_id) + api.InternalTestSuite.finish(suite_4_id) - api.TestModule.finish(module_3_id) + api.InternalTestModule.finish(module_3_id) - api.TestModule.start(module_4_id) + api.InternalTestModule.start(module_4_id) - api.TestSuite.start(suite_5_id) + api.InternalTestSuite.start(suite_5_id) # # suite_5_test_1 test - api.Test.start(suite_5_test_1_id) - api.Test.mark_itr_skipped(suite_5_test_1_id) + api.InternalTest.start(suite_5_test_1_id) + api.InternalTest.mark_itr_skipped(suite_5_test_1_id) # # suite_5_test_2 test - api.Test.start(suite_5_test_2_id) - api.Test.mark_itr_skipped(suite_5_test_2_id) + api.InternalTest.start(suite_5_test_2_id) + api.InternalTest.mark_itr_skipped(suite_5_test_2_id) # # suite_5_test_3 test - api.Test.start(suite_5_test_3_id) - api.Test.mark_itr_skipped(suite_5_test_3_id) + api.InternalTest.start(suite_5_test_3_id) + api.InternalTest.mark_itr_skipped(suite_5_test_3_id) - api.TestSuite.mark_itr_skipped(suite_5_id) + api.InternalTestSuite.mark_itr_skipped(suite_5_id) - api.TestSuite.start(suite_6_id) + api.InternalTestSuite.start(suite_6_id) # # suite_6_test_1 test - api.Test.start(suite_6_test_1_id) - api.Test.mark_itr_skipped(suite_6_test_1_id) + api.InternalTest.start(suite_6_test_1_id) + api.InternalTest.mark_itr_skipped(suite_6_test_1_id) # # suite_6_test_2 test - api.Test.start(suite_6_test_2_id) - api.Test.mark_itr_forced_run(suite_6_test_2_id) - api.Test.mark_pass(suite_6_test_2_id) + api.InternalTest.start(suite_6_test_2_id) + api.InternalTest.mark_itr_forced_run(suite_6_test_2_id) + api.InternalTest.mark_pass(suite_6_test_2_id) # # suite_6_test_3 test - api.Test.start(suite_6_test_3_id) - api.Test.mark_itr_skipped(suite_6_test_3_id) + api.InternalTest.start(suite_6_test_3_id) + api.InternalTest.mark_itr_skipped(suite_6_test_3_id) - api.TestSuite.finish(suite_6_id) + api.InternalTestSuite.finish(suite_6_id) - api.TestModule.finish(module_4_id) + api.InternalTestModule.finish(module_4_id) - api.TestSession.finish() + api.InternalTestSession.finish() # FINISH TESTS diff --git a/tests/ci_visibility/api/fake_runner_mix_pass.py b/tests/ci_visibility/api/fake_runner_mix_pass.py index a2196a2f49c..4c55df4d00a 100644 --- a/tests/ci_visibility/api/fake_runner_mix_pass.py +++ b/tests/ci_visibility/api/fake_runner_mix_pass.py @@ -30,9 +30,6 @@ def main(): suite_1_test_1_id = api.TestId(suite_1_id, "test_1") suite_1_test_2_id = api.TestId(suite_1_id, "test_2") suite_1_test_3_id = api.TestId(suite_1_id, "test_3") - suite_1_test_3_retry_1_id = api.TestId(suite_1_id, "test_3", retry_number=1) - suite_1_test_3_retry_2_id = api.TestId(suite_1_id, "test_3", retry_number=2) - suite_1_test_3_retry_3_id = api.TestId(suite_1_id, "test_3", retry_number=3) suite_1_test_4_parametrized_1_id = api.TestId(suite_1_id, "test_4", parameters=json.dumps({"param1": "value1"})) suite_1_test_4_parametrized_2_id = api.TestId(suite_1_id, "test_4", parameters=json.dumps({"param1": "value2"})) @@ -46,9 +43,6 @@ def main(): source_file_info=api.TestSourceFileInfo(Path("my_file_1.py"), 4, 12), is_early_flake_detection=True, ) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_1_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_2_id) - api.Test.discover_early_flake_retry(suite_1_test_3_retry_3_id) api.Test.discover(suite_1_test_4_parametrized_1_id) api.Test.discover(suite_1_test_4_parametrized_2_id) @@ -103,16 +97,8 @@ def main(): # # suite_1_test_3 test and EFD retries api.Test.start(suite_1_test_3_id) - api.Test.mark_itr_skipped(suite_1_test_3_id) + api.Test.mark_skip(suite_1_test_3_id) # - api.Test.start(suite_1_test_3_retry_1_id) - api.Test.mark_pass(suite_1_test_3_retry_1_id) - # - api.Test.start(suite_1_test_3_retry_2_id) - api.Test.mark_pass(suite_1_test_3_retry_2_id) - # - api.Test.start(suite_1_test_3_retry_3_id) - api.Test.mark_pass(suite_1_test_3_retry_3_id) # # suite1_test_4 parametrized tests @@ -170,7 +156,7 @@ def main(): api.Test.mark_pass(suite_2_test_2_parametrized_2_id) # api.Test.start(suite_2_test_2_parametrized_3_id) - api.Test.mark_itr_skipped(suite_2_test_2_parametrized_3_id) + api.Test.mark_skip(suite_2_test_2_parametrized_3_id) # api.Test.start(suite_2_test_2_parametrized_4_id) api.Test.mark_pass(suite_2_test_2_parametrized_4_id)