ci: fix subprocess wrapper [backport 1.18] (#6646)

Backport 8863df6 from #6624 to 1.18.

This change fixes the `pytest.mark.subprocess` decorator

## Checklist
- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed. If no release note is required, add label
`changelog/no-changelog`.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist

- [x] Title is accurate.
- [x] No unnecessary changes are introduced.
- [x] Description motivates each change.
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [x] Testing strategy adequately addresses listed risk(s).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] Release note makes sense to a user of the library.
- [x] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Co-authored-by: Federico Mon <[email protected]>

.riot/requirements/1c84c16.txt

@@ -0,0 +1,20 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --no-annotate --resolver=backtracking .riot/requirements/1c84c16.in
+#
+attrs==23.1.0
+coverage[toml]==7.2.7
+exceptiongroup==1.1.2
+hypothesis==6.45.0
+iniconfig==2.0.0
+mock==5.1.0
+opentracing==2.4.0
+packaging==23.1
+pluggy==1.2.0
+pytest==7.4.0
+pytest-cov==4.1.0
+pytest-mock==3.11.1
+sortedcontainers==2.4.0
+tomli==2.0.1

riotfile.py

@@ -121,6 +121,11 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
             "scripts/needs_testrun.py "
             "tests/suitespec.py",
         ),
+        Venv(
+            pys=["3"],
+            name="meta-testing",
+            command="pytest {cmdargs} tests/meta",
+        ),
         Venv(
             name="circleci-gen-config",
             command="python scripts/gen_circleci_config.py {cmdargs}",

scripts/gen_circleci_config.py

@@ -66,7 +66,7 @@ def check(name: str, command: str, paths: t.Set[str]) -> None:
     )
     check(
         name="Style: Test snapshots",
-        command='hatch run lint:fmt-snapshots && git diff --exit-code tests/snapshots hatch.toml',
+        command="hatch run lint:fmt-snapshots && git diff --exit-code tests/snapshots hatch.toml",
         paths={"tests/snapshots/*", "hatch.toml"},
     )
     check(
@@ -79,6 +79,11 @@ def check(name: str, command: str, paths: t.Set[str]) -> None:
         command="riot -v run -s scripts",
         paths={"scripts/*.py"},
     )
+    check(
+        name="Run conftest tests",
+        command="riot -v run meta-testing",
+        paths={"tests/*conftest.py", "tests/meta/*"},
+    )
     check(
         name="Validate suitespec JSON file",
         command="python -m tests.suitespec",

tests/conftest.py

@@ -9,8 +9,6 @@
 from tempfile import NamedTemporaryFile
 import time
 
-from _pytest.runner import CallInfo
-from _pytest.runner import TestReport
 from _pytest.runner import call_and_report
 from _pytest.runner import pytest_runtest_protocol as default_pytest_runtest_protocol
 import pytest
@@ -257,7 +255,7 @@ def _subprocess_wrapper():
             if not is_stream_ok(err, expected_err):
                 raise AssertionError("STDERR: Expected [%s] got [%s]" % (expected_err, err))
 
-        return TestReport.from_item_and_call(item, CallInfo.from_call(_subprocess_wrapper, "call"))
+        return _subprocess_wrapper()
 
 
 @pytest.hookimpl(tryfirst=True)

tests/meta/test_conftest.py

@@ -0,0 +1,88 @@
+import os
+
+import pytest
+
+from tests.utils import TracerTestCase
+from tests.utils import override_env
+
+
+class ConftestTestCase(TracerTestCase):
+    """
+    Test case to verify conftest code works as expected
+    """
+
+    @pytest.fixture(autouse=True)
+    def fixtures(self, testdir):
+        """
+        Fixtures to use in tests
+        """
+        self.testdir = testdir
+
+    def makeconftest(self, conftest_rel_path="../conftest.py"):
+        """
+        Copy tests/conftest.py contents to the testdir conftest
+        """
+        this_dir = os.path.dirname(__file__)
+        abs_file_path = os.path.join(this_dir, conftest_rel_path)
+        with open(abs_file_path, encoding="utf-8") as conftest:
+            self.testdir.makeconftest(conftest.read())
+
+    def inline_run(self, *args):
+        """
+        Override inline_run to use our conftest
+        """
+        with override_env({"DD_API_KEY": "foobar.baz"}):
+            self.makeconftest()
+            return self.testdir.inline_run(*args)
+
+    def test_subprocess_fail(self):
+        """
+        Test a failing test with subprocess decorator
+        """
+        py_file = self.testdir.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.subprocess()
+            def test_fail():
+                assert False
+            """
+        )
+        file_name = os.path.basename(py_file.strpath)
+        rec = self.inline_run(file_name)
+        rec.assertoutcome(passed=0, failed=1, skipped=0)
+
+    def test_subprocess_pass(self):
+        """
+        Test a passing test with subprocess decorator
+        """
+        py_file = self.testdir.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.subprocess()
+            def test_pass():
+                assert True
+            """
+        )
+        file_name = os.path.basename(py_file.strpath)
+        rec = self.inline_run(file_name)
+        rec.assertoutcome(passed=1, failed=0, skipped=0)
+
+    def test_subprocess_skip(self):
+        """
+        Test a skipping test with subprocess decorator
+        """
+        py_file = self.testdir.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.skip
+            @pytest.mark.subprocess()
+            def test_pass():
+                assert True
+            """
+        )
+        file_name = os.path.basename(py_file.strpath)
+        rec = self.inline_run(file_name)
+        rec.assertoutcome(passed=0, failed=0, skipped=1)