Merge branch 'main' into romain.komorn/SDTEST-225/use_bytearrays_for_…

…coverage

.circleci/config.templ.yml

@@ -573,14 +573,6 @@ jobs:
       - run_test:
           pattern: "datastreams"
 
-  tracer:
-    parallelism: 10
-    <<: *contrib_job_large
-    steps:
-      - run_test:
-          pattern: "tracer"
-          trace_agent_url: ""
-
   ci_visibility:
     <<: *machine_executor
     parallelism: 4

.github/workflows/generate-package-versions.yml

@@ -0,0 +1,84 @@
+name: Generate Package Versions
+
+on:
+  workflow_dispatch: # can be triggered manually
+  schedule:
+    - cron: '0 0 * * 0' # weekly on Sunday at midnight
+
+
+jobs:
+  generate-package-versions:
+    name: Generate package versions
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python 3.7
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.7"
+
+      - name: Setup Python 3.8
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"
+
+      - name: Setup Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+
+      - name: Setup Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Setup Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Setup Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Install Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+          docker-compose --version
+
+      - name: Start docker service
+        run: docker-compose up -d testagent
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libmariadb-dev
+          
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install packaging
+          pip install riot==0.19.1
+          pip install hatch~=1.8.0 hatch-containers==0.7.0
+
+      - name: Run regenerate-riot-latest
+        run: scripts/regenerate-riot-latest.sh
+
+      - name: Create Pull Request
+        id: pr
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          branch: "upgrade-${{ env.VENV_NAME }}-version"
+          commit-message: "Test Package Versions update"
+          delete-branch: true
+          base: main
+          title: "chore: update ${{ env.VENV_NAME }} latest version"
+          body: |
+            Updates the ${{ env.VENV_NAME }} integration latest version and regenerates lockfiles.

.github/workflows/testrunner.yml

@@ -1,6 +1,7 @@
 name: Testrunner
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - 'main'

.gitlab-ci.yml

@@ -6,6 +6,7 @@ stages:
   - benchmarks
   - macrobenchmarks
   - dogfood
+  - release
 
 variables:
   REPO_LANG: python # "python" is used everywhere rather than "py"
@@ -26,7 +27,7 @@ include:
   - local: ".gitlab/dogfood.yml"
 
 package-oci:
-  needs: [ download_dependency_wheels, download_ddtrace_wheels ]
+  needs: [ download_dependency_wheels, download_ddtrace_artifacts ]
 
 onboarding_tests_installer:
   parallel:
@@ -55,3 +56,16 @@ deploy_to_di_backend:manual:
     UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR
     UPSTREAM_TAG: $CI_COMMIT_TAG
     UPSTREAM_PACKAGE_JOB: build
+
+
+# Final step which only runs when a pipeline has finished successfully.
+# This gives us something block on/wait for in GitHub
+finished:
+  image: registry.ddbuild.io/images/mirror/library/alpine:3.19.3
+  tags: [ "arch:amd64" ]
+  stage: .post
+  rules:
+    - when: on_success
+  script:
+    # TODO: Can we get this to reflect the status from the whole pipeline?
+    - exit 0

.gitlab/package.yml

@@ -17,7 +17,7 @@ build_base_venvs:
       - ddtrace/**/*.so*
       - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe
 
-download_ddtrace_wheels:
+download_ddtrace_artifacts:
   image: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0
   tags: [ "arch:amd64" ]
   stage: package
@@ -29,12 +29,13 @@ download_ddtrace_wheels:
   artifacts:
     paths:
       - "pywheels/*.whl"
+      - "pywheels/*.tar.gz"
 
 download_dependency_wheels:
   image: registry.ddbuild.io/images/mirror/python:$PYTHON_IMAGE_TAG
   tags: [ "arch:amd64" ]
   stage: package
-  needs: [ download_ddtrace_wheels ]
+  needs: [ download_ddtrace_artifacts ]
   parallel:
     matrix: # The image tags that are mirrored are in: https://github.com/DataDog/images/blob/master/mirror.yaml
       - PYTHON_IMAGE_TAG: "3.7"

.gitlab/release.yml

@@ -0,0 +1,42 @@
+variables:
+  PYPI_PUBLISH_IMAGE: registry.ddbuild.io/images/mirror/python:3.12.0
+
+.release_base:
+  stage: release
+  only:
+    # v2.10.0
+    # v2.10.1
+    # v2.10.0rc0
+    # v2.10.0rc5
+    - /^v[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]+)?$/
+
+.release_pypi:
+  extends: .release_base
+  image: ${PYPI_PUBLISH_IMAGE}
+  tags: [ "arch:amd64" ]
+  variables:
+    TWINE_USERNAME: "__token__"
+    TWINE_NON_INTERACTIVE: "1"
+  before_script:
+    - export TWINE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name "ci.${CI_PROJECT_NAME}.${PYPI_REPOSITORY}_token" --with-decryption --query "Parameter.Value" --out text)
+    - python -m pip install twine
+    - python -m twine check --strict pywheels/*
+  script:
+    - echo "python -m twine upload --repository ${PYPI_REPOSITORY} pywheels/*"
+  artifacts:
+    paths:
+      - pywheels/*.whl
+      - pywheels/*.tar.gz
+
+release_pypi_test:
+  extends: .release_pypi
+  dependencies: [ "download_ddtrace_wheels" ]
+  variables:
+    PYPI_REPOSITORY: testpypi
+
+# TODO: Replace GitHub Action PyPI upload with this job
+# release_pypi_prod:
+#   extends: .release_pypi
+#   needs: [ "release_pypi_test" ]
+#   variables:
+#     PYPI_REPOSITORY: pypi

.gitlab/tests.yml

@@ -1,3 +1,6 @@
+variables:
+  RIOT_RUN_CMD: riot -P -v run --exitfirst --pass-env -s
+
 .test_base_hatch:
   extends: .testrunner
   stage: tests
@@ -7,5 +10,26 @@
   script:
     - hatch env show --json | jq -r --arg suite_name "$SUITE_NAME" 'keys[] | select(. | contains($suite_name))' | sort | ./.gitlab/ci-split-input.sh | xargs -n 1 -I {} hatch run {}:test
 
+.test_base_riot:
+  extends: .testrunner
+  stage: tests
+  needs: [ build_base_venvs ]
+  parallel: 4
+  script:
+    - pip install riot~=0.19.1
+    - unset DD_SERVICE
+    - unset DD_ENV
+    - unset DD_TAGS
+    - unset DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED
+    - |
+      hashes=( $(riot list --hash-only "${SUITE_NAME}" | sort | ./.gitlab/ci-split-input.sh) )
+      for hash in "${hashes[@]}"
+      do
+        echo "Running riot hash: ${hash}"
+        riot list "${hash}"
+        ${RIOT_RUN_CMD} "${hash}"
+      done
+
 include:
   - local: ".gitlab/tests/appsec.yml"
+  - local: ".gitlab/tests/tracer.yml"

.gitlab/tests/tracer.yml

@@ -0,0 +1,5 @@
+tracer:
+  extends: .test_base_riot
+  parallel: 9
+  variables:
+    SUITE_NAME: "tracer"

ddtrace/appsec/_iast/_ast/visitor.py

@@ -50,7 +50,6 @@ def _mark_avoid_convert_recursively(node):
         "join": "ddtrace_aspects.join_aspect",
         "encode": "ddtrace_aspects.encode_aspect",
         "extend": "ddtrace_aspects.bytearray_extend_aspect",
-        "findall": "ddtrace_aspects.re_findall_aspect",
         "upper": "ddtrace_aspects.upper_aspect",
         "lower": "ddtrace_aspects.lower_aspect",
         "replace": "ddtrace_aspects.replace_aspect",
@@ -63,13 +62,20 @@ def _mark_avoid_convert_recursively(node):
         "format_map": "ddtrace_aspects.format_map_aspect",
         "zfill": "ddtrace_aspects.zfill_aspect",
         "ljust": "ddtrace_aspects.ljust_aspect",
-        "split": "ddtrace_aspects.split_aspect",
+        "split": "ddtrace_aspects.split_aspect",  # Both regular split and re.split
         "rsplit": "ddtrace_aspects.rsplit_aspect",
         "splitlines": "ddtrace_aspects.splitlines_aspect",
+        # re module and re.Match methods
+        "findall": "ddtrace_aspects.re_findall_aspect",
+        "finditer": "ddtrace_aspects.re_finditer_aspect",
+        "fullmatch": "ddtrace_aspects.re_fullmatch_aspect",
+        "expand": "ddtrace_aspects.re_expand_aspect",
+        "group": "ddtrace_aspects.re_group_aspect",
+        "groups": "ddtrace_aspects.re_groups_aspect",
+        "match": "ddtrace_aspects.re_match_aspect",
+        "search": "ddtrace_aspects.re_search_aspect",
         "sub": "ddtrace_aspects.re_sub_aspect",
         "subn": "ddtrace_aspects.re_subn_aspect",
-        "match": "ddtrace_aspects.re_match_aspect",
-        "groups": "ddtrace_aspects.re_groups_aspect",
     },
     # Replacement function for indexes and ranges
     "slices": {

ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectIndex.cpp

@@ -46,15 +46,14 @@ api_index_aspect(PyObject* self, PyObject* const* args, const Py_ssize_t nargs)
         return nullptr;
     }
 
-    try {
-        PyObject* result_o = nullptr;
-        PyObject* candidate_text = args[0];
-        PyObject* idx = args[1];
-        if (!is_text(candidate_text) or !is_some_number(idx)) {
-            return PyObject_GetItem(candidate_text, idx);
-        }
+    PyObject* candidate_text = args[0];
+    PyObject* idx = args[1];
+    auto result_o = PyObject_GetItem(candidate_text, idx);
+    if (!is_text(candidate_text) or !is_some_number(idx)) {
+        return result_o;
+    }
+    TRY_CATCH_ASPECT("index_aspect", {
         const auto ctx_map = initializer->get_tainting_map();
-        result_o = PyObject_GetItem(candidate_text, idx);
         if (not ctx_map or ctx_map->empty()) {
             return result_o;
         }
@@ -68,15 +67,5 @@ api_index_aspect(PyObject* self, PyObject* const* args, const Py_ssize_t nargs)
         }
 
         return index_aspect(result_o, candidate_text, idx, ctx_map);
-    } catch (const std::exception& e) {
-        const std::string error_message = "IAST propagation error in index_aspect. " + std::string(e.what());
-        iast_taint_log_error(error_message);
-        py::set_error(PyExc_TypeError, error_message.c_str());
-        return nullptr;
-    } catch (...) {
-        const std::string error_message = "Unkown IAST propagation error in index_aspect. ";
-        iast_taint_log_error(error_message);
-        py::set_error(PyExc_TypeError, error_message.c_str());
-        return nullptr;
-    }
+    });
 }