ci: display backtrace from core dumps (#8675)

We make it easier to debug segmentation faults and other crashes in CI
that can generate a core dump. We rely on gdb to display the complete
backtraces available from any generated core dumps during test runs in
Circle CI.

## Checklist

- [x] Change(s) are motivated and described in the PR description
- [x] Testing strategy is described if automated tests are not included
in the PR
- [x] Risks are described (performance impact, potential for breakage,
maintainability)
- [x] Change is maintainable (easy to change, telemetry, documentation)
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed or label `changelog/no-changelog` is set
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/))
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))
- [x] If this PR changes the public interface, I've notified
`@DataDog/apm-tees`.
- [x] If change touches code that signs or publishes builds or packages,
or handles credentials of any kind, I've requested a review from
`@DataDog/security-design-and-guidance`.

## Reviewer Checklist

- [ ] Title is accurate
- [ ] All changes are related to the pull request's stated goal
- [ ] Description motivates each change
- [ ] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- [ ] Testing strategy adequately addresses listed risks
- [ ] Change is maintainable (easy to change, telemetry, documentation)
- [ ] Release note makes sense to a user of the library
- [ ] Author has acknowledged and discussed the performance implications
of this PR as reported in the benchmarks PR comment
- [ ] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

.circleci/config.templ.yml

@@ -141,12 +141,22 @@ commands:
             - start_docker_services:
                 env: SNAPSHOT_CI=1
                 services: testagent << parameters.docker_services >>
+            - run: ulimit -c unlimited
             - run:
                 environment:
                   DD_TRACE_AGENT_URL: << parameters.trace_agent_url >>
                   RIOT_RUN_RECOMPILE_REQS: "<< pipeline.parameters.riot_run_latest >>"
                 command: |
                   ./scripts/run-test-suite '<<parameters.pattern>>' <<pipeline.parameters.coverage>> 1
+            - run:
+                command: |
+                  mkdir -p /tmp/core_dumps
+                  cp core.* /tmp/core_dumps
+                  ./scripts/bt
+                when: on_fail
+            - store_artifacts:
+                name: "Store core dumps"
+                path: /tmp/core_dumps
       - unless:
           condition:
               << parameters.snapshot >>
@@ -167,11 +177,21 @@ commands:
                       command: |
                         echo 'export DD_TRACE_AGENT_URL=<< parameters.trace_agent_url >>' >> "$BASH_ENV"
                         source "$BASH_ENV"
+            - run: ulimit -c unlimited
             - run:
                 environment:
                   RIOT_RUN_RECOMPILE_REQS: "<< pipeline.parameters.riot_run_latest >>"
                 command: |
                   ./scripts/run-test-suite '<<parameters.pattern>>' <<pipeline.parameters.coverage>>
+            - run:
+                command: |
+                  mkdir -p /tmp/core_dumps
+                  cp core.* /tmp/core_dumps
+                  ./scripts/bt
+                when: on_fail
+            - store_artifacts:
+                name: "Store core dumps"
+                path: /tmp/core_dumps
       - when:
           condition:
             and:
@@ -253,12 +273,22 @@ commands:
             - start_docker_services:
                 env: SNAPSHOT_CI=1
                 services: testagent << parameters.docker_services >>
+            - run: ulimit -c unlimited
             - run:
                 name: Run tests
                 environment:
                   DD_TRACE_AGENT_URL: << parameters.trace_agent_url >>
                 command: |
                   ./scripts/run-test-suite-hatch '<<parameters.env>>' 1
+            - run:
+                command: |
+                  mkdir -p /tmp/core_dumps
+                  cp core.* /tmp/core_dumps
+                  ./scripts/bt
+                when: on_fail
+            - store_artifacts:
+                name: "Store core dumps"
+                path: /tmp/core_dumps
       - unless:
           condition:
               << parameters.snapshot >>
@@ -627,11 +657,21 @@ jobs:
       - setup_riot
       - start_docker_services:
           services: ddagent
+      - run: ulimit -c unlimited
       - run:
           environment:
             RIOT_RUN_RECOMPILE_REQS: "<< pipeline.parameters.riot_run_latest >>"
           command: |
             ./scripts/run-test-suite 'integration-latest*' <<pipeline.parameters.coverage>> 1
+      - run:
+          command: |
+            mkdir -p /tmp/core_dumps
+            cp core.* /tmp/core_dumps
+            ./scripts/bt
+          when: on_fail
+      - store_artifacts:
+          name: "Store core dumps"
+          path: /tmp/core_dumps
 
   integration_testagent:
     <<: *machine_executor

scripts/bt

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+for core in core.*
+do
+    echo "========  Core dump: ${core}  ========"
+    binary=$(echo ${core} | sed -e 's/core\.[0-9]*\.//' -e 's/!/\//g')
+    echo "Binary: ${binary}"
+    echo "Backtrace:"
+    gdb -q -batch -ex "bt full" -ex q ${binary} ${core}
+done

scripts/run-test-suite

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -o pipefail
+
 CHECKPOINT_FILENAME="latest-success-commit"
 RIOT_PATTERN=${1}
 DDTRACE_FLAG=$([ -v _CI_DD_API_KEY ] && echo '--ddtrace')
@@ -39,18 +41,19 @@ if ! [[ -v CIRCLECI && $CIRCLE_BRANCH =~ main ]]; then
             echo "None of the changes on this branch since that commit affect the $CIRCLE_JOB job."
             echo "Skipping this job."
             circleci step halt
-            exit 0
         fi
     fi
 fi
 
 for hash in ${RIOT_HASHES[@]}; do
     echo "Running riot hash: $hash"
-    if ! $DDTEST_CMD riot -P -v run --exitfirst --pass-env -s $hash $DDTRACE_FLAG $COVERAGE_FLAG; then
+    ($DDTEST_CMD riot -P -v run --exitfirst --pass-env -s $hash $DDTRACE_FLAG $COVERAGE_FLAG)
+    exit_code=$?
+    if [ exit_code -ne 0 ] ; then
         if [[ -v CIRCLECI ]]; then
             circleci step halt
         fi
-        exit 1
+        exit $exit_code
     fi
 done