Thread Sanitizer analysis #3797
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Thread Sanitizer analysis | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| fastdds_branch: | |
| description: 'Branch or tag of Fast DDS repository (https://github.com/eProsima/Fast-DDS)' | |
| required: true | |
| default: 'master' | |
| optional_cmake_args: | |
| description: 'Optional CMake Compilation Flags' | |
| required: false | |
| type: string | |
| optional_ctest_args: | |
| description: 'Optional CTest Testing Flags' | |
| required: false | |
| type: string | |
| pull_request: | |
| types: | |
| - review_requested | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| - '!**/CMakeLists.txt' | |
| schedule: | |
| - cron: '0 1 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| ubuntu-sanitizer-run: | |
| name: Sanitizer Evaluation | |
| if: ${{ ( | |
| !contains(github.event.pull_request.labels.*.name, 'skip-ci') && | |
| !contains(github.event.pull_request.labels.*.name, 'no-test') && | |
| !contains(github.event.pull_request.labels.*.name, 'conflicts') | |
| ) }} | |
| runs-on: ubuntu-22.04 | |
| env: | |
| FASTDDS_BRANCH: ${{ github.head_ref || github.event.inputs.fastdds_branch || 'master' }} | |
| TSAN_OPTIONS: second_deadlock_stack=1 history_size=7 memory_limit_mb=5000 | |
| # GCC 11.3 (Ubuntu Jammy default) produces several false positives regarding timed synchronization protocols | |
| # These issues were fixed in GCC 12 so we upgrade to that version. | |
| CC: gcc-12 | |
| CXX: g++-12 | |
| steps: | |
| - name: Add ci-pending label if PR | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: eProsima/eProsima-CI/external/add_labels@v0 | |
| with: | |
| labels: ci-pending | |
| number: ${{ github.event.number }} | |
| repo: eProsima/Fast-DDS | |
| # https://github.com/actions/runner-images/issues/9491 | |
| - name: Fix kernel mmap rnd bits | |
| run: sudo sysctl vm.mmap_rnd_bits=28 | |
| - name: Install apt packages | |
| uses: eProsima/eProsima-CI/ubuntu/install_apt_packages@v0 | |
| with: | |
| packages: curl grep libasio-dev libtinyxml2-dev python3 python3-pip software-properties-common wget | |
| - name: Get minimum supported version of CMake | |
| uses: eProsima/eProsima-CI/external/get-cmake@v0 | |
| with: | |
| cmakeVersion: '3.22.6' | |
| - name: Setup CCache | |
| uses: eProsima/eProsima-CI/external/setup-ccache-action@v0 | |
| - name: Install colcon | |
| uses: eProsima/eProsima-CI/ubuntu/install_colcon@v0 | |
| - name: Install Python dependencies | |
| uses: eProsima/eProsima-CI/ubuntu/install_python_packages@v0 | |
| with: | |
| packages: vcstool setuptools gcovr tomark xmltodict jsondiff pandas | |
| - name: Get fastrtps.repos file | |
| uses: eProsima/eProsima-CI/ubuntu/get_file_from_repo@v0 | |
| with: | |
| source_repository: eProsima/Fast-DDS | |
| source_repository_branch: ${{ env.FASTDDS_BRANCH }} | |
| file_name: fastrtps.repos | |
| file_result: fastrtps.repos | |
| - name: Fetch Fast DDS & dependencies | |
| uses: eProsima/eProsima-CI/ubuntu/vcs_import@v0 | |
| with: | |
| vcs_repos_file: fastrtps.repos | |
| destination_workspace: src | |
| - name: Checkout Fast DDS branch | |
| run: | | |
| cd ./src/fastrtps | |
| git checkout ${{ env.FASTDDS_BRANCH }} | |
| - name: Fetch Fast DDS CI dependencies | |
| uses: eProsima/eProsima-CI/multiplatform/vcs_import@v0 | |
| with: | |
| vcs_repos_file: ${{ github.workspace }}/src/fastrtps/.github/workflows/config/ci.repos | |
| destination_workspace: src | |
| skip_existing: 'true' | |
| - name: Build workspace | |
| run: | | |
| cat src/fastrtps/.github/workflows/config/tsan_colcon.meta | |
| colcon build \ | |
| --event-handlers=console_direct+ \ | |
| --metas src/fastrtps/.github/workflows/config/tsan_colcon.meta \ | |
| --cmake-args -DCMAKE_BUILD_TYPE=Debug ${{ inputs.optional_cmake_args }} | |
| - name: Run tests Fast DDS | |
| run: | | |
| source install/setup.bash && \ | |
| colcon test \ | |
| --packages-select fastrtps \ | |
| --event-handlers=console_direct+ \ | |
| --return-code-on-test-failure \ | |
| --ctest-args \ | |
| --timeout 30 \ | |
| -V -E DDSSQLFilterValueTests ${{ inputs.optional_ctest_args }} | |
| continue-on-error: true | |
| - name: Upload Logs | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: tsan-logs | |
| path: log/ | |
| if: always() | |
| - name: Process sanitizer reports | |
| id: report_summary | |
| shell: pwsh | |
| continue-on-error: true | |
| run: | | |
| # Create a dir for the exports | |
| $exports = New-Item -ItemType Directory -Path ./exports | |
| # Move to the reports dir | |
| pushd ./log/latest_test/fastrtps | |
| # Install the report parser module | |
| Find-Module -Repository PSGallery -Name SanReportParser | Install-Module -Scope CurrentUser -Force | |
| # Parse the report files | |
| $rp = Show-Tsan -Path ./stdout_stderr.log | |
| # filter duplicates | |
| $rp = $rp | group md5hash | % { $_.group[0] } | |
| # Export raw data | |
| $rp | Export-CliXML (Join-Path $exports all_reports.xml) | |
| # Group the reports by issue | |
| $g = $rp | group fuzzhash | |
| # Split up deadlocks and race reports | |
| $gd = $rp | ? type -match dead | group fuzzhash | |
| $gr = $rp | ? type -match race | group fuzzhash | |
| # Simplified deadlock summary (only one representative report and tests associated) | |
| $sd = $gd | Sort-Object count -desc | select @{l="fuzzhash";e="name"}, count, ` | |
| @{l="échantillon";e={$_.group[0].report}}, @{l="tests"; ` | |
| e={$_.group.file | sls "(.*)\.\d+$" | % { $_.Matches.Groups[1].Value } | Sort-Object | get-unique}} | |
| # Simplified race summary (only one representative report and tests associated) | |
| $sr = $gr | Sort-Object count -desc | select @{l="fuzzhash";e="name"}, count, ` | |
| @{l="échantillon";e={$_.group[0].report}}, @{l="tests"; ` | |
| e={$_.group.file | sls "(.*)\.\d+$" | % { $_.Matches.Groups[1].Value } | Sort-Object | get-unique}} | |
| # Export simplified summaries | |
| $sd, $sr | Export-Clixml (Join-Path $exports summary_data.xml) | |
| # Export CSV summary of frequencies | |
| $sd | select fuzzhash, count | Export-CSV -Path (Join-Path $exports deadlocks.csv) | |
| $sr | select fuzzhash, count | Export-CSV -Path (Join-Path $exports races.csv) | |
| # Export test mappings | |
| & {$sd; $sr} | Sort-Object { [int]$_.fuzzhash} | % { $fuzzy=$_.fuzzhash;$_.tests | | |
| select @{l="fuzzhash";e={$fuzzy}}, @{l="test";e={$_}}} | | |
| Export-Csv (Join-Path $exports issue_test_map.csv) | |
| # Keep a file per issue | |
| $dir = New-Item -ItemType Directory -Path (Join-Path $exports reports) | |
| & {$sd; $sr} | % { $_.échantillon | Out-File (Join-Path $dir "$($_.fuzzhash).tsan") } | |
| # Create a summary table | |
| @{Type="Deadlock";Failed=$sd.count;Hashes=$sd.fuzzhash}, | |
| @{Type="Data race";Failed=$sr.count;Hashes=$sr.fuzzhash} | | |
| % { $_.Summary = $_.Hashes | select -First 5 | Join-String -Separator ", " | |
| if ($_.Hashes.count -gt 5 ) { | |
| $_.Summary += ", ..." }; $_ } | | |
| % { [PSCustomObject]$_} | | |
| New-MDTable -Columns ([ordered]@{Failed=$null;Type=$null;Summary=$null}) | | |
| Out-File $Env:GITHUB_STEP_SUMMARY | |
| # The step fails on new deadlocks | |
| $LASTEXITCODE=$sd.count | |
| - name: Archive sanitation results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sanitation-report | |
| path: exports/* | |
| - name: Check on failures | |
| if: ${{ steps.report_summary.outcome == 'failure' }} | |
| shell: pwsh | |
| run: | | |
| Write-Host ${{ steps.report_summary.outcome }} | |
| exit 1 |