[20843] Detect flaky tests Github workflow

.github/workflows/nightly-mac-ci.yml

@@ -2,8 +2,6 @@ name: Fast DDS MacOS CI (nightly)
 
 on:
   workflow_dispatch:
-  schedule:
-    - cron: '0 1 * * *'
 
 jobs:
   nightly-mac-ci-master:
@@ -12,66 +10,10 @@ jobs:
       matrix:
         security:
           - 'ON'
-          - 'OFF'
-    uses: eProsima/Fast-DDS/.github/workflows/reusable-mac-ci.yml@master
+    uses: eProsima/Fast-DDS/.github/workflows/reusable-mac-ci.yml@feature/detect_flaky_tests
     with:
-      label: 'nightly-sec-${{ matrix.security }}-mac-ci-master'
+      label: 'nightly-sec-${{ matrix.security }}-mac-ci-master-flaky'
       cmake-args: "-DSECURITY=${{ matrix.security }}"
       ctest-args: "-LE xfail"
       fastdds-branch: 'master'
-
-  nightly-mac-ci-2_14_x:
-    strategy:
-      fail-fast: false
-      matrix:
-        security:
-          - 'ON'
-          - 'OFF'
-    uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-    with:
-      label: 'nightly-sec-${{ matrix.security }}-mac-ci-2.14.x'
-      cmake-args: "-DSECURITY=${{ matrix.security }}"
-      ctest-args: "-LE xfail"
-      fastdds-branch: '2.14.x'
-
-  nightly-mac-ci-2_13_x:
-    strategy:
-      fail-fast: false
-      matrix:
-        security:
-          - 'ON'
-          - 'OFF'
-    uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-    with:
-      label: 'nightly-sec-${{ matrix.security }}-mac-ci-2.13.x'
-      cmake-args: "-DSECURITY=${{ matrix.security }}"
-      ctest-args: "-LE xfail"
-      fastdds-branch: '2.13.x'
-
-  nightly-mac-ci-2_10_x:
-    strategy:
-      fail-fast: false
-      matrix:
-        security:
-          - 'ON'
-          - 'OFF'
-    uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-    with:
-      label: 'nightly-sec-${{ matrix.security }}-mac-ci-2.10.x'
-      cmake-args: "-DSECURITY=${{ matrix.security }}"
-      ctest-args: "-LE xfail"
-      fastdds-branch: '2.10.x'
-
-  nightly-mac-ci-2_6_x:
-    strategy:
-      fail-fast: false
-      matrix:
-        security:
-          - 'ON'
-          - 'OFF'
-    uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-    with:
-      label: 'nightly-sec-${{ matrix.security }}-mac-ci-2.6.x'
-      cmake-args: "-DSECURITY=${{ matrix.security }}"
-      ctest-args: "-LE xfail"
-      fastdds-branch: '2.6.x'
+      flakiness-report: true

.github/workflows/nightly-ubuntu-ci.yml

@@ -2,8 +2,6 @@ name: Fast DDS Ubuntu CI (nightly)
 
 on:
   workflow_dispatch:
-  schedule:
-    - cron: '0 1 * * *'
 
 jobs:
   nightly-ubuntu-ci-master:
@@ -14,89 +12,13 @@ jobs:
           - 'ubuntu-22.04'
         security:
           - true
-          - false
-    uses: eProsima/Fast-DDS/.github/workflows/reusable-ubuntu-ci.yml@master
+    uses: eProsima/Fast-DDS/.github/workflows/reusable-ubuntu-ci.yml@feature/detect_flaky_tests
     with:
       os-image: ${{ matrix.os-image }}
-      label: '${{ matrix.os-image }}-nightly-sec-${{ matrix.security }}-ubuntu-ci-master'
+      label: '${{ matrix.os-image }}-nightly-sec-${{ matrix.security }}-ubuntu-ci-master-flaky'
       ctest-args: "-LE xfail"
       fastdds-branch: 'master'
       security: ${{ matrix.security }}
       run-tests: true
       use-ccache: true
-
-  nightly-ubuntu-ci-2_14_x:
-    strategy:
-      fail-fast: false
-      matrix:
-        os-image:
-          - 'ubuntu-22.04'
-        security:
-          - true
-          - false
-    uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-    with:
-      os-image: ${{ matrix.os-image }}
-      label: '${{ matrix.os-image }}-nightly-sec-${{ matrix.security }}-ubuntu-ci-2.14.x'
-      ctest-args: "-LE xfail"
-      fastdds-branch: '2.14.x'
-      security: ${{ matrix.security }}
-      run-tests: true
-      use-ccache: true
-
-  nightly-ubuntu-ci-2_13_x:
-    strategy:
-      fail-fast: false
-      matrix:
-        os-image:
-          - 'ubuntu-22.04'
-        security:
-          - true
-          - false
-    uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-    with:
-      os-image: ${{ matrix.os-image }}
-      label: '${{ matrix.os-image }}-nightly-sec-${{ matrix.security }}-ubuntu-ci-2.13.x'
-      ctest-args: "-LE xfail"
-      fastdds-branch: '2.13.x'
-      security: ${{ matrix.security }}
-      run-tests: true
-      use-ccache: true
-
-  nightly-ubuntu-ci-2_10_x:
-    strategy:
-      fail-fast: false
-      matrix:
-        os-image:
-          - 'ubuntu-22.04'
-        security:
-          - true
-          - false
-    uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-    with:
-      os-image: ${{ matrix.os-image }}
-      label: '${{ matrix.os-image }}-nightly-sec-${{ matrix.security }}-ubuntu-ci-2.10.x'
-      ctest-args: "-LE xfail"
-      fastdds-branch: '2.10.x'
-      security: ${{ matrix.security }}
-      run-tests: true
-      use-ccache: true
-
-  nightly-ubuntu-ci-2_6_x:
-    strategy:
-      fail-fast: false
-      matrix:
-        os-image:
-          - 'ubuntu-22.04'
-        security:
-          - true
-          - false
-    uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-    with:
-      os-image: ${{ matrix.os-image }}
-      label: '${{ matrix.os-image }}-nightly-sec-${{ matrix.security }}-ubuntu-ci-2.6.x'
-      ctest-args: "-LE xfail"
-      fastdds-branch: '2.6.x'
-      security: ${{ matrix.security }}
-      run-tests: true
-      use-ccache: true
+      flakiness-report: true

.github/workflows/nightly-windows-ci.yml

@@ -2,8 +2,6 @@ name: Fast DDS Windows CI (nightly)
 
 on:
   workflow_dispatch:
-  schedule:
-    - cron: '0 1 * * *'
 
 jobs:
   nightly-windows-ci-master:
@@ -12,66 +10,10 @@ jobs:
       matrix:
         security:
           - 'ON'
-          - 'OFF'
-    uses: eProsima/Fast-DDS/.github/workflows/reusable-windows-ci.yml@master
+    uses: eProsima/Fast-DDS/.github/workflows/reusable-windows-ci.yml@feature/detect_flaky_tests
     with:
-      label: 'nightly-sec-${{ matrix.security }}-windows-ci-master'
+      label: 'nightly-sec-${{ matrix.security }}-windows-ci-master-flaky'
       cmake-args: "-DSECURITY=${{ matrix.security }}"
       ctest-args: "-LE xfail"
       fastdds_branch: 'master'
-
-  nightly-windows-ci-2_14_x:
-      strategy:
-        fail-fast: false
-        matrix:
-          security:
-            - 'ON'
-            - 'OFF'
-      uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-      with:
-        label: 'nightly-sec-${{ matrix.security }}-windows-ci-2.14.x'
-        cmake-args: "-DSECURITY=${{ matrix.security }}"
-        ctest-args: "-LE xfail"
-        fastdds_branch: '2.14.x'
-
-  nightly-windows-ci-2_13_x:
-    strategy:
-      fail-fast: false
-      matrix:
-        security:
-          - 'ON'
-          - 'OFF'
-    uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-    with:
-      label: 'nightly-sec-${{ matrix.security }}-windows-ci-2.13.x'
-      cmake-args: "-DSECURITY=${{ matrix.security }}"
-      ctest-args: "-LE xfail"
-      fastdds_branch: '2.13.x'
-
-  nightly-windows-ci-2_10_x:
-    strategy:
-      fail-fast: false
-      matrix:
-        security:
-          - 'ON'
-          - 'OFF'
-    uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-    with:
-      label: 'nightly-sec-${{ matrix.security }}-windows-ci-2.10.x'
-      cmake-args: "-DSECURITY=${{ matrix.security }}"
-      ctest-args: "-LE xfail"
-      fastdds_branch: '2.10.x'
-
-  nightly-windows-ci-2_6_x:
-    strategy:
-      fail-fast: false
-      matrix:
-        security:
-          - 'ON'
-          - 'OFF'
-    uses: eProsima/Fast-DDS/.github/workflows/[email protected]
-    with:
-      label: 'nightly-sec-${{ matrix.security }}-windows-ci-2.6.x'
-      cmake-args: "-DSECURITY=${{ matrix.security }}"
-      ctest-args: "-LE xfail"
-      fastdds_branch: '2.6.x'
+      flakiness-report: true

.github/workflows/reusable-mac-ci.yml

@@ -23,6 +23,11 @@ on:
         description: 'Branch or tag of Fast DDS repository (https://github.com/eProsima/Fast-DDS)'
         required: true
         type: string
+      flakiness-report:
+        description: 'Whether to generate a flakiness report'
+        required: false
+        type: boolean
+        default: false
 
 defaults:
   run:
@@ -45,45 +50,45 @@ jobs:
     steps:
       - name: Add ci-pending label if PR
         if: ${{ github.event_name == 'pull_request' }}
-        uses: eProsima/eProsima-CI/external/add_labels@v0
+        uses: eProsima/eProsima-CI/external/add_labels@feature/detect_flaky_tests
         with:
           labels: ci-pending
           number: ${{ github.event.number }}
           repo: eProsima/Fast-DDS
 
       - name: Sync eProsima/Fast-DDS repository
-        uses: eProsima/eProsima-CI/external/checkout@v0
+        uses: eProsima/eProsima-CI/external/checkout@feature/detect_flaky_tests
         with:
           path: src/fastdds
           ref: ${{ inputs.fastdds-branch }}
 
-      - uses: eProsima/eProsima-CI/external/setup-python@v0
+      - uses: eProsima/eProsima-CI/external/setup-python@feature/detect_flaky_tests
         with:
           python-version: '3.11'
 
       - name: Get minimum supported version of CMake
-        uses: eProsima/eProsima-CI/external/get-cmake@v0
+        uses: eProsima/eProsima-CI/external/get-cmake@feature/detect_flaky_tests
         with:
           cmakeVersion: '3.22.6'
 
       - name: Install brew dependencies
-        uses: eProsima/eProsima-CI/macos/install_brew_packages@v0
+        uses: eProsima/eProsima-CI/macos/install_brew_packages@feature/detect_flaky_tests
         with:
           packages: llvm asio tinyxml2 [email protected]
           update: false
           upgrade: false
 
       - name: Install colcon
-        uses: eProsima/eProsima-CI/multiplatform/install_colcon@v0
+        uses: eProsima/eProsima-CI/multiplatform/install_colcon@feature/detect_flaky_tests
 
       - name: Install Python dependencies
-        uses: eProsima/eProsima-CI/multiplatform/install_python_packages@v0
+        uses: eProsima/eProsima-CI/multiplatform/install_python_packages@feature/detect_flaky_tests
         with:
           packages: vcstool xmlschema
           upgrade: false
 
       - name: Setup CCache
-        uses: eProsima/eProsima-CI/external/setup-ccache-action@v0
+        uses: eProsima/eProsima-CI/external/setup-ccache-action@feature/detect_flaky_tests
         with:
           api_token: ${{ secrets.GITHUB_TOKEN }}
 
@@ -103,14 +108,14 @@ jobs:
       #                which entails logout/login or rebooting the machine. This is not feasible in a CI environment.
 
       - name: Fetch Fast DDS dependencies
-        uses: eProsima/eProsima-CI/multiplatform/vcs_import@v0
+        uses: eProsima/eProsima-CI/multiplatform/vcs_import@feature/detect_flaky_tests
         with:
           vcs_repos_file: ${{ github.workspace }}/src/fastdds/fastdds.repos
           destination_workspace: src
           skip_existing: 'true'
 
       - name: Fetch Fast DDS CI dependencies
-        uses: eProsima/eProsima-CI/multiplatform/vcs_import@v0
+        uses: eProsima/eProsima-CI/multiplatform/vcs_import@feature/detect_flaky_tests
         with:
           vcs_repos_file: ${{ github.workspace }}/src/fastdds/.github/workflows/config/fastdds_test.repos
           destination_workspace: src
@@ -122,7 +127,7 @@ jobs:
       #   - Not working solution: https://github.com/macports/macports-ports/pull/21839/files
       - name: Colcon build
         continue-on-error: false
-        uses: eProsima/eProsima-CI/multiplatform/colcon_build@v0
+        uses: eProsima/eProsima-CI/multiplatform/colcon_build@feature/detect_flaky_tests
         with:
           colcon_meta_file: ${{ github.workspace }}/src/fastdds/.github/workflows/config/fastdds_build.meta ${{ github.workspace }}/src/fastdds/.github/workflows/config/fastdds_test.meta
           colcon_build_args: ${{ inputs.colcon-args }}
@@ -134,22 +139,40 @@ jobs:
       - name: Colcon test
         id: test
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-test') }}
-        uses: eProsima/eProsima-CI/multiplatform/colcon_test@v0
+        uses: eProsima/eProsima-CI/multiplatform/colcon_test@feature/detect_flaky_tests
         with:
           colcon_meta_file: ${{ github.workspace }}/src/fastdds/.github/workflows/config/fastdds_test.meta
           colcon_test_args: ${{ inputs.colcon-args }}
           colcon_test_args_default: --event-handlers=console_direct+
-          ctest_args: ${{ inputs.ctest-args }}
+          ctest_args: ${{ inputs.ctest-args }} -R DynamicTypes
           packages_names: fastdds
           workspace: ${{ github.workspace }}
           test_report_artifact: ${{ format('test_report_{0}_{1}_{2}', inputs.label, github.job, join(matrix.*, '_')) }}
 
+      - name: Flakiness report
+        uses: eProsima/eProsima-CI/multiplatform/flakiness_report@feature/detect_flaky_tests
+        if: ${{ inputs.flakiness-report }}
+        with:
+          junit_reports_dir: "${{ steps.test.outputs.ctest_results_path }}"
+          junit_archive_artifact: ${{ format('flakiness_archive_{0}_{1}_{2}', inputs.label, github.job, join(matrix.*, '_')) }}
+          print_report: 'True'
+          window_size: '30'
+          fail_on_flaky_tests: 'False'
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download Flakiness report
+        uses: eProsima/eProsima-CI/external/download-artifact@feature/detect_flaky_tests
+        with:
+          name: ${{ format('flakiness_archive_{0}_{1}_{2}_json', inputs.label, github.job, join(matrix.*, '_')) }}
+          path: ${{ github.workspace }}/flakiness_report.json
+
       - name: Test summary
-        uses: eProsima/eProsima-CI/multiplatform/junit_summary@v0
+        uses: eProsima/eProsima-CI/multiplatform/junit_summary@feature/detect_flaky_tests
         if: ${{ !cancelled() && !contains(github.event.pull_request.labels.*.name, 'no-test') }}
         with:
           junit_reports_dir: "${{ steps.test.outputs.ctest_results_path }}"
           print_summary: 'True'
           show_failed: 'True'
           show_disabled: 'False'
           show_skipped: 'False'
+          flaky_json_report: "${{ github.workspace }}/flakiness_report.json"