issue-1795: backpressure for LargeDeletionMarkers #5597
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR-check | |
| on: | |
| pull_request_target: | |
| branches: | |
| - 'main' | |
| paths-ignore: | |
| - 'ydb/docs/**' | |
| - '.github/**' | |
| - 'example/**' | |
| - 'doc/**' | |
| - '**.md' | |
| types: | |
| - 'opened' | |
| - 'synchronize' | |
| - 'reopened' | |
| - 'labeled' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-running-allowed: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| result: ${{ steps.check-ownership-membership.outputs.result }} | |
| steps: | |
| - name: Check if running tests is allowed | |
| id: check-ownership-membership | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| script: | | |
| // How to interpret membership status code: | |
| // https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#check-organization-membership-for-a-user | |
| const userLogin = context.payload.pull_request.user.login; | |
| const isOrgMember = async function () { | |
| try { | |
| const response = await github.rest.orgs.checkMembershipForUser({ | |
| org: context.payload.organization.login, | |
| username: userLogin, | |
| }); | |
| return response.status == 204; | |
| } catch (error) { | |
| if (error.status && error.status == 404) { | |
| return false; | |
| } | |
| throw error; | |
| } | |
| } | |
| if (context.payload.repository.owner.login == userLogin) { | |
| console.log('User is repo owner') | |
| return true; | |
| } | |
| if (await isOrgMember()) { | |
| console.log('User is member') | |
| return true; | |
| } | |
| const labels = context.payload.pull_request.labels; | |
| const okToTestLabel = labels.find( | |
| label => label.name == 'ok-to-test' | |
| ); | |
| return okToTestLabel !== undefined; | |
| - name: comment-if-waiting-on-ok | |
| if: steps.check-ownership-membership.outputs.result == 'false' && | |
| github.event.action == 'opened' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: 'Hi! Thank you for contributing!\nThe tests on this PR will run after a maintainer adds an `ok-to-test` label to this PR manually. Thank you for your patience!' | |
| }); | |
| - name: cleanup-labels | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| let labelsToRemove = ['ok-to-test', 'recheck']; | |
| const prNumber = context.payload.pull_request.number; | |
| const prLabels = new Set(context.payload.pull_request.labels.map(l => l.name)); | |
| for await (const label of labelsToRemove.filter(l => prLabels.has(l))) { | |
| core.info(`remove label=${label} for pr=${prNumber}`); | |
| try { | |
| const result = await github.rest.issues.removeLabel({ | |
| ...context.repo, | |
| issue_number: prNumber, | |
| name: label | |
| }); | |
| } catch(error) { | |
| // ignore the 404 error that arises | |
| // when the label did not exist for the | |
| // organization member | |
| if (error.status && error.status != 404) { | |
| throw error; | |
| } | |
| } | |
| } | |
| create-build-and-test-target-var: | |
| needs: | |
| - check-running-allowed | |
| if: needs.check-running-allowed.outputs.result == 'true' | |
| outputs: | |
| build_target: ${{ steps.set-build-and-test-targets.outputs.build_target }} | |
| build_target_asan: ${{ steps.set-build-and-test-targets.outputs.build_target_asan }} | |
| build_target_tsan: ${{ steps.set-build-and-test-targets.outputs.build_target_tsan }} | |
| build_target_msan: ${{ steps.set-build-and-test-targets.outputs.build_target_msan }} | |
| build_target_ubsan: ${{ steps.set-build-and-test-targets.outputs.build_target_ubsan }} | |
| test_target: ${{ steps.set-build-and-test-targets.outputs.test_target }} | |
| test_target_asan: ${{ steps.set-build-and-test-targets.outputs.test_target_asan }} | |
| test_target_tsan: ${{ steps.set-build-and-test-targets.outputs.test_target_tsan }} | |
| test_target_msan: ${{ steps.set-build-and-test-targets.outputs.test_target_msan }} | |
| test_target_ubsan: ${{ steps.set-build-and-test-targets.outputs.test_target_ubsan }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set build and test targets | |
| id: set-build-and-test-targets | |
| shell: bash | |
| run: | | |
| # Initialize variables | |
| build_target_components="" | |
| test_target_components="" | |
| true_count=0 | |
| false_count=0 | |
| # Function to add components | |
| add_components() { | |
| local component=$1 | |
| local contains=$2 | |
| local build_component=$3 | |
| local test_component=$4 | |
| case "$component" in | |
| "blockstore"|"filestore") | |
| should_asan="true" | |
| should_tsan="true" | |
| should_msan="true" | |
| should_ubsan="true" | |
| ;; | |
| *) | |
| should_asan="false" | |
| should_tsan="false" | |
| should_msan="false" | |
| should_ubsan="false" | |
| ;; | |
| esac | |
| if [ "$contains" = "true" ]; then | |
| build_target_components="${build_target_components}${build_component}," | |
| test_target_components="${test_target_components}${test_component}," | |
| if [ "$should_asan" = "true" ] && [ "$has_asan_label" = "true" ]; then | |
| build_target_asan="${build_target_asan}${build_component}," | |
| test_target_asan="${test_target_asan}${test_component}," | |
| fi | |
| if [ "$should_tsan" = "true" ] && [ "$has_tsan_label" = "true" ]; then | |
| build_target_tsan="${build_target_tsan}${build_component}," | |
| test_target_tsan="${test_target_tsan}${test_component}," | |
| fi | |
| if [ "$should_msan" = "true" ] && [ "$has_msan_label" = "true" ]; then | |
| build_target_msan="${build_target_msan}${build_component}," | |
| test_target_msan="${test_target_msan}${test_component}," | |
| fi | |
| if [ "$should_ubsan" = "true" ] && [ "$has_ubsan_label" = "true" ]; then | |
| build_target_ubsan="${build_target_ubsan}${build_component}," | |
| test_target_ubsan="${test_target_ubsan}${test_component}," | |
| fi | |
| true_count=$((true_count + 1)) | |
| else | |
| false_count=$((false_count + 1)) | |
| fi | |
| } | |
| # Add components based on conditions | |
| add_components "blockstore" "$contains_blockstore" "cloud/blockstore/apps/" "cloud/blockstore/" | |
| add_components "filestore" "$contains_filestore" "cloud/filestore/apps/" "cloud/filestore/" | |
| add_components "disk_manager" "$contains_disk_manager" "cloud/disk_manager/" "cloud/disk_manager/" | |
| add_components "tasks" "$contains_tasks" "cloud/tasks/" "cloud/tasks/" | |
| # Remove trailing commas | |
| build_target_components=${build_target_components%,} | |
| test_target_components=${test_target_components%,} | |
| # Determine build_target and test_target based on conditions | |
| if [ "$true_count" -eq 4 ] || [ "$false_count" -eq 4 ]; then | |
| build_target="cloud/blockstore/apps/,cloud/filestore/apps/,cloud/disk_manager/,cloud/tasks/" | |
| build_target_asan="cloud/blockstore/apps/,cloud/filestore/apps/" | |
| build_target_tsan="cloud/blockstore/apps/,cloud/filestore/apps/" | |
| build_target_msan="cloud/blockstore/apps/,cloud/filestore/apps/" | |
| build_target_ubsan="cloud/blockstore/apps/,cloud/filestore/apps/" | |
| test_target="cloud/blockstore/,cloud/filestore/,cloud/disk_manager/,cloud/tasks/" | |
| test_target_asan="cloud/blockstore/,cloud/filestore/" | |
| test_target_tsan="cloud/blockstore/,cloud/filestore/" | |
| test_target_msan="cloud/blockstore/,cloud/filestore/" | |
| test_target_ubsan="cloud/blockstore/,cloud/filestore/" | |
| else | |
| build_target=$build_target_components | |
| test_target=$test_target_components | |
| fi | |
| # Output to GitHub environment file | |
| echo "build_target=\"$build_target\"" >> $GITHUB_OUTPUT | |
| echo "build_target_asan=\"$build_target_asan\"" >> $GITHUB_OUTPUT | |
| echo "build_target_tsan=\"$build_target_tsan\"" >> $GITHUB_OUTPUT | |
| echo "build_target_msan=\"$build_target_msan\"" >> $GITHUB_OUTPUT | |
| echo "build_target_ubsan=\"$build_target_ubsan\"" >> $GITHUB_OUTPUT | |
| echo "test_target=\"$test_target\"" >> $GITHUB_OUTPUT | |
| echo "test_target_asan=\"$test_target_asan\"" >> $GITHUB_OUTPUT | |
| echo "test_target_tsan=\"$test_target_tsan\"" >> $GITHUB_OUTPUT | |
| echo "test_target_msan=\"$test_target_msan\"" >> $GITHUB_OUTPUT | |
| echo "test_target_ubsan=\"$test_target_ubsan\"" >> $GITHUB_OUTPUT | |
| env: | |
| contains_blockstore: ${{ contains(github.event.pull_request.labels.*.name, 'blockstore') && 'true' || 'false' }} | |
| contains_filestore: ${{ contains(github.event.pull_request.labels.*.name, 'filestore') && 'true' || 'false' }} | |
| contains_disk_manager: ${{ contains(github.event.pull_request.labels.*.name, 'disk_manager') && 'true' || 'false' }} | |
| contains_tasks: ${{ contains(github.event.pull_request.labels.*.name, 'tasks') && 'true' || 'false' }} | |
| has_asan_label: ${{ contains(github.event.pull_request.labels.*.name, 'asan') && 'true' || 'false' }} | |
| has_tsan_label: ${{ contains(github.event.pull_request.labels.*.name, 'tsan') && 'true' || 'false' }} | |
| has_msan_label: ${{ contains(github.event.pull_request.labels.*.name, 'msan') && 'true' || 'false' }} | |
| has_ubsan_label: ${{ contains(github.event.pull_request.labels.*.name, 'ubsan') && 'true' || 'false' }} | |
| build_and_test: | |
| needs: | |
| - check-running-allowed | |
| - create-build-and-test-target-var | |
| if: needs.check-running-allowed.outputs.result == 'true' | |
| name: Build and test | |
| uses: ./.github/workflows/build_and_test_on_demand.yaml | |
| with: | |
| build_target: ${{ needs.create-build-and-test-target-var.outputs.build_target }} | |
| test_target: ${{ needs.create-build-and-test-target-var.outputs.test_target }} | |
| build_preset: "relwithdebinfo" | |
| test_size: ${{ contains(github.event.pull_request.labels.*.name, 'large-tests') && 'small,medium,large' || 'small,medium' }} | |
| test_type: "unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,go_test,gofmt" | |
| run_tests: true | |
| cache_update_build: false | |
| cache_update_tests: false | |
| sleep_after_tests: ${{ contains(github.event.pull_request.labels.*.name, 'sleep') && '7200' || '1' }} | |
| secrets: inherit | |
| build_and_test_asan: | |
| needs: | |
| - check-running-allowed | |
| - create-build-and-test-target-var | |
| if: needs.check-running-allowed.outputs.result == 'true' && contains(github.event.pull_request.labels.*.name, 'asan') | |
| name: Build and test (asan) | |
| uses: ./.github/workflows/build_and_test_on_demand.yaml | |
| with: | |
| build_target: ${{ needs.create-build-and-test-target-var.outputs.build_target_asan }} | |
| test_target: ${{ needs.create-build-and-test-target-var.outputs.test_target_asan }} | |
| build_preset: release-asan | |
| vm_name_suffix: "-asan" | |
| test_size: ${{ contains(github.event.pull_request.labels.*.name, 'large-tests') && 'small,medium,large' || 'small,medium' }} | |
| test_type: "unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,gofmt" | |
| run_tests: true | |
| cache_update_build: false | |
| cache_update_tests: false | |
| sleep_after_tests: ${{ contains(github.event.pull_request.labels.*.name, 'sleep') && '7200' || '1' }} | |
| secrets: inherit | |
| build_and_test_tsan: | |
| needs: | |
| - check-running-allowed | |
| - create-build-and-test-target-var | |
| if: needs.check-running-allowed.outputs.result == 'true' && contains(github.event.pull_request.labels.*.name, 'tsan') | |
| name: Build and test (tsan) | |
| uses: ./.github/workflows/build_and_test_on_demand.yaml | |
| with: | |
| build_target: ${{ needs.create-build-and-test-target-var.outputs.build_target_tsan }} | |
| test_target: ${{ needs.create-build-and-test-target-var.outputs.test_target_tsan }} | |
| build_preset: release-tsan | |
| vm_name_suffix: "-tsan" | |
| test_size: ${{ contains(github.event.pull_request.labels.*.name, 'large-tests') && 'small,medium,large' || 'small,medium' }} | |
| test_type: "unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,gofmt" | |
| run_tests: true | |
| cache_update_build: false | |
| cache_update_tests: false | |
| sleep_after_tests: ${{ contains(github.event.pull_request.labels.*.name, 'sleep') && '7200' || '1' }} | |
| secrets: inherit | |
| build_and_test_msan: | |
| needs: | |
| - check-running-allowed | |
| - create-build-and-test-target-var | |
| if: needs.check-running-allowed.outputs.result == 'true' && contains(github.event.pull_request.labels.*.name, 'msan') | |
| name: Build and test (msan) | |
| uses: ./.github/workflows/build_and_test_on_demand.yaml | |
| with: | |
| build_target: ${{ needs.create-build-and-test-target-var.outputs.build_target_msan }} | |
| test_target: ${{ needs.create-build-and-test-target-var.outputs.test_target_msan }} | |
| build_preset: release-msan | |
| vm_name_suffix: "-msan" | |
| test_size: ${{ contains(github.event.pull_request.labels.*.name, 'large-tests') && 'small,medium,large' || 'small,medium' }} | |
| test_type: "unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,gofmt" | |
| run_tests: true | |
| cache_update_build: false | |
| cache_update_tests: false | |
| sleep_after_tests: ${{ contains(github.event.pull_request.labels.*.name, 'sleep') && '7200' || '1' }} | |
| secrets: inherit | |
| build_and_test_ubsan: | |
| needs: | |
| - check-running-allowed | |
| - create-build-and-test-target-var | |
| if: needs.check-running-allowed.outputs.result == 'true' && contains(github.event.pull_request.labels.*.name, 'ubsan') | |
| name: Build and test (ubsan) | |
| uses: ./.github/workflows/build_and_test_on_demand.yaml | |
| with: | |
| build_target: ${{ needs.create-build-and-test-target-var.outputs.build_target_ubsan }} | |
| test_target: ${{ needs.create-build-and-test-target-var.outputs.test_target_ubsan }} | |
| build_preset: release-ubsan | |
| vm_name_suffix: "-ubsan" | |
| test_size: ${{ contains(github.event.pull_request.labels.*.name, 'large-tests') && 'small,medium,large' || 'small,medium' }} | |
| test_type: "unittest,clang_tidy,gtest,py3test,py2test,pytest,flake8,black,py2_flake8,gofmt" | |
| run_tests: true | |
| cache_update_build: false | |
| cache_update_tests: false | |
| sleep_after_tests: ${{ contains(github.event.pull_request.labels.*.name, 'sleep') && '7200' || '1' }} | |
| secrets: inherit |