Simulator Tests #114
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Simulator Tests | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| on: | |
| schedule: | |
| - cron: '0 9 * * *' # equivalent to 1am PST | |
| workflow_dispatch: | |
| inputs: | |
| sui_ref: | |
| description: "Branch / commit to test" | |
| type: string | |
| required: true | |
| default: main | |
| test_num: | |
| description: "MSIM_TEST_NUM (test iterations)" | |
| type: string | |
| required: false | |
| default: "30" | |
| env: | |
| SUI_REF: "${{ github.event.inputs.sui_ref || 'main' }}" | |
| TEST_NUM: "${{ github.event.inputs.test_num || '30' }}" | |
| jobs: | |
| simtest: | |
| # Allow running the job for up to 6 hours, maximum for GitHub Actions. | |
| timeout-minutes: 360 | |
| permissions: | |
| # The "id-token: write" permission is required or Machine ID will not be | |
| # able to authenticate with the cluster. | |
| id-token: write | |
| contents: read | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install Teleport | |
| uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # [email protected] | |
| with: | |
| version: 11.3.1 | |
| - name: Authorize against Teleport | |
| id: auth | |
| uses: teleport-actions/auth@9091dad16a564f3c5b9c2ec520b234a4872b6879 # pin@v1 | |
| with: | |
| # Specify the publically accessible address of your Teleport proxy. | |
| proxy: proxy.mysten-int.com:443 | |
| # Specify the name of the join token for your bot. | |
| token: sui-simtest-token | |
| # Specify the length of time that the generated credentials should be | |
| # valid for. This is optional and defaults to "1h" | |
| certificate-ttl: 2h | |
| # Cargo clean and git restore on any left over files from git checkout, and deletes all remote tracking branches | |
| - name: Environment clean | |
| run: | | |
| tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && rm -rf ~/sui" | |
| tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/ && git clone [email protected]:MystenLabs/sui.git" | |
| # Deleting files in tmpfs that usually fill up pretty quickly after each run. Cargo clean to free up space as well. | |
| - name: Tmpfs and cargo clean | |
| run: | | |
| tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "sudo rm -rf /tmp/*" | |
| tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && cargo clean" | |
| # Checkout out the latest sui repo | |
| - name: Checkout sui repo | |
| run: | | |
| tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 10 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && git fetch origin && git checkout ${{ env.SUI_REF }}" | |
| # Setting up cargo and simtest | |
| - name: Install simtest | |
| run: | | |
| tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 10 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && ./scripts/simtest/install.sh" | |
| # Run simulator tests | |
| - name: Run simtest | |
| run: | | |
| tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 120 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && RUSTUP_MAX_RETRIES=10 CARGO_TERM_COLOR=always CARGO_INCREMENTAL=0 CARGO_NET_RETRY=10 RUST_BACKTRACE=short RUST_LOG=off NUM_CPUS=24 TEST_NUM=${{ env.TEST_NUM }} ./scripts/simtest/simtest-run.sh" | |
| notify: | |
| name: Notify | |
| needs: [simtest] | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' && failure() | |
| steps: | |
| - uses: technote-space/workflow-conclusion-action@45ce8e0eb155657ab8ccf346ade734257fd196a5 # Pin v4.1.1 | |
| - name: Checkout sui repo main branch | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1 | |
| - name: Get sui commit | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| export sui_sha=$(git rev-parse HEAD) | |
| echo "sui_sha=${sui_sha}" >> $GITHUB_ENV | |
| - name: Get link to logs | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh_job_link=$(gh api -X GET 'repos/MystenLabs/sui/actions/runs/${{ github.run_id }}/jobs' --jq '.jobs.[0].html_url') | |
| echo "gh_job_link=${gh_job_link}" >> $GITHUB_ENV | |
| - name: Get current oncall | |
| run: | | |
| export current_oncall=$(curl -s --request GET \ | |
| --url 'https://api.pagerduty.com/oncalls?schedule_ids[]=PGCQ3YS' \ | |
| --header 'Accept: application/json' \ | |
| --header 'Authorization: Token token=${{ secrets.PAGERDUTY_ACCESS_KEY }}' \ | |
| --header 'Content-Type: application/json' \ | |
| | jq '.oncalls[].user.summary' | tr -d '"') | |
| echo "current_oncall=$(echo ${current_oncall})" >> $GITHUB_ENV | |
| export oncall_name=$(curl -s --request GET \ | |
| --url 'https://api.pagerduty.com/oncalls?schedule_ids[]=PGCQ3YS' \ | |
| --header 'Accept: application/json' \ | |
| --header 'Authorization: Token token=${{ secrets.PAGERDUTY_ACCESS_KEY }}' \ | |
| --header 'Content-Type: application/json' \ | |
| | jq '.oncalls[].escalation_policy.summary' | tr -d '"') | |
| echo "oncall_name=$(echo ${oncall_name})" >> $GITHUB_ENV | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # pin v4.0.2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Get slack id for the oncall | |
| run: | | |
| export slack_id=$(aws s3 cp s3://mysten-employees-dir/employees.json - | jq --arg ONCALL "${{ env.current_oncall }}" '.[] | if .name == $ONCALL then .slack_id else empty end') | |
| echo "slack_id=$(echo ${slack_id} | tr -d '"')" >> $GITHUB_ENV | |
| - name: Post to slack | |
| uses: slackapi/[email protected] # [email protected] | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| SUI_SHA: ${{ env.sui_sha }} | |
| GH_JOB_LINK: ${{ env.gh_job_link }} | |
| SLACK_ID: ${{ env.slack_id }} | |
| ONCALL_NAME: ${{ env.oncall_name }} | |
| with: | |
| channel-id: 'simtest-nightly' | |
| payload: | | |
| { | |
| "text": "*${{ github.workflow }}* workflow status: `${{ env.WORKFLOW_CONCLUSION }}`", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "*${{ github.workflow }}* workflow status: `${{ env.WORKFLOW_CONCLUSION }}`" | |
| } | |
| }, | |
| { | |
| "type": "divider" | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Sui commit: <https://github.com/MystenLabs/sui/commit/${{ env.SUI_SHA }}|${{ env.SUI_SHA }}> \nRun: <${{ env.GH_JOB_LINK }}|${{ github.run_id }}>" | |
| } | |
| }, | |
| { | |
| "type": "divider" | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "<@${{ env.SLACK_ID }}>, current `${{ env.ONCALL_NAME }}` oncall, please debug failures: `tsh ssh ubuntu@simtest-01` and look in the `/home/ubuntu/simtest_logs/{date}` folder for test results" | |
| } | |
| } | |
| ] | |
| } |