sync github actions to 23-3 (#318)

ydb-platform · Feb 1, 2024 · ee1ea02 · ee1ea02
1 parent 3e3896f
commit ee1ea02
Show file tree

Hide file tree

Showing 13 changed files with 63 additions and 33 deletions.
diff --git a/.github/actions/s3cmd/action.yaml b/.github/actions/s3cmd/action.yaml
@@ -22,6 +22,9 @@ inputs:
   build_preset:
     required: true
     description: "build preset like relwithdebinfo"
+  user:
+    required: false
+    description: "github user"
 runs:
   using: composite
   steps:
@@ -33,11 +36,11 @@ runs:
     - name: configure s3cmd
       shell: bash
       run: |
-        export S3CMD_CONFIG=$(mktemp -p /home/github)
-        chown github:github $S3CMD_CONFIG
+        export S3CMD_CONFIG=$(sudo -E -H -u $user mktemp -p /home/$user)
+        sudo chown $user: $S3CMD_CONFIG
         echo "S3CMD_CONFIG=$S3CMD_CONFIG" >> $GITHUB_ENV
         export GITHUB_WORKFLOW_NO_SPACES=${GITHUB_WORKFLOW// /-}
-        cat <<EOF > $S3CMD_CONFIG
+        cat <<EOF | sudo -E -H -u $user tee $S3CMD_CONFIG
         [default]
         access_key = ${s3_key_id}
         secret_key = ${s3_secret_access_key}
@@ -46,25 +49,25 @@ runs:
         host_bucket = %(bucket)s.storage.ai.nebius.cloud
         EOF
 
-        mkdir -p /home/github/.aws/
-        cat <<EOF > /home/github/.aws/credentials
+        sudo -E -H -u $user mkdir -p /home/$user/.aws/
+        cat <<EOF | sudo -E -H -u $user tee /home/$user/.aws/credentials
         [default]
         aws_access_key_id = ${s3_key_id}
         aws_secret_access_key = ${s3_secret_access_key}
         EOF
-        cat <<EOF > /home/github/.aws/config
+        cat <<EOF | sudo -E -H -u $user tee /home/$user/.aws/config
         [default]
         region = eu-north1
         endpoint_url=https://storage.ai.nebius.cloud/
         EOF
 
-        mkdir -p /root/.aws/
-        cat <<EOF > /root/.aws/credentials
+        sudo mkdir -p /root/.aws/
+        cat <<EOF | sudo tee /root/.aws/credentials
         [default]
         aws_access_key_id = ${s3_key_id}
         aws_secret_access_key = ${s3_secret_access_key}
         EOF
-        cat <<EOF > /root/.aws/config
+        cat <<EOF | sudo tee /root/.aws/config
         [default]
         region = eu-north1
         endpoint_url=https://storage.ai.nebius.cloud/
@@ -96,6 +99,9 @@ runs:
       env:
         s3_key_id: ${{ inputs.s3_key_id }}
         s3_secret_access_key: ${{ inputs.s3_key_secret }}
+        user: ${{ inputs.user || 'github'}}
     - name: test s3cmd
       shell: bash
-      run: s3cmd ls
+      run: sudo -E -H -u $user s3cmd ls
+      env:
+        user: ${{ inputs.user || 'github'}}
diff --git a/.github/actions/test/action.yaml b/.github/actions/test/action.yaml
@@ -197,9 +197,14 @@ runs:
           find "$TESTS_DATA_DIR" -type f -print0 | xargs -0 -n 10 file -i  | grep "application/x-executable" | awk -F: '{print $1}'
           find "$TESTS_DATA_DIR" -type f -print0 | xargs -0 -n 10 file -i  | grep "application/x-executable" | awk -F: '{print $1}' | xargs rm
           echo "::endgroup::"
+          echo "::group::remove-images-from-tests-data-dir"
+          find "$TESTS_DATA_DIR" -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image
+          find "$TESTS_DATA_DIR" -name generated_raw_image -o -name generated_vmdk_image -o -name invalid_qcow2_image -o -name qcow2_fuzzing_image -delete
+          echo "::endgroup::"
           echo "::group::s3-sync"
           if [ "$SYNC_TO_S3" = "true" ];
           then
+            du -h "$TESTS_DATA_DIR/"
             sudo -E -H -u github s3cmd sync --follow-symlinks --acl-private --no-progress --stats --no-check-md5 "$TESTS_DATA_DIR/" "$S3_BUCKET_PATH/test_data/"
           fi
           echo "::endgroup::"

diff --git a/.github/actions/test_cmake/action.yaml b/.github/actions/test_cmake/action.yaml
@@ -27,6 +27,8 @@ runs:
     - name: Generate summary
       if: always()
       shell: bash
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
       run: |
         if [ -f "${ARTIFACTS_DIR}/junit.xml" ]; then
           if [ -f "${ARTIFACTS_DIR}/junit_rerun.xml" ]; then

diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt
@@ -8,6 +8,9 @@ cloud/filestore/tests/fio_index_migration/qemu-intrahost-migration-local-test *
 cloud/filestore/tests/fio_index/qemu-kikimr-nemesis-test *
 cloud/filestore/tests/fio_index/qemu-kikimr-test *
 cloud/filestore/tests/fio_index/qemu-local-test *
+cloud/filestore/tests/fio_migration/qemu-intrahost-migration-kikimr-nemesis-test *
+cloud/filestore/tests/fio_migration/qemu-intrahost-migration-kikimr-test *
+cloud/filestore/tests/fio_migration/qemu-intrahost-migration-local-test *
 cloud/filestore/tests/fio/qemu-kikimr-nemesis-test *
 cloud/filestore/tests/fio/qemu-kikimr-test *
 cloud/filestore/tests/fio/qemu-local-test *

diff --git a/.github/scripts/index.py b/.github/scripts/index.py
@@ -14,6 +14,15 @@
 )
 
 
+def sizeof_fmt(num, suffix="B"):
+    """Converts file size to a human-readable format."""
+    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
+        if abs(num) < 1024.0:
+            return f"{num:3.1f}{unit}{suffix}"
+        num /= 1024.0
+    return f"{num:.1f}Yi{suffix}"
+
+
 def list_files(client, bucket, prefix):
     paginator = client.get_paginator("list_objects_v2")
     for page in paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter="/"):
@@ -49,6 +58,7 @@ def generate_index_html(bucket, files, dirs, current_prefix):
                 "url": generate_absolute_url(bucket, parent_dir),
                 "type": "directory",
                 "date": "",
+                "size": "",  # Directories don't have a size
             }
         )
     for d in dirs:
@@ -61,6 +71,7 @@ def generate_index_html(bucket, files, dirs, current_prefix):
                     "url": dir_url,
                     "type": "directory",
                     "date": "",
+                    "size": "",  # Directories don't have a size
                 }
             )
     for f in files:
@@ -70,12 +81,14 @@ def generate_index_html(bucket, files, dirs, current_prefix):
             file_date = datetime.fromtimestamp(f["LastModified"].timestamp()).strftime(
                 "%Y-%m-%d %H:%M:%S"
             )
+            file_size = sizeof_fmt(f["Size"])
             entries.append(
                 {
                     "name": os.path.basename(file_key),
                     "url": file_url,
                     "type": "file",
                     "date": file_date,
+                    "size": file_size,
                 }
             )
 

diff --git a/.github/scripts/templates/index.html b/.github/scripts/templates/index.html
@@ -44,9 +44,10 @@
         .directory-icon:before {
             content: '📁'; /* Folder icon */
         }
-        .date {
+        .date, .size {
             float: right;
-            color: #666;
+            margin-left: 20px;
+            color: #999;
         }
 
         #timestamp {
@@ -65,6 +66,9 @@ <h1>Directory listing for {{ directory }}</h1>
                 {% if entry.date %}
                     <span class="date">{{ entry.date }}</span>
                 {% endif %}
+                {% if entry.size %}
+                    <span class="size">{{ entry.size }}</span>
+                {% endif %}
             </li>
         {% endfor %}
     </ul>

diff --git a/.github/workflows/build_and_test_act.yaml b/.github/workflows/build_and_test_act.yaml
@@ -109,12 +109,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout PR
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       if: github.event.pull_request.head.sha != ''
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       if: github.event.pull_request.head.sha == ''
     - name: Configure NCP
       uses: ./.github/actions/ncp

diff --git a/.github/workflows/build_and_test_act_cmake.yaml b/.github/workflows/build_and_test_act_cmake.yaml
@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout PR
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: refs/heads/stable-23-3
 

diff --git a/.github/workflows/build_and_test_cmake.yaml b/.github/workflows/build_and_test_cmake.yaml
@@ -48,12 +48,12 @@ jobs:
     timeout-minutes: 1440
     steps:
     - name: Checkout PR
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       if: github.event.pull_request.head.sha != ''
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       if: github.event.pull_request.head.sha == ''
 
     - name: Prepare s3cmd

diff --git a/.github/workflows/build_and_test_on_demand.yaml b/.github/workflows/build_and_test_on_demand.yaml
@@ -103,7 +103,7 @@ jobs:
     steps:
       - name: Start YC runner
         id: start-yc-runner
-        uses: librarian/[email protected].13
+        uses: librarian/[email protected].15
         timeout-minutes: 60
         with:
           mode: start
@@ -130,12 +130,12 @@ jobs:
       runner_ipv4: ${{ steps.configure-ncp.outputs.runner_ipv4 }}
     steps:
       - name: Checkout PR
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         if: github.event.pull_request.head.sha != ''
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         if: github.event.pull_request.head.sha == ''
       - id: configure-ncp
         name: Configure NCP
@@ -192,7 +192,7 @@ jobs:
         run: sleep ${{ needs.build-and-test.outputs.sleep_after_tests }}
 
       - name: Stop YC runner
-        uses: librarian/[email protected].13
+        uses: librarian/[email protected].15
         if: always()
         timeout-minutes: 60
         with:

diff --git a/.github/workflows/build_and_test_on_demand_cmake.yaml b/.github/workflows/build_and_test_on_demand_cmake.yaml
@@ -51,7 +51,7 @@ jobs:
     steps:
       - name: Start YC runner
         id: start-yc-runner
-        uses: librarian/[email protected].13
+        uses: librarian/[email protected].15
         timeout-minutes: 60
         with:
           mode: start
@@ -78,12 +78,12 @@ jobs:
       runner_ipv4: ${{ steps.configure-ncp.outputs.runner_ipv4 }}
     steps:
       - name: Checkout PR
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         if: github.event.pull_request.head.sha != ''
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         if: github.event.pull_request.head.sha == ''
       - id: configure-ncp
         name: Configure NCP
@@ -132,7 +132,7 @@ jobs:
         run: sleep ${{ needs.build-and-test.outputs.sleep_after_tests }}
 
       - name: Stop YC runner
-        uses: librarian/[email protected].13
+        uses: librarian/[email protected].15
         if: always()
         timeout-minutes: 60
         with:

diff --git a/.github/workflows/build_and_test_ya.yaml b/.github/workflows/build_and_test_ya.yaml
@@ -80,12 +80,12 @@ jobs:
     timeout-minutes: 1440
     steps:
     - name: Checkout PR
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       if: github.event.pull_request.head.sha != ''
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       if: github.event.pull_request.head.sha == ''
 
     - name: Prepare s3cmd

diff --git a/.github/workflows/nightly-index-rebuild.yaml b/.github/workflows/nightly-index-rebuild.yaml
@@ -11,13 +11,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout PR
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Prepare VM
       uses: ./.github/actions/prepare
-    - name: add user github if not exists
-      shell: bash
-      run: |
-        id -u github || sudo adduser github --gecos "" --disabled-password
     - name: Prepare s3cmd
       uses: ./.github/actions/s3cmd
       with:
@@ -28,13 +24,14 @@ jobs:
         s3_key_secret: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         folder_prefix: nebius-
         build_preset: "release"
+        user: runner
 
     - name: Create indices for repo
       shell: bash
       run: |
         set -x
         echo "::group::generate-listing"
-        sudo -E -H -u github python3 .github/scripts/index.py s3://${S3_BUCKET}/${GITHUB_REPOSITORY}/
+        python3 .github/scripts/index.py s3://${S3_BUCKET}/${GITHUB_REPOSITORY}/
         echo "::endgroup::"
       env:
         S3_BUCKET: ${{ vars.AWS_BUCKET }}