From 2ffaf87b12e2116dddd05f32c0fb6d52394d4f9a Mon Sep 17 00:00:00 2001 From: Xin Yang <105740670+xyang16@users.noreply.github.com> Date: Tue, 17 Sep 2024 11:32:40 -0700 Subject: [PATCH] [ci] Fixes sentencepiece native build for nodejs20 issue (#3479) --- .github/workflows/native_s3_sentencepiece.yml | 101 ++++++++++-------- 1 file changed, 56 insertions(+), 45 deletions(-) diff --git a/.github/workflows/native_s3_sentencepiece.yml b/.github/workflows/native_s3_sentencepiece.yml index 72d20d09d52..f37cc1b1342 100644 --- a/.github/workflows/native_s3_sentencepiece.yml +++ b/.github/workflows/native_s3_sentencepiece.yml @@ -11,37 +11,45 @@ jobs: build-sentencepiece-jni-linux: if: ${{ github.repository == 'deepjavalibrary/djl' && always() }} runs-on: ubuntu-latest - container: centos:centos7 + container: + image: amazonlinux:2 + env: + JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto steps: - name: Install Environment run: | yum -y update - yum -y install centos-release-scl-rh epel-release - yum -y install devtoolset-7 git patch cmake3 libstdc++-static + yum -y groupinstall "Development Tools" + yum -y install patch cmake3 jq + yum -y install java-17-amazon-corretto-devel ln -sf /usr/bin/cmake3 /usr/bin/cmake pip3 install awscli --upgrade - - uses: actions/checkout@v3 - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'corretto' - java-version: 17 - - uses: actions/cache@v3 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }} - restore-keys: | - ${{ runner.os }}-gradle- + - uses: taiki-e/checkout-action@v1 - name: Release JNI prep run: | - export PATH=$PATH:/opt/rh/devtoolset-7/root/usr/bin ./gradlew :extensions:sentencepiece:compileJNI ./gradlew -Pjni :extensions:sentencepiece:test - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::425969335547:role/djl-ci-publish-role - aws-region: us-east-2 + run: | + oidc_token=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.amazonaws.com" | jq -r ".value") + echo "::add-mask::$oidc_token" + + read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"$(aws sts assume-role-with-web-identity \ + --region "us-east-2" \ + --role-arn "arn:aws:iam::425969335547:role/djl-ci-publish-role" \ + --role-session-name "build-fasttext-jni-linux" \ + --web-identity-token "$oidc_token" \ + --query "[Credentials.AccessKeyId, Credentials.SecretAccessKey, Credentials.SessionToken]" \ + --output text)" + + echo "::add-mask::$AWS_ACCESS_KEY_ID" + echo "::add-mask::$AWS_SECRET_ACCESS_KEY" + echo "::add-mask::$AWS_SESSION_TOKEN" + + echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> "$GITHUB_ENV" + echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> "$GITHUB_ENV" + echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> "$GITHUB_ENV" - name: Copy files to S3 with the AWS CLI run: | SENTENCEPIECE_VERSION="$(awk -F '=' '/sentencepiece/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)" @@ -120,41 +128,44 @@ jobs: runs-on: [ self-hosted, aarch64 ] timeout-minutes: 30 needs: create-aarch64-runner - container: centos:centos7 + container: + image: amazonlinux:2 + env: + JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto.aarch64 steps: - name: Install Environment run: | - yum -y update - yum -y install centos-release-scl-rh epel-release - yum -y install devtoolset-7 git patch cmake3 libstdc++-static + yum -y groupinstall "Development Tools" + yum -y install patch perl-IPC-Cmd cmake3 curl jq + yum -y install java-17-amazon-corretto-devel ln -s /usr/bin/cmake3 /usr/bin/cmake pip3 install awscli --upgrade - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - uses: actions/checkout@v3 - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - java-version: 17 - distribution: corretto - architecture: aarch64 - - uses: actions/cache@v3 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }} - restore-keys: | - ${{ runner.os }}-gradle- + - uses: taiki-e/checkout-action@v1 - name: Release JNI prep run: | - export PATH=$PATH:/opt/rh/devtoolset-7/root/usr/bin ./gradlew :extensions:sentencepiece:compileJNI ./gradlew -Pjni :extensions:sentencepiece:test - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::425969335547:role/djl-ci-publish-role - aws-region: us-east-2 + run: | + oidc_token=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.amazonaws.com" | jq -r ".value") + echo "::add-mask::$oidc_token" + + read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"$(aws sts assume-role-with-web-identity \ + --region "us-east-2" \ + --role-arn "arn:aws:iam::425969335547:role/djl-ci-publish-role" \ + --role-session-name "build-tokenizer-jni-aarch64" \ + --web-identity-token "$oidc_token" \ + --query "[Credentials.AccessKeyId, Credentials.SecretAccessKey, Credentials.SessionToken]" \ + --output text)" + + echo "::add-mask::$AWS_ACCESS_KEY_ID" + echo "::add-mask::$AWS_SECRET_ACCESS_KEY" + echo "::add-mask::$AWS_SESSION_TOKEN" + + echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> "$GITHUB_ENV" + echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> "$GITHUB_ENV" + echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> "$GITHUB_ENV" - name: Copy files to S3 with the AWS CLI run: | SENTENCEPIECE_VERSION="$(awk -F '=' '/sentencepiece/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"