Skip to content

Commit

Permalink
[ci] Fixes sentencepiece native build for nodejs20 issue (#3479)
Browse files Browse the repository at this point in the history
  • Loading branch information
xyang16 committed Sep 17, 2024
1 parent 5362332 commit 2ffaf87
Showing 1 changed file with 56 additions and 45 deletions.
101 changes: 56 additions & 45 deletions .github/workflows/native_s3_sentencepiece.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,37 +11,45 @@ jobs:
build-sentencepiece-jni-linux:
if: ${{ github.repository == 'deepjavalibrary/djl' && always() }}
runs-on: ubuntu-latest
container: centos:centos7
container:
image: amazonlinux:2
env:
JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto
steps:
- name: Install Environment
run: |
yum -y update
yum -y install centos-release-scl-rh epel-release
yum -y install devtoolset-7 git patch cmake3 libstdc++-static
yum -y groupinstall "Development Tools"
yum -y install patch cmake3 jq
yum -y install java-17-amazon-corretto-devel
ln -sf /usr/bin/cmake3 /usr/bin/cmake
pip3 install awscli --upgrade
- uses: actions/checkout@v3
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
distribution: 'corretto'
java-version: 17
- uses: actions/cache@v3
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }}
restore-keys: |
${{ runner.os }}-gradle-
- uses: taiki-e/checkout-action@v1
- name: Release JNI prep
run: |
export PATH=$PATH:/opt/rh/devtoolset-7/root/usr/bin
./gradlew :extensions:sentencepiece:compileJNI
./gradlew -Pjni :extensions:sentencepiece:test
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::425969335547:role/djl-ci-publish-role
aws-region: us-east-2
run: |
oidc_token=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
"$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.amazonaws.com" | jq -r ".value")
echo "::add-mask::$oidc_token"
read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"$(aws sts assume-role-with-web-identity \
--region "us-east-2" \
--role-arn "arn:aws:iam::425969335547:role/djl-ci-publish-role" \
--role-session-name "build-fasttext-jni-linux" \
--web-identity-token "$oidc_token" \
--query "[Credentials.AccessKeyId, Credentials.SecretAccessKey, Credentials.SessionToken]" \
--output text)"
echo "::add-mask::$AWS_ACCESS_KEY_ID"
echo "::add-mask::$AWS_SECRET_ACCESS_KEY"
echo "::add-mask::$AWS_SESSION_TOKEN"
echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> "$GITHUB_ENV"
echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> "$GITHUB_ENV"
echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> "$GITHUB_ENV"
- name: Copy files to S3 with the AWS CLI
run: |
SENTENCEPIECE_VERSION="$(awk -F '=' '/sentencepiece/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
Expand Down Expand Up @@ -120,41 +128,44 @@ jobs:
runs-on: [ self-hosted, aarch64 ]
timeout-minutes: 30
needs: create-aarch64-runner
container: centos:centos7
container:
image: amazonlinux:2
env:
JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto.aarch64
steps:
- name: Install Environment
run: |
yum -y update
yum -y install centos-release-scl-rh epel-release
yum -y install devtoolset-7 git patch cmake3 libstdc++-static
yum -y groupinstall "Development Tools"
yum -y install patch perl-IPC-Cmd cmake3 curl jq
yum -y install java-17-amazon-corretto-devel
ln -s /usr/bin/cmake3 /usr/bin/cmake
pip3 install awscli --upgrade
- uses: actions-rs/toolchain@v1
with:
toolchain: stable
- uses: actions/checkout@v3
- name: Set up JDK 17
uses: actions/setup-java@v3
with:
java-version: 17
distribution: corretto
architecture: aarch64
- uses: actions/cache@v3
with:
path: ~/.gradle/caches
key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }}
restore-keys: |
${{ runner.os }}-gradle-
- uses: taiki-e/checkout-action@v1
- name: Release JNI prep
run: |
export PATH=$PATH:/opt/rh/devtoolset-7/root/usr/bin
./gradlew :extensions:sentencepiece:compileJNI
./gradlew -Pjni :extensions:sentencepiece:test
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::425969335547:role/djl-ci-publish-role
aws-region: us-east-2
run: |
oidc_token=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
"$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.amazonaws.com" | jq -r ".value")
echo "::add-mask::$oidc_token"
read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"$(aws sts assume-role-with-web-identity \
--region "us-east-2" \
--role-arn "arn:aws:iam::425969335547:role/djl-ci-publish-role" \
--role-session-name "build-tokenizer-jni-aarch64" \
--web-identity-token "$oidc_token" \
--query "[Credentials.AccessKeyId, Credentials.SecretAccessKey, Credentials.SessionToken]" \
--output text)"
echo "::add-mask::$AWS_ACCESS_KEY_ID"
echo "::add-mask::$AWS_SECRET_ACCESS_KEY"
echo "::add-mask::$AWS_SESSION_TOKEN"
echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> "$GITHUB_ENV"
echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> "$GITHUB_ENV"
echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> "$GITHUB_ENV"
- name: Copy files to S3 with the AWS CLI
run: |
SENTENCEPIECE_VERSION="$(awk -F '=' '/sentencepiece/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
Expand Down

0 comments on commit 2ffaf87

Please sign in to comment.