[ci] Removes macOS x86_64 and PyTorch cu121-precxx11 support (#3356)

deepjavalibrary · Jul 19, 2024 · d2b854e · d2b854e
1 parent ecbc5da
commit d2b854e
Show file tree

Hide file tree

Showing 13 changed files with 85 additions and 259 deletions.
diff --git a/.github/workflows/native_jni_s3_pytorch.yml b/.github/workflows/native_jni_s3_pytorch.yml
@@ -14,49 +14,6 @@ on:
     - cron: '0 5 * * *'
 
 jobs:
-  build-pytorch-jni-macos:
-    if: github.repository == 'deepjavalibrary/djl'
-    runs-on: macos-13
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK 17
-        uses: actions/setup-java@v4
-        with:
-          distribution: 'corretto'
-          java-version: 17
-      - uses: actions/cache@v4
-        with:
-          path: ~/.gradle/caches
-          key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle-
-      - name: Release JNI prep
-        shell: bash
-        run: |
-          PYTORCH_VERSION=${{ github.event.inputs.pt_version }}
-          export PYTORCH_VERSION=${PYTORCH_VERSION:-$(awk -F '=' '/pytorch/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)}
-          echo $PYTORCH_VERSION
-          if [[ $PYTORCH_VERSION =~ ^(?\.[1-2]?\..*)$ ]]; then \
-            ./gradlew :engines:pytorch:pytorch-native:compileJNI -Ppt_version=$PYTORCH_VERSION; \
-            ./gradlew -Pjni -Ppt_version=$PYTORCH_VERSION :integration:test "-Dai.djl.default_engine=PyTorch"; \
-          fi;
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-2
-      - name: Copy files to S3 with the AWS CLI
-        shell: bash
-        run: |
-          PYTORCH_VERSION=${{ github.event.inputs.pt_version }}
-          PYTORCH_VERSION=${PYTORCH_VERSION:-$(awk -F '=' '/pytorch/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)}
-          if [[ $PYTORCH_VERSION =~ ^(?\.[1-2]?\..*)$ ]]; then \
-            aws s3 sync engines/pytorch/pytorch-native/jnilib s3://djl-ai/publish/pytorch/${PYTORCH_VERSION}/jnilib; \
-            aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/pytorch/${PYTORCH_VERSION}/jnilib*"; \
-          fi;
-
   build-pytorch-jni-linux:
     if: github.repository == 'deepjavalibrary/djl'
     runs-on: ubuntu-latest
@@ -111,48 +68,37 @@ jobs:
   build-pytorch-jni-precxx11:
     if: github.repository == 'deepjavalibrary/djl'
     runs-on: ubuntu-latest
-    container: nvidia/cuda:12.1.1-cudnn8-devel-centos7
+    container:
+      image: amazonlinux:2
+      env:
+        JAVA_HOME: /usr/lib/jvm/java-17-amazon-corretto
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    timeout-minutes: 30
+    needs: create-aarch64-runner
     steps:
-      - name: Install Environment
+      - name: Install compiler environment
         run: |
           yum -y update
-          yum -y install centos-release-scl-rh epel-release
-          yum -y install devtoolset-9 rh-git218 patch cmake3
-          ln -s /usr/bin/cmake3 /usr/bin/cmake
+          yum -y install patch git cmake3 python3-devel java-17-amazon-corretto-devel
+          yum -y install gcc10 gcc10-c++
+          ln -sf /usr/bin/gcc10-gcc /usr/bin/gcc
+          ln -sf /usr/bin/gcc10-cc /usr/bin/cc
+          ln -sf /usr/bin/gcc10-g++ /usr/bin/g++
+          ln -sf /usr/bin/gcc10-c++ /usr/bin/c++
+          ln -sf /usr/bin/gcc10-ar /usr/bin/ar
+          ln -sf /usr/bin/cmake3 /usr/bin/cmake
           pip3 install awscli --upgrade
-      - uses: actions/checkout@v3
-      - name: Set up JDK 17
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'corretto'
-          java-version: 17
-      - uses: actions/cache@v3
-        with:
-          path: ~/.gradle/caches
-          key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle-
+      - uses: taiki-e/checkout-action@v1
       - name: Release JNI prep
         run: |
-          source scl_source enable devtoolset-9 rh-git218 || true
+          export PATH=$PATH:$JAVA_HOME/bin
           PYTORCH_VERSION=${{ github.event.inputs.pt_version }}
           export PYTORCH_VERSION=${PYTORCH_VERSION:-$(awk -F '=' '/pytorch/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)}
-          export PYTORCH_PRECXX11=true
           echo $PYTORCH_VERSION
           ./gradlew :engines:pytorch:pytorch-native:compileJNI -Pprecxx11 -Ppt_version=$PYTORCH_VERSION
+          export PYTORCH_PRECXX11=true
           ./gradlew -Pjni -Ppt_version=$PYTORCH_VERSION :integration:test "-Dai.djl.default_engine=PyTorch"
-          ./gradlew :engines:pytorch:pytorch-native:cleanJNI
-          rm -rf ~/.djl.ai
-          export TORCH_CUDA_ARCH_LIST="8.0 8.6 8.9 9.0"
-          CUDA_VERSION=${{ github.event.inputs.cuda }}
-          export CUDA_VERSION=${CUDA_VERSION:-cu121}
-          ./gradlew :engines:pytorch:pytorch-native:compileJNI -Pcuda=$CUDA_VERSION -Pprecxx11 -Ppt_version=$PYTORCH_VERSION
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v2
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-2
       - name: Copy files to S3 with the AWS CLI
         run: |
           PYTORCH_VERSION=${{ github.event.inputs.pt_version }}

diff --git a/.github/workflows/native_s3_fasttext.yml b/.github/workflows/native_s3_fasttext.yml
@@ -4,36 +4,6 @@ on:
   workflow_dispatch:
 
 jobs:
-  build-fasttext-jni-osx:
-    runs-on: macos-13
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK 17
-        uses: actions/setup-java@v4
-        with:
-          distribution: 'corretto'
-          java-version: 17
-      - uses: actions/cache@v4
-        with:
-          path: ~/.gradle/caches
-          key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle-
-      - name: Release JNI prep
-        run: |
-          ./gradlew :extensions:fasttext:compileJNI
-          ./gradlew -Pjni :extensions:fasttext:test
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-2
-      - name: Copy files to S3 with the AWS CLI
-        run: |
-          FASTTEXT_VERSION="$(awk -F '=' '/fasttext/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
-          aws s3 sync extensions/fasttext/jnilib s3://djl-ai/publish/fasttext-${FASTTEXT_VERSION}/jnilib/
-
   build-fasttext-jni-linux:
     runs-on: ubuntu-latest
     container: amazonlinux:2

diff --git a/.github/workflows/native_s3_huggingface.yml b/.github/workflows/native_s3_huggingface.yml
@@ -9,41 +9,6 @@ on:
       - extensions/tokenizers/rust/**
 
 jobs:
-  build-tokenizers-jni-osx:
-    runs-on: macos-13
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK 17
-        uses: actions/setup-java@v4
-        with:
-          distribution: 'corretto'
-          java-version: 17
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-      - uses: actions/cache@v4
-        with:
-          path: ~/.gradle/caches
-          key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle-
-      - name: Release JNI prep
-        run: |
-          ./gradlew :extensions:tokenizers:compileJNI
-          PYTORCH_VERSION=2.2.2 ./gradlew -Pjni :extensions:tokenizers:test
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-2
-      - name: Copy files to S3 with the AWS CLI
-        run: |
-          DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)
-          TOKENIZERS_VERSION="$(awk -F '=' '/tokenizers/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
-          aws s3 sync extensions/tokenizers/jnilib/$DJL_VERSION/osx-x86_64 s3://djl-ai/publish/tokenizers/${TOKENIZERS_VERSION}/jnilib/$DJL_VERSION/osx-x86_64/
-          aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tokenizers/${TOKENIZERS_VERSION}/jnilib/*"
-
   build-tokenizers-jni-linux:
     runs-on: ubuntu-latest
     container:

diff --git a/.github/workflows/native_s3_sentencepiece.yml b/.github/workflows/native_s3_sentencepiece.yml
@@ -4,38 +4,6 @@ on:
   workflow_dispatch:
 
 jobs:
-  build-sentencepiece-jni-osx:
-    if: ${{ github.repository == 'deepjavalibrary/djl' && always() }}
-    runs-on: macos-13
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK 17
-        uses: actions/setup-java@v4
-        with:
-          distribution: 'corretto'
-          java-version: 17
-      - uses: actions/cache@v4
-        with:
-          path: ~/.gradle/caches
-          key: ${{ runner.os }}-gradle-${{ hashFiles('*/build.gradle.kts', 'engines/**/build.gradle.kts', 'extensions/**/build.gradle.kts') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle-
-      - name: Release JNI prep
-        run: |
-          ./gradlew :extensions:sentencepiece:compileJNI
-          ./gradlew -Pjni :extensions:sentencepiece:test
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-2
-      - name: Copy files to S3 with the AWS CLI
-        run: |
-          SENTENCEPIECE_VERSION="$(awk -F '=' '/sentencepiece/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)"
-          aws s3 sync extensions/sentencepiece/jnilib s3://djl-ai/publish/sentencepiece-${SENTENCEPIECE_VERSION}/jnilib/
-          aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/sentencepiece-${SENTENCEPIECE_VERSION}/jnilib/*"
-
   build-sentencepiece-jni-linux:
     if: ${{ github.repository == 'deepjavalibrary/djl' && always() }}
     runs-on: ubuntu-latest

diff --git a/.github/workflows/native_s3_xgboost.yml b/.github/workflows/native_s3_xgboost.yml
@@ -2,9 +2,10 @@ name: Native S3 XGBoost
 
 on:
   workflow_dispatch:
-    xgb_version:
-      description: 'xgboost version'
-      required: false
+    inputs:
+      xgb_version:
+        description: 'xgboost version'
+        required: false
 
 jobs:
   create-aarch64-runner:

diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts
@@ -117,7 +117,6 @@ publishing {
                         }
 
                         val mxnet = libs.versions.mxnet.get()
-                        +"ai.djl.mxnet:mxnet-native-mkl:$mxnet:osx-x86_64"
                         +"ai.djl.mxnet:mxnet-native-mkl:$mxnet:linux-x86_64"
                         +"ai.djl.mxnet:mxnet-native-mkl:$mxnet:win-x86_64"
                         +"ai.djl.mxnet:mxnet-native-cu112mkl:$mxnet:linux-x86_64"
@@ -129,13 +128,10 @@ publishing {
                         +"ai.djl.pytorch:pytorch-native-cpu-precxx11:$pytorch:linux-aarch64"
                         +"ai.djl.pytorch:pytorch-native-cu121:$pytorch:linux-x86_64"
                         +"ai.djl.pytorch:pytorch-native-cu121:$pytorch:win-x86_64"
-                        +"ai.djl.pytorch:pytorch-native-cu121-precxx11:$pytorch:linux-x86_64"
                         pytorch = "1.13.1"
                         +"ai.djl.pytorch:pytorch-native-cu117:$pytorch:linux-x86_64"
                         +"ai.djl.pytorch:pytorch-native-cu117:$pytorch:win-x86_64"
-                        +"ai.djl.pytorch:pytorch-native-cu117-precxx11:$pytorch:linux-x86_64"
                         val tensorflow = "${libs.versions.tensorflow.get()}"
-                        +"ai.djl.tensorflow:tensorflow-native-cpu:$tensorflow:osx-x86_64"
                         +"ai.djl.tensorflow:tensorflow-native-cpu:$tensorflow:osx-aarch64"
                         +"ai.djl.tensorflow:tensorflow-native-cpu:$tensorflow:linux-x86_64"
                         +"ai.djl.tensorflow:tensorflow-native-cpu:$tensorflow:linux-aarch64"