Merge remote-tracking branch 'origin/main' into xenova-bigcode-testing

huggingface · Jul 27, 2023 · b03c369 · b03c369
2 parents bd187ae + bdc3ecc
commit b03c369
Show file tree

Hide file tree

Showing 117 changed files with 7,976 additions and 1,914 deletions.
diff --git a/.github/workflows/build_main_documentation.yml b/.github/workflows/build_main_documentation.yml
@@ -44,6 +44,11 @@ jobs:
           repository: 'huggingface/optimum-intel'
           path: optimum-intel
 
+      - uses: actions/checkout@v2
+        with:
+          repository: 'huggingface/optimum-furiosa'
+          path: optimum-furiosa
+
       - name: Set environment variables
         run: |
           cd optimum
@@ -76,18 +81,41 @@ jobs:
 
       - name: Make Habana documentation
         run: |
+          sudo docker system prune -a -f
           cd optimum-habana
           make doc BUILD_DIR=habana-doc-build VERSION=${{ env.VERSION }}
           sudo mv habana-doc-build ../optimum
           cd ..
 
       - name: Make Intel documentation
         run: |
+          sudo docker system prune -a -f
           cd optimum-intel
           make doc BUILD_DIR=intel-doc-build VERSION=${{ env.VERSION }}
           sudo mv intel-doc-build ../optimum
           cd ..
 
+      - name: Make Furiosa documentation
+        run: |
+          cd optimum-furiosa
+          pip install .
+          sudo apt update
+          sudo apt install -y ca-certificates apt-transport-https gnupg
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-key 5F03AFA423A751913F249259814F888B20B09A7E
+          sudo tee -a /etc/apt/auth.conf.d/furiosa.conf > /dev/null <<EOT
+            machine archive.furiosa.ai
+            login ${{ secrets.FURIOSA_ACCESS_KEY }}
+            password ${{ secrets.FURIOSA_SECRET_ACCESS_KEY }}
+          EOT
+          sudo chmod 400 /etc/apt/auth.conf.d/furiosa.conf
+          sudo tee -a /etc/apt/sources.list.d/furiosa.list <<EOT
+            deb [arch=amd64] https://archive.furiosa.ai/ubuntu jammy restricted
+          EOT
+          sudo apt update && sudo apt install -y furiosa-libnux
+          doc-builder build optimum.furiosa docs/source/ --build_dir furiosa-doc-build --version pr_$PR_NUMBER --version_tag_suffix "" --html --clean
+          mv furiosa-doc-build ../optimum
+          cd ..
+
       - name: Make Optimum documentation
         run: |
           sudo docker system prune -a -f
@@ -101,12 +129,12 @@ jobs:
       - name: Combine subpackage documentation
         run: |
           cd optimum
-          sudo python docs/combine_docs.py --subpackages graphcore habana intel --version ${{ env.VERSION }}
+          sudo python docs/combine_docs.py --subpackages graphcore habana intel neuron furiosa --version ${{ env.VERSION }}
           cd ..
 
       - name: Push to repositories
         run: |
           cd optimum/optimum-doc-build
-          chmod ugo+rwx optimum
-          doc-builder push optimum --doc_build_repo_id "hf-doc-build/doc-build" --token "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" --commit_msg "Updated with commit ${{ github.sha }} See: https://github.com/huggingface/optimum/commit/${{ github.sha }}" --n_retries 5 --upload_version_yml
+          sudo chmod -R ugo+rwx optimum
+          doc-builder push optimum --doc_build_repo_id "hf-doc-build/doc-build" --token "${{ secrets.HF_DOC_BUILD_PUSH }}" --commit_msg "Updated with commit ${{ github.sha }} See: https://github.com/huggingface/optimum/commit/${{ github.sha }}" --n_retries 5 --upload_version_yml
         shell: bash
diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml
@@ -43,9 +43,10 @@ jobs:
           repository: 'huggingface/optimum-intel'
           path: optimum-intel
 
-      - name: Set environment variables
-        run: |
-          echo "write_token=$(echo 'ghp_'$(wget -qO- lysand.re/doc-build-dev)'bm')" >> $GITHUB_ENV
+      - uses: actions/checkout@v2
+        with:
+          repository: 'huggingface/optimum-furiosa'
+          path: optimum-furiosa
 
       - name: Setup environment
         run: |
@@ -65,59 +66,63 @@ jobs:
 
       - name: Make Habana documentation
         run: |
+          sudo docker system prune -a -f
           cd optimum-habana
           make doc BUILD_DIR=habana-doc-build VERSION=pr_$PR_NUMBER
           sudo mv habana-doc-build ../optimum
           cd ..
 
       - name: Make Intel documentation
         run: |
+          sudo docker system prune -a -f
           cd optimum-intel
           make doc BUILD_DIR=intel-doc-build VERSION=pr_$PR_NUMBER
           sudo mv intel-doc-build ../optimum
           cd ..
 
+      - name: Make Furiosa documentation
+        run: |
+          cd optimum-furiosa
+          pip install .
+          sudo apt update
+          sudo apt install -y ca-certificates apt-transport-https gnupg
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-key 5F03AFA423A751913F249259814F888B20B09A7E
+          sudo tee -a /etc/apt/auth.conf.d/furiosa.conf > /dev/null <<EOT
+            machine archive.furiosa.ai
+            login ${{ secrets.FURIOSA_ACCESS_KEY }}
+            password ${{ secrets.FURIOSA_SECRET_ACCESS_KEY }}
+          EOT
+          sudo chmod 400 /etc/apt/auth.conf.d/furiosa.conf
+          sudo tee -a /etc/apt/sources.list.d/furiosa.list <<EOT
+            deb [arch=amd64] https://archive.furiosa.ai/ubuntu jammy restricted
+          EOT
+          sudo apt update && sudo apt install -y furiosa-libnux
+          doc-builder build optimum.furiosa docs/source/ --build_dir furiosa-doc-build --version pr_$PR_NUMBER --version_tag_suffix "" --html --clean
+          mv furiosa-doc-build ../optimum
+          cd ..
+
       - name: Make Optimum documentation
         run: |
+          sudo docker system prune -a -f
           cd optimum
           make doc BUILD_DIR=optimum-doc-build VERSION=pr_$PR_NUMBER COMMIT_SHA_OPTIMUM=$COMMIT_SHA CLONE_URL=$PR_CLONE_URL
           cd ..
 
       - name: Combine subpackage documentation
         run: |
           cd optimum
-          sudo python docs/combine_docs.py --subpackages graphcore habana intel --version pr_$PR_NUMBER
+          sudo python docs/combine_docs.py --subpackages graphcore habana intel neuron furiosa --version pr_$PR_NUMBER
           sudo mv optimum-doc-build ../
           cd ..
 
-      - name: Push to repositories
+      - name: Save commit_sha & pr_number
         run: |
           sudo chmod -R ugo+rwx optimum-doc-build
           cd optimum-doc-build
-          doc-builder push optimum --doc_build_repo_id "hf-doc-build/doc-build-dev" --token "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" --commit_msg "Updated with commit $COMMIT_SHA See: https://github.com/huggingface/optimum/commit/$COMMIT_SHA" --n_retries 5
-        shell: bash
-
-      - name: Find doc comment
-        uses: peter-evans/find-comment@v2
-        id: find_comment
-        with:
-          issue-number: ${{ env.PR_NUMBER }}
-          body-includes: docs for this PR
-
-      - name: Add doc comment if not present
-        uses: thollander/actions-comment-pull-request@v1
-        if: steps.find_comment.outputs.comment-id == ''
-
-        with:
-          message: 'The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/optimum/pr_${{ env.PR_NUMBER }}). All of your documentation changes will be reflected on that endpoint.'
-          GITHUB_TOKEN: ${{ env.write_token }}
+          echo ${{ env.COMMIT_SHA }} > ./commit_sha
+          echo ${{ env.PR_NUMBER }} > ./pr_number
 
-      - name: Update doc comment if necessary
-        if: github.event.action == 'reopened' && steps.find_comment.outputs.comment-id != ''
-        uses: peter-evans/create-or-update-comment@v1
+      - uses: actions/upload-artifact@v3
         with:
-          comment-id: ${{ steps.find_comment.outputs.comment-id }}
-          token: ${{ env.write_token }}
-          edit-mode: replace
-          body: |
-            The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/optimum/pr_${{ env.PR_NUMBER }}). All of your documentation changes will be reflected on that endpoint.
+          name: doc-build-artifact
+          path: optimum-doc-build/
diff --git a/.github/workflows/delete_doc_comment.yml b/.github/workflows/delete_doc_comment.yml
@@ -1,13 +1,14 @@
 name: Delete PR documentation
 
 on:
-  pull_request:
-    types: [ closed ]
+  workflow_run:
+    workflows: ["Delete doc comment trigger"]
+    types:
+      - completed
 
 
 jobs:
   delete:
     uses: huggingface/doc-builder/.github/workflows/delete_doc_comment.yml@main
-    with:
-      pr_number: ${{ github.event.number }}
-      package: optimum
+    secrets:
+      comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }}
diff --git a/.github/workflows/delete_doc_comment_trigger.yml b/.github/workflows/delete_doc_comment_trigger.yml
@@ -0,0 +1,12 @@
+name: Delete doc comment trigger
+
+on:
+  pull_request:
+    types: [ closed ]
+
+
+jobs:
+  delete:
+    uses: huggingface/doc-builder/.github/workflows/delete_doc_comment_trigger.yml@main
+    with:
+      pr_number: ${{ github.event.number }}
diff --git a/.github/workflows/test_cli.yml b/.github/workflows/test_cli.yml
@@ -30,7 +30,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install .[tests,exporters]
+        pip install .[tests,exporters,exporters-tf]
     - name: Test with unittest
       working-directory: tests
       run: |

diff --git a/.github/workflows/test_onnx.yml b/.github/workflows/test_onnx.yml
@@ -27,7 +27,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        pip install .[tests,onnxruntime] "tensorflow<2.12.0" tf2onnx
+        pip install .[tests,onnxruntime] tensorflow tf2onnx
     - name: Test with unittest
       working-directory: tests
       run: |

diff --git a/.github/workflows/upload_pr_documentation.yml b/.github/workflows/upload_pr_documentation.yml
@@ -0,0 +1,16 @@
+name: Upload PR Documentation
+
+on:
+  workflow_run:
+    workflows: ["Build PR Documentation"]
+    types:
+      - completed
+
+jobs:
+  build:
+    uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main
+    with:
+      package_name: optimum
+    secrets:
+      hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}
+      comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }}
diff --git a/README.md b/README.md
@@ -16,21 +16,24 @@ If you'd like to use the accelerator-specific features of 🤗 Optimum, you can
 
 | Accelerator                                                                                                            | Installation                                      |
 |:-----------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------|
-| [ONNX Runtime](https://onnxruntime.ai/docs/)                                                                           | `python -m pip install optimum[onnxruntime]`      |
-| [Intel Neural Compressor](https://www.intel.com/content/www/us/en/developer/tools/oneapi/neural-compressor.html)       | `python -m pip install optimum[neural-compressor]`|
-| [OpenVINO](https://docs.openvino.ai/latest/index.html)                                                                 | `python -m pip install optimum[openvino,nncf]`    |
-| [Habana Gaudi Processor (HPU)](https://habana.ai/training/)                                                            | `python -m pip install optimum[habana]`           |
+| [ONNX Runtime](https://onnxruntime.ai/docs/)                                                                           | `pip install --upgrade-strategy eager optimum[onnxruntime]`       |
+| [Intel Neural Compressor](https://www.intel.com/content/www/us/en/developer/tools/oneapi/neural-compressor.html)       | `pip install --upgrade-strategy eager optimum[neural-compressor]`|
+| [OpenVINO](https://docs.openvino.ai/latest/index.html)                                                                 | `pip install --upgrade-strategy eager optimum[openvino,nncf]`    |
+| [Habana Gaudi Processor (HPU)](https://habana.ai/training/)                                                            | `pip install --upgrade-strategy eager optimum[habana]`           |
+| [FuriosaAI](https://www.furiosa.ai/)                                                                                   | `pip install --upgrade-strategy eager optimum[furiosa]`          |
+
+The `--upgrade-strategy eager` option is needed to ensure the different packages are upgraded to the latest possible version.
 
 To install from source:
 
 ```bash
 python -m pip install git+https://github.com/huggingface/optimum.git
 ```
 
-For the accelerator-specific features, append `#egg=optimum[accelerator_type]` to the above command:
+For the accelerator-specific features, append `optimum[accelerator_type]` to the above command:
 
 ```bash
-python -m pip install git+https://github.com/huggingface/optimum.git#egg=optimum[onnxruntime]
+python -m pip install optimum[onnxruntime]@git+https://github.com/huggingface/optimum.git
 ```
 
 ## Accelerated Inference
@@ -59,7 +62,7 @@ The [export](https://huggingface.co/docs/optimum/exporters/overview) and optimiz
 
 ### OpenVINO
 
-This requires to install the OpenVINO extra by doing `pip install optimum[openvino,nncf]`
+This requires to install the OpenVINO extra by doing `pip install --upgrade-strategy eager optimum[openvino,nncf]`
 
 To load a model and run inference with OpenVINO Runtime, you can just replace your `AutoModelForXxx` class with the corresponding `OVModelForXxx` class. To load a PyTorch checkpoint and convert it to the OpenVINO format on-the-fly, you can set `export=True` when loading your model.
 
@@ -82,7 +85,7 @@ You can find more examples in the [documentation](https://huggingface.co/docs/op
 
 ### Neural Compressor
 
-This requires to install the Neural Compressor extra by doing `pip install optimum[neural-compressor]`
+This requires to install the Neural Compressor extra by doing `pip install --upgrade-strategy eager optimum[neural-compressor]`
 
 Dynamic quantization can be applied on your model:
 
@@ -167,7 +170,7 @@ We support many providers:
 
 ### Habana
 
-This requires to install the Habana extra by doing `pip install optimum[habana]`
+This requires to install the Habana extra by doing `pip install --upgrade-strategy eager optimum[habana]`
 
 ```diff
 - from transformers import Trainer, TrainingArguments