Merge remote-tracking branch 'upstream/main' into andreyan/exporters_…

…model_configs

.github/workflows/build_main_documentation.yml

@@ -44,6 +44,11 @@ jobs:
           repository: 'huggingface/optimum-intel'
           path: optimum-intel
 
+      - uses: actions/checkout@v2
+        with:
+          repository: 'huggingface/optimum-furiosa'
+          path: optimum-furiosa
+
       - name: Set environment variables
         run: |
           cd optimum
@@ -76,18 +81,41 @@ jobs:
 
       - name: Make Habana documentation
         run: |
+          sudo docker system prune -a -f
           cd optimum-habana
           make doc BUILD_DIR=habana-doc-build VERSION=${{ env.VERSION }}
           sudo mv habana-doc-build ../optimum
           cd ..
 
       - name: Make Intel documentation
         run: |
+          sudo docker system prune -a -f
           cd optimum-intel
           make doc BUILD_DIR=intel-doc-build VERSION=${{ env.VERSION }}
           sudo mv intel-doc-build ../optimum
           cd ..
 
+      - name: Make Furiosa documentation
+        run: |
+          cd optimum-furiosa
+          pip install .
+          sudo apt update
+          sudo apt install -y ca-certificates apt-transport-https gnupg
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-key 5F03AFA423A751913F249259814F888B20B09A7E
+          sudo tee -a /etc/apt/auth.conf.d/furiosa.conf > /dev/null <<EOT
+            machine archive.furiosa.ai
+            login ${{ secrets.FURIOSA_ACCESS_KEY }}
+            password ${{ secrets.FURIOSA_SECRET_ACCESS_KEY }}
+          EOT
+          sudo chmod 400 /etc/apt/auth.conf.d/furiosa.conf
+          sudo tee -a /etc/apt/sources.list.d/furiosa.list <<EOT
+            deb [arch=amd64] https://archive.furiosa.ai/ubuntu jammy restricted
+          EOT
+          sudo apt update && sudo apt install -y furiosa-libnux
+          doc-builder build optimum.furiosa docs/source/ --build_dir furiosa-doc-build --version pr_$PR_NUMBER --version_tag_suffix "" --html --clean
+          mv furiosa-doc-build ../optimum
+          cd ..
+
       - name: Make Optimum documentation
         run: |
           sudo docker system prune -a -f
@@ -101,12 +129,12 @@ jobs:
       - name: Combine subpackage documentation
         run: |
           cd optimum
-          sudo python docs/combine_docs.py --subpackages graphcore habana intel --version ${{ env.VERSION }}
+          sudo python docs/combine_docs.py --subpackages graphcore habana intel neuron furiosa --version ${{ env.VERSION }}
           cd ..
 
       - name: Push to repositories
         run: |
           cd optimum/optimum-doc-build
-          chmod ugo+rwx optimum
+          sudo chmod -R ugo+rwx optimum
           doc-builder push optimum --doc_build_repo_id "hf-doc-build/doc-build" --token "${{ secrets.HF_DOC_BUILD_PUSH }}" --commit_msg "Updated with commit ${{ github.sha }} See: https://github.com/huggingface/optimum/commit/${{ github.sha }}" --n_retries 5 --upload_version_yml
         shell: bash

.github/workflows/build_pr_documentation.yml

@@ -43,6 +43,11 @@ jobs:
           repository: 'huggingface/optimum-intel'
           path: optimum-intel
 
+      - uses: actions/checkout@v2
+        with:
+          repository: 'huggingface/optimum-furiosa'
+          path: optimum-furiosa
+
       - name: Setup environment
         run: |
           pip uninstall -y doc-builder
@@ -61,18 +66,41 @@ jobs:
 
       - name: Make Habana documentation
         run: |
+          sudo docker system prune -a -f
           cd optimum-habana
           make doc BUILD_DIR=habana-doc-build VERSION=pr_$PR_NUMBER
           sudo mv habana-doc-build ../optimum
           cd ..
 
       - name: Make Intel documentation
         run: |
+          sudo docker system prune -a -f
           cd optimum-intel
           make doc BUILD_DIR=intel-doc-build VERSION=pr_$PR_NUMBER
           sudo mv intel-doc-build ../optimum
           cd ..
 
+      - name: Make Furiosa documentation
+        run: |
+          cd optimum-furiosa
+          pip install .
+          sudo apt update
+          sudo apt install -y ca-certificates apt-transport-https gnupg
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-key 5F03AFA423A751913F249259814F888B20B09A7E
+          sudo tee -a /etc/apt/auth.conf.d/furiosa.conf > /dev/null <<EOT
+            machine archive.furiosa.ai
+            login ${{ secrets.FURIOSA_ACCESS_KEY }}
+            password ${{ secrets.FURIOSA_SECRET_ACCESS_KEY }}
+          EOT
+          sudo chmod 400 /etc/apt/auth.conf.d/furiosa.conf
+          sudo tee -a /etc/apt/sources.list.d/furiosa.list <<EOT
+            deb [arch=amd64] https://archive.furiosa.ai/ubuntu jammy restricted
+          EOT
+          sudo apt update && sudo apt install -y furiosa-libnux
+          doc-builder build optimum.furiosa docs/source/ --build_dir furiosa-doc-build --version pr_$PR_NUMBER --version_tag_suffix "" --html --clean
+          mv furiosa-doc-build ../optimum
+          cd ..
+
       - name: Make Optimum documentation
         run: |
           sudo docker system prune -a -f
@@ -83,7 +111,7 @@ jobs:
       - name: Combine subpackage documentation
         run: |
           cd optimum
-          sudo python docs/combine_docs.py --subpackages graphcore habana intel --version pr_$PR_NUMBER
+          sudo python docs/combine_docs.py --subpackages graphcore habana intel neuron furiosa --version pr_$PR_NUMBER
           sudo mv optimum-doc-build ../
           cd ..
 

.github/workflows/test_onnx.yml

@@ -27,7 +27,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        pip install .[tests,onnxruntime] "tensorflow<2.12.0" tf2onnx
+        pip install .[tests,onnxruntime] tensorflow tf2onnx
     - name: Test with unittest
       working-directory: tests
       run: |

docs/combine_docs.py

@@ -1,14 +1,11 @@
 import argparse
 import shutil
 from pathlib import Path
-from typing import Dict
+from typing import Dict, List
 
 import yaml
 
 
-SECTIONS_AT_THE_END = ["Utilities"]
-
-
 parser = argparse.ArgumentParser(
     description="Script to combine doc builds from subpackages with base doc build of Optimum. "
     "Assumes all subpackage doc builds are present in the root of the `optimum` repo."
@@ -23,7 +20,7 @@
 
 def rename_subpackage_toc(subpackage: str, toc: Dict):
     """
-    Extend table of contents sections with the subpackage name as the parent folder.
+    Extends table of contents sections with the subpackage name as the parent folder.
 
     Args:
         subpackage (str): subpackage name.
@@ -83,6 +80,31 @@ def rename_copy_subpackage_html_paths(subpackage: str, subpackage_path: Path, op
         shutil.copyfile(html_path, new_path_in_optimum)
 
 
+def add_neuron_doc(base_toc: List):
+    """
+    Extends the table of content with a section about Optimum Neuron.
+
+    Args:
+        base_toc (List): table of content for the doc of Optimum.
+    """
+    # Update optimum table of contents
+    base_toc.insert(
+        1,
+        {
+            "sections": [
+                {
+                    # Ideally this should directly point at https://huggingface.co/docs/optimum-neuron/index
+                    # Current hacky solution is to have a redirection in _redirects.yml
+                    "local": "docs/optimum-neuron/index",
+                    "title": "🤗 Optimum Neuron",
+                }
+            ],
+            "title": "AWS Trainium/Inferentia",
+            "isExpanded": False,
+        },
+    )
+
+
 def main():
     args = parser.parse_args()
     optimum_path = Path("optimum-doc-build")
@@ -91,39 +113,34 @@ def main():
     with open(base_toc_path, "r") as f:
         base_toc = yaml.safe_load(f)
 
-    # Pop specific sections to add them after subpackages
-    sections_to_pop = {title: None for title in SECTIONS_AT_THE_END}
-    for i, section in enumerate(base_toc[:]):
-        if section["title"] in SECTIONS_AT_THE_END:
-            sections_to_pop[section["title"]] = base_toc.pop(i)
-    # Raise an error if a section was not found
-    for key, value in sections_to_pop.items():
-        if value is None:
-            raise ValueError(f"No section was found for title '{key}'.")
-
     # Copy and rename all files from subpackages' docs to Optimum doc
-    for subpackage in args.subpackages:
-        subpackage_path = Path(f"{subpackage}-doc-build")
-
-        # Copy all HTML files from subpackage into optimum
-        rename_copy_subpackage_html_paths(
-            subpackage,
-            subpackage_path,
-            optimum_path,
-            args.version,
-        )
+    for subpackage in args.subpackages[::-1]:
+        if subpackage == "neuron":
+            # Neuron has its own doc so it is managed differently
+            add_neuron_doc(base_toc)
+        else:
+            subpackage_path = Path(f"{subpackage}-doc-build")
+
+            # Copy all HTML files from subpackage into optimum
+            rename_copy_subpackage_html_paths(
+                subpackage,
+                subpackage_path,
+                optimum_path,
+                args.version,
+            )
+
+            # Load subpackage table of contents
+            subpackage_toc_path = next(subpackage_path.rglob("_toctree.yml"))
+            with open(subpackage_toc_path, "r") as f:
+                subpackage_toc = yaml.safe_load(f)
+            # Extend table of contents sections with the subpackage name as the parent folder
+            rename_subpackage_toc(subpackage, subpackage_toc)
+            # Just keep the name of the partner in the TOC title
+            subpackage_toc[0]["title"] = subpackage_toc[0]["title"].split("Optimum ")[-1]
+            if subpackage != "graphcore":
+                # Update optimum table of contents
+                base_toc.insert(1, subpackage_toc[0])
 
-        # Load subpackage table of contents
-        subpackage_toc_path = next(subpackage_path.rglob("_toctree.yml"))
-        with open(subpackage_toc_path, "r") as f:
-            subpackage_toc = yaml.safe_load(f)
-        # Extend table of contents sections with the subpackage name as the parent folder
-        rename_subpackage_toc(subpackage, subpackage_toc)
-        # Update optimum table of contents
-        base_toc.extend(subpackage_toc)
-
-    # Add popped sections at the end
-    base_toc.extend(sections_to_pop.values())
     # Write final table of contents
     with open(base_toc_path, "w") as f:
         yaml.safe_dump(base_toc, f, allow_unicode=True)

docs/source/_redirects.yml

@@ -25,3 +25,6 @@ intel_optimization: intel/optimization_inc
 intel_quantization: intel/optimization_inc
 intel_pruning: intel/optimization_inc
 intel_trainer: intel/reference_inc
+
+# Optimum Neuron
+docs/optimum-neuron/index: /docs/optimum-neuron/index

docs/source/_toctree.yml

@@ -51,6 +51,7 @@
     title: Reference
     isExpanded: false
   title: ONNX Runtime
+  isExpanded: false
 - sections:
   - local: exporters/overview
     title: Overview
@@ -91,6 +92,7 @@
       isExpanded: false
     title: "TFLite"
   title: Exporters
+  isExpanded: false
 - sections:
   - local: torch_fx/overview
     title: Overview
@@ -110,6 +112,7 @@
     title: Reference
     isExpanded: false
   title: Torch FX
+  isExpanded: false
 - sections:
   - local: bettertransformer/overview
     title: Overview
@@ -121,6 +124,7 @@
     title: Tutorials
     isExpanded: false
   title: BetterTransformer
+  isExpanded: false
 - sections:
   - local: utils/dummy_input_generators
     title: Dummy input generators

docs/source/bettertransformer/overview.mdx

@@ -32,6 +32,7 @@ The list of supported model below:
 - [BART](https://arxiv.org/abs/1910.13461)
 - [BERT](https://arxiv.org/abs/1810.04805)
 - [BERT-generation](https://arxiv.org/abs/1907.12461)
+- [BLIP-2](https://arxiv.org/abs/2301.12597)
 - [CamemBERT](https://arxiv.org/abs/1911.03894)
 - [CLIP](https://arxiv.org/abs/2103.00020)
 - [CodeGen](https://arxiv.org/abs/2203.13474)