Support arm64 for Hugging Face trainer (kubeflow#2028)

* echoed disk usage before cleanup Signed-off-by: tariq-hasan <[email protected]> * pruned docker images Signed-off-by: tariq-hasan <[email protected]> * moved docker data directory Signed-off-by: tariq-hasan <[email protected]> * added arm64 in the list of platforms for trainer-huggingface Signed-off-by: tariq-hasan <[email protected]> --------- Signed-off-by: tariq-hasan <[email protected]>
andreyvelich · Mar 13, 2024 · 8433edc · 8433edc
1 parent 395e8ca
commit 8433edc
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 4 deletions.
diff --git a/.github/workflows/publish-core-images.yaml b/.github/workflows/publish-core-images.yaml
@@ -34,4 +34,4 @@ jobs:
           - component-name: trainer-huggingface
             dockerfile: sdk/python/kubeflow/trainer/Dockerfile
             context: sdk/python/kubeflow/trainer
-            platforms: linux/amd64
+            platforms: linux/amd64,linux/arm64
diff --git a/.github/workflows/template-publish-image/action.yaml b/.github/workflows/template-publish-image/action.yaml
@@ -28,16 +28,44 @@ runs:
     - name: Remove unnecessary files
       shell: bash
       run: |
+        echo "Disk usage before cleanup:"
+        df -hT
+
         sudo rm -rf /usr/share/dotnet
         sudo rm -rf /opt/ghc
-        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf /usr/local/share/boost
         sudo rm -rf "$AGENT_TOOLSDIRECTORY"
         sudo rm -rf /usr/local/lib/android
         sudo rm -rf /usr/local/share/powershell
         sudo rm -rf /usr/share/swift
-        
+
         echo "Disk usage after cleanup:"
-        df -h
+        df -hT
+
+    - name: Prune docker images
+      shell: bash
+      run: |
+        docker image prune -a -f
+        docker system df
+        df -hT
+
+    - name: Move docker data directory
+      shell: bash
+      run: |
+        echo "Stopping docker service ..."
+        sudo systemctl stop docker
+        DOCKER_DEFAULT_ROOT_DIR=/var/lib/docker
+        DOCKER_ROOT_DIR=/mnt/docker
+        echo "Moving ${DOCKER_DEFAULT_ROOT_DIR} -> ${DOCKER_ROOT_DIR}"
+        sudo mv ${DOCKER_DEFAULT_ROOT_DIR} ${DOCKER_ROOT_DIR}
+        echo "Creating symlink ${DOCKER_DEFAULT_ROOT_DIR} -> ${DOCKER_ROOT_DIR}"
+        sudo ln -s ${DOCKER_ROOT_DIR} ${DOCKER_DEFAULT_ROOT_DIR}
+        echo "$(sudo ls -l ${DOCKER_DEFAULT_ROOT_DIR})"
+        echo "Starting docker service ..."
+        sudo systemctl daemon-reload
+        sudo systemctl start docker
+        echo "Docker service status:"
+        sudo systemctl --no-pager -l -o short status docker
 
     - name: Setup QEMU
       uses: docker/setup-qemu-action@v2