Merge remote-tracking branch 'upstream/master' into add-e2e-test-for-…

…train-api merge upstream/master into local branch
kubeflow · Sep 21, 2024 · c3f04c3 · c3f04c3
2 parents d2c3cac + da11d11
commit c3f04c3
Show file tree

Hide file tree

Showing 235 changed files with 31,888 additions and 5,325 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,7 +1,7 @@
 <!--  Thanks for sending a pull request! Here are some tips for you:
 1. If this is your first time, check our contributor guidelines: https://www.kubeflow.org/docs/about/contributing
 2. To know more about Training Operator, check the developer guide:
-    https://github.com/kubeflow/training-operator/blob/master/docs/development/developer_guide.md
+    https://github.com/kubeflow/training-operator/blob/master/CONTRIBUTING.md
 3. If you want *faster* PR reviews, check how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
 -->
 

diff --git a/.github/workflows/build-and-publish-images.yaml b/.github/workflows/build-and-publish-images.yaml
@@ -17,6 +17,10 @@ on:
         required: false
         type: string
         default: .
+      tag-prefix:
+        required: false
+        type: string
+        default: v1
     secrets:
       DOCKERHUB_USERNAME:
         required: false
@@ -57,6 +61,7 @@ jobs:
           platforms: ${{ inputs.platforms }}
           context: ${{ inputs.context }}
           push: true
+          tag-prefix: ${{ inputs.tag-prefix }}
 
       - name: Test Build For Component ${{ inputs.component-name }}
         if: steps.publish.outcome == 'skipped'
@@ -67,3 +72,4 @@ jobs:
           platforms: ${{ inputs.platforms }}
           context: ${{ inputs.context }}
           push: false
+          tag-prefix: ${{ inputs.tag-prefix }}
diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml
@@ -0,0 +1,14 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
+    - uses: pre-commit/[email protected]
diff --git a/.github/workflows/publish-core-images.yaml b/.github/workflows/publish-core-images.yaml
@@ -13,6 +13,7 @@ jobs:
       platforms: ${{ matrix.platforms }}
       dockerfile: ${{ matrix.dockerfile }}
       context: ${{ matrix.context }}
+      tag-prefix: ${{ matrix.tag-prefix }}
     secrets:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -24,14 +25,22 @@ jobs:
           - component-name: training-operator
             dockerfile: build/images/training-operator/Dockerfile
             platforms: linux/amd64,linux/arm64,linux/ppc64le
+            tag-prefix: v1
+          - component-name: training-operator
+            dockerfile: cmd/training-operator.v2alpha1/Dockerfile
+            platforms: linux/amd64,linux/arm64,linux/ppc64le
+            tag-prefix: v2alpha1
           - component-name: kubectl-delivery
             dockerfile: build/images/kubectl-delivery/Dockerfile
             platforms: linux/amd64,linux/arm64,linux/ppc64le
+            tag-prefix: v1
           - component-name: storage-initializer
             dockerfile: sdk/python/kubeflow/storage_initializer/Dockerfile
             context: sdk/python/kubeflow/storage_initializer
             platforms: linux/amd64,linux/arm64
+            tag-prefix: v1
           - component-name: trainer-huggingface
             dockerfile: sdk/python/kubeflow/trainer/Dockerfile
             context: sdk/python/kubeflow/trainer
             platforms: linux/amd64,linux/arm64
+            tag-prefix: v1
diff --git a/.github/workflows/publish-example-images.yaml b/.github/workflows/publish-example-images.yaml
@@ -61,11 +61,6 @@ jobs:
             platforms: linux/amd64,linux/arm64
             dockerfile: examples/xgboost/smoke-dist/Dockerfile
             context: examples/xgboost/smoke-dist
-          - component-name: mxnet-gpu
-            dockerfile: examples/mxnet/train/Dockerfile
-          - component-name: mxnet-auto-tuning
-            dockerfile: examples/mxnet/tune/Dockerfile
-            context: examples/mxnet/tune
           - component-name: pytorch-dist-mnist
             platforms: linux/amd64,linux/arm64
             dockerfile: examples/pytorch/mnist/Dockerfile
@@ -74,3 +69,7 @@ jobs:
             platforms: linux/amd64,linux/arm64
             dockerfile: examples/pytorch/mnist/Dockerfile-mpi
             context: examples/pytorch/mnist
+          - component-name: jaxjob-simple
+            platforms: linux/amd64,linux/arm64
+            dockerfile: examples/jax/cpu-demo/Dockerfile
+            context: examples/jax/cpu-demo
diff --git a/.github/workflows/template-publish-image/action.yaml b/.github/workflows/template-publish-image/action.yaml
@@ -19,6 +19,10 @@ inputs:
   push:
     required: true
     description: whether to push container images or not
+  tag-prefix:
+    required: false
+    default: v1
+    description: Prefix for the image tag, e.g. v1 or v2alpha1
 
 runs:
   using: composite
@@ -38,7 +42,7 @@ runs:
         images: ${{ inputs.image }}
         tags: |
           type=raw,latest
-          type=sha,prefix=v1-
+          type=sha,prefix=${{ inputs.tag-prefix }}-
 
     - name: Build and Push
       uses: docker/build-push-action@v5

diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml
@@ -27,7 +27,7 @@ jobs:
 
       - name: Check Go modules
         run: |
-          go mod tidy && git add go.* &&
+          go mod tidy && pushd hack/swagger && go mod tidy && popd && git add go.* &&
           git diff --cached --exit-code || (echo 'Please run "go mod tidy" to sync Go modules' && exit 1);
       - name: Check manifests
         run: |

diff --git a/.github/workflows/test-python.yaml b/.github/workflows/test-python.yaml
@@ -26,15 +26,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-        # TODO (andreyvelich): We need to replace this action with script to do
-        # linting and formatting for Training Operator SDK.
-      - name: Check Python code with Black
-        uses: psf/black@stable
-        with:
-          version: 24.2.0
-          options: --check --exclude '/*kubeflow_org_v1*|__init__.py|api_client.py|configuration.py|exceptions.py|rest.py'
-          src: sdk/
-
       - name: Install dependencies
         run: |
           pip install pytest python-dateutil urllib3 kubernetes

diff --git a/.github/workflows/trigger-rerun-test.yaml b/.github/workflows/trigger-rerun-test.yaml
@@ -0,0 +1,16 @@
+name: Re-Run PR Tests
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  rerun_pr_tests:
+    name: rerun_pr_tests
+    if: ${{ github.event.issue.pull_request }}
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: estroz/rerun-actions@main
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          comment_id: ${{ github.event.comment.id }}
diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml
@@ -31,10 +31,14 @@ jobs:
         with:
           go-version-file: ${{ env.GOPATH }}/src/github.com/kubeflow/training-operator/go.mod
 
-      - name: Run Go test
+      - name: Run Go test for v1
         run: |
           make test ENVTEST_K8S_VERSION=${{ matrix.kubernetes-version }}
 
+      - name: Run Go test for v2
+        run: |
+          make test-integrationv2 ENVTEST_K8S_VERSION=${{ matrix.kubernetes-version }}
+
       - name: Coveralls report
         uses: shogo82148/actions-goveralls@v1
         with:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,58 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: check-yaml
+        args: [--allow-multiple-documents]
+      - id: check-json
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/pycqa/isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+        name: isort
+        entry: isort --profile black
+  - repo: https://github.com/psf/black
+    rev: 24.2.0
+    hooks:
+      - id: black
+        files: (sdk|examples)/.*
+        exclude: |
+          (?x)^(
+            /*kubeflow_org_v1*|
+            __init__.py|
+            api_client.py|
+            configuration.py|
+            exceptions.py|
+            rest.py
+          )$
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+      - id: flake8
+        files: ^(sdk|examples)/.*$
+        exclude: |
+          (?x)^(
+            .*kubeflow_org_v1.*|
+            __init__\.py|
+            api_client\.py|
+            configuration\.py|
+            exceptions\.py|
+            rest\.py
+          )$
+exclude: |
+  (?x)^(
+    pkg/apis/kubeflow.org/v1/openapi_generated.go|
+    pkg/apis/kubeflow.org/v1/zz_.*|
+    pkg/client/.*|
+    test_job/apis/test_job/v1/.*generated.*.go|
+    test_job/client/.*|
+    sdk/python/kubeflow/training/[^/]*.py|
+    sdk/python/kubeflow/training/models/.*|
+    sdk/python/test/.*|
+    docs/api/.*|
+    sdk/python/docs/.*|
+    sdk/python/.openapi-generator/VERSION|
+    sdk/python/kubeflow/__init__.py
+  )$
diff --git a/docs/adopters.md → ADOPTERS.md b/docs/adopters.md → ADOPTERS.md