[RHOAIEDGE-15] Adding option to fetch from git

pipelines/README.md

@@ -32,7 +32,7 @@ They are also referenced by these names in the example pipeline YAML files.
   - All the following steps can and should be done as a regular user with no extra privileges.
 - OpenShift project / namespace. E.g.  `oc new-project model-to-edge`
 - A repository on [Quay.io](https://quay.io/)
-- S3 bucket for storing the models
+- S3 bucket for storing the models or Git repository storing the model files
 - A clone of this repository
 
 ## Deploy build pipeline for AI runtime container images
@@ -56,12 +56,16 @@ oc create -f tekton/build-container-image-pipeline/aws-env-real.yaml
 
 ### Deploy and run the build pipeline
 
+#### For S3 fetch
 Update the `aws-bucket-name` parameter value from its default `rhoai-edge-models` in
 [`build-container-image-pipelinerun-bike-rentals.yaml`](tekton/build-container-image-pipeline/build-container-image-pipelinerun-bike-rentals.yaml)
 and/or
 [`build-container-image-pipelinerun-tensorflow-housing.yaml`](tekton/build-container-image-pipeline/build-container-image-pipelinerun-tensorflow-housing.yaml)
 to match your S3 bucket name.
 
+#### For Git fetch
+Update the `git-model-repo` parameter with the repository url, the `modelRelativePath` parameter to the model files path and the `git-revision` parameter for the version/branch of the repository in [`build-container-image-pipelinerun-bike-rentals.yaml`](tekton/build-container-image-pipeline/build-container-image-pipelinerun-bike-rentals.yaml).
+
 Then create the pipeline(s) to build the container image with AI runtime:
 
 ```bash

pipelines/tekton/build-container-image-pipeline/build-container-image-pipeline.yaml

@@ -19,7 +19,37 @@ spec:
     type: string
   - name: containerfileRelativePath
     type: string
+  - name: modelRelativePath
+    type: string
+  - name: fetch-model
+    type: string
+  - name: git-model-repo
+    type: string
+    default: ""
+  - name: git-revision
+    type: string
+    default: "main"
   tasks:
+  - name: git-clone-model-repo
+    taskRef:
+      kind: ClusterTask
+      name: git-clone
+    params:
+    - name: url
+      value: $(params.git-model-repo)
+    - name: revision
+      value: $(params.git-revision)
+    - name: gitInitImage
+      value: registry.redhat.io/openshift-pipelines/pipelines-git-init-rhel8@sha256:1a50511583fc02a27012d17d942e247813404104ddd282d7e26f99765174392c
+    - name: subdirectory
+      value: /model_dir/
+    workspaces:
+    - name: output
+      workspace: buildah-cache
+    when:
+    - input: "$(params.fetch-model)"
+      operator: in
+      values: ["git"]
   - name: kserve-download-model
     params:
     - name: model-name
@@ -36,34 +66,41 @@ spec:
       workspace: buildah-cache
     - name: aws-secret
       workspace: aws-secret
+    when:
+    - input: "$(params.fetch-model)"
+      operator: in
+      values: ["s3"]
   - name: git-clone-containerfile-repo
     params:
-      - name: url
-        value: $(params.gitServer)/$(params.gitOrgName)/$(params.gitRepoName)
-      - name: gitInitImage
-        value: registry.redhat.io/openshift-pipelines/pipelines-git-init-rhel8@sha256:1a50511583fc02a27012d17d942e247813404104ddd282d7e26f99765174392c
-      - name: subdirectory
-        value: /containerfile_repo/
+    - name: url
+      value: $(params.gitServer)/$(params.gitOrgName)/$(params.gitRepoName)
+    - name: gitInitImage
+      value: registry.redhat.io/openshift-pipelines/pipelines-git-init-rhel8@sha256:1a50511583fc02a27012d17d942e247813404104ddd282d7e26f99765174392c
+    - name: subdirectory
+      value: /containerfile_repo/
     runAfter:
-      - kserve-download-model
+    - kserve-download-model
+    - git-clone-model-repo
     taskRef:
       kind: ClusterTask
       name: git-clone
     workspaces:
-      - name: output
-        workspace: buildah-cache
-      - name: basic-auth
-        workspace: git-basic-auth
+    - name: output
+      workspace: buildah-cache
+    - name: basic-auth
+      workspace: git-basic-auth
   - name: check-model-and-containerfile-exists
     params:
     - name: model-name
       value: $(params.model-name)
+    - name: modelRelativePath
+      value: model_dir/$(params.modelRelativePath)
     - name: containerfilePath
       value: containerfile_repo/$(params.containerfileRelativePath)
     - name: current-namespace
       value: $(context.pipelineRun.namespace)
     runAfter:
-      - git-clone-containerfile-repo
+    - git-clone-containerfile-repo
     taskRef:
       kind: Task
       name: check-model-and-containerfile-exists
@@ -78,7 +115,7 @@ spec:
     - name: VERSION
       value: latest
     runAfter:
-     - check-model-and-containerfile-exists
+    - check-model-and-containerfile-exists
     taskRef:
       kind: ClusterTask
       name: openshift-client
@@ -93,7 +130,7 @@ spec:
     - name: DOCKERFILE
       value: containerfile_repo/$(params.containerfileRelativePath)
     - name: CONTEXT
-      value: $(params.model-name)
+      value: model_dir/$(params.modelRelativePath)/$(params.model-name)
     - name: TLSVERIFY
       value: "true"
     - name: FORMAT

pipelines/tekton/build-container-image-pipeline/build-container-image-pipelinerun-bike-rentals.yaml

@@ -20,6 +20,14 @@ spec:
     value: ai-edge
   - name: containerfileRelativePath
     value: pipelines/containerfiles/Containerfile.seldonio.mlserver.mlflow
+  - name: fetch-model
+    value: "git"
+  - name: git-model-repo
+    value: https://github.com/opendatahub-io/ai-edge.git
+  - name: modelRelativePath
+    value: pipelines/models/
+  - name: git-revision
+    value: "main"
   pipelineRef:
     name: build-container-image
   serviceAccountName: pipeline

pipelines/tekton/build-container-image-pipeline/build-container-image-pipelinerun-tensorflow-housing.yaml

@@ -20,6 +20,10 @@ spec:
     value: ai-edge
   - name: containerfileRelativePath
     value: pipelines/containerfiles/Containerfile.openvino.mlserver.mlflow
+  - name: fetch-model
+    value: "s3"
+  - name: modelRelativePath
+    value: ""
   pipelineRef:
     name: build-container-image
   serviceAccountName: pipeline

pipelines/tekton/build-container-image-pipeline/check-model-and-containerfile-exists.yaml

@@ -12,14 +12,20 @@ spec:
     type: string
   - name: containerfilePath
     type: string
+  - name: modelRelativePath
+    type: string
   steps:
   - name: check-model-and-containerfile
     image: registry.access.redhat.com/ubi9/ubi-micro
     script: |
       #!/usr/bin/env bash
 
       # Check model files exist
-      ls -l $(params.model-name)/
+      if [ -n "$(params.modelRelativePath)" ]; then
+        ls -l model_dir/$(params.modelRelativePath)/$(params.model-name)/
+      else
+        ls -l model_dir/$(params.model-name)/
+      fi
 
       # Check containerfile exists
       cat $(params.containerfilePath)

pipelines/tekton/build-container-image-pipeline/kserve-download-model.yaml

@@ -13,12 +13,13 @@ spec:
   - name: aws-bucket-name
     type: string
   steps:
-  - name: download-model
+  - name: download-model-s3
     image: quay.io/opendatahub/kserve-storage-initializer:v0.11
     script: |
+      mkdir -p $(workspaces.workspace.path)/model_dir/
       STORAGE_CONFIG="$(cat $(workspaces.aws-secret.path)/aws-storage-config)" /storage-initializer/scripts/initializer-entrypoint \
       's3://$(params.aws-bucket-name)/$(params.model-name)' \
-      $(workspaces.workspace.path)/$(params.model-name)
+      $(workspaces.workspace.path)/model_dir/$(params.model-name)
   workspaces:
   - description: The workspace for the downloaded model.
     name: workspace