update deps

Signed-off-by: kpango <[email protected]>
vdaas · Jul 27, 2024 · fc6fc82 · fc6fc82
1 parent caa8f03
commit fc6fc82
Show file tree

Hide file tree

Showing 32 changed files with 167 additions and 90 deletions.
diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml
@@ -19,7 +19,7 @@ inputs:
   require_libhdf5:
     description: "If libhdf5 is not required, set this to false"
     required: false
-    default: "true"
+    default: "false"
   require_go:
     description: "If go is not required, set this to false"
     required: false

diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml
@@ -93,9 +93,23 @@ runs:
       shell: bash
       id: start_k3d
       run: |
-        k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }}
+        k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} --api-port 0.0.0.0:6443
+        k3d kubeconfig merge ${{ inputs.name }} --kubeconfig-switch-context
     - name: Ready k8s cluster
       shell: bash
       run: |
         k3d cluster list
+    - name: Set k3d cluster config to KUBECONFIG
+      shell: bash
+      run: |
+        echo "KUBECONFIG=$(k3d kubeconfig write ${{ inputs.name }})" >> $GITHUB_ENV
+    - name: Check KUBECONFIG environment variable
+      shell: bash
+      run: |
+        echo $KUBECONFIG
+        cat $KUBECONFIG
+        cat /etc/hosts
+    - name: Show Kubernetes Cluster Info
+      shell: bash
+      run: |
         kubectl cluster-info dump
diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml
@@ -35,11 +35,15 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
+  detect-ci-container:
+    uses: ./.github/workflows/_detect-ci-container.yml
   agent-failure:
     name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -75,9 +79,11 @@ jobs:
             e2e/insert/search
   random-pod-failure:
     name: "E2E chaos test (random Pod failure: to test redundancy)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -113,9 +119,11 @@ jobs:
             e2e/insert/search
   agent-network-partition:
     name: "E2E chaos test (agent network partition: to test retries)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -151,9 +159,11 @@ jobs:
             e2e/insert/search
   clusterwide-network-bandwidth:
     name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config

diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml
@@ -32,11 +32,15 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
+  detect-ci-container:
+    uses: ./.github/workflows/_detect-ci-container.yml
   e2e-max-dimension-insert:
     name: "E2E test (Max Dimension Insert: skip strict exist check)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config

diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml
@@ -33,11 +33,15 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
+  detect-ci-container:
+    uses: ./.github/workflows/_detect-ci-container.yml
   e2e-profiling:
     name: "E2E profiling"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -33,11 +33,15 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
+  detect-ci-container:
+    uses: ./.github/workflows/_detect-ci-container.yml
   e2e-stream-crud:
     name: "E2E test (Stream CRUD)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -73,9 +77,11 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-stream-crud-for-operator:
     name: "E2E test (Stream CRUD) for operator"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -133,9 +139,11 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-stream-crud-under-index-management-jobs:
     name: "E2E test (Stream CRUD) under index management jobs"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -175,9 +183,11 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-stream-crud-skip-exist-check:
     name: "E2E test (Stream CRUD: skip strict exist check)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -213,9 +223,11 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-multiapis-crud:
     name: "E2E test (Multi-APIs CRUD)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -247,9 +259,11 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-jobs:
     name: "E2E test (Jobs)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -278,9 +292,11 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-stream-crud-with-readreplica:
     name: "E2E test (Stream CRUD) with read replica"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -326,9 +342,11 @@ jobs:
           POD_NAME: ${{ steps.deploy_vald_readreplica.outputs.POD_NAME }}
   e2e-stream-crud-with-mirror:
     name: "E2E test (Stream CRUD) with mirror"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config

diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml
@@ -75,6 +75,9 @@ jobs:
     name: CRUD test on remote Helm chart
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    needs: [detect-ci-container]
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config

diff --git a/Makefile b/Makefile
@@ -139,7 +139,7 @@ PBGOS = $(PROTOS:apis/proto/%.proto=apis/grpc/%.pb.go)
 SWAGGERS = $(PROTOS:apis/proto/%.proto=apis/swagger/%.swagger.json)
 PBDOCS = apis/docs/v1/docs.md
 
-LDFLAGS = -static -fPIC -pthread -std=gnu++20 -lstdc++ -lm -z relro -z now -flto=auto -march=native -mtune=native -fno-plt -Ofast -fvisibility=hidden -ffp-contract=fast -fomit-frame-pointer -fmerge-all-constants -funroll-loops -falign-functions=32 -ffunction-sections -fdata-sections
+LDFLAGS = -static -fPIC -pthread -std=gnu++23 -lstdc++ -lm -z relro -z now -flto=auto -march=native -mtune=native -fno-plt -Ofast -fvisibility=hidden -ffp-contract=fast -fomit-frame-pointer -fmerge-all-constants -funroll-loops -falign-functions=32 -ffunction-sections -fdata-sections
 
 NGT_LDFLAGS = -fopenmp -lopenblas -llapack
 FAISS_LDFLAGS = $(NGT_LDFLAGS) -lgfortran

diff --git a/dockers/agent/core/agent/Dockerfile b/dockers/agent/core/agent/Dockerfile
@@ -95,4 +95,4 @@ ENV APP_NAME=agent
 
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/agent"]
+ENTRYPOINT ["/usr/bin/agent"]
diff --git a/dockers/agent/core/faiss/Dockerfile b/dockers/agent/core/faiss/Dockerfile
@@ -108,4 +108,4 @@ ENV APP_NAME=faiss
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/faiss"]
+ENTRYPOINT ["/usr/bin/faiss"]
diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile
@@ -107,4 +107,4 @@ ENV APP_NAME=ngt
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/ngt"]
+ENTRYPOINT ["/usr/bin/ngt"]
diff --git a/dockers/agent/sidecar/Dockerfile b/dockers/agent/sidecar/Dockerfile
@@ -97,4 +97,4 @@ ENV APP_NAME=sidecar
 
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/sidecar"]
+ENTRYPOINT ["/usr/bin/sidecar"]
diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile
@@ -19,6 +19,8 @@
 
 ARG UPX_OPTIONS=-9
 # skipcq: DOK-DL3026
+FROM docker:rc-dind AS docker
+# skipcq: DOK-DL3026
 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly
 ARG MAINTAINER="vdaas.org vald team <[email protected]>"
 LABEL maintainer="${MAINTAINER}"
@@ -39,8 +41,8 @@ ENV ORG=vdaas
 ENV PKG=ci/base
 ENV REPO=vald
 ENV RUST_HOME=/usr/loacl/lib/rust
-ENV RUSTUP_HOME=${RUST_HOME}/rustup
 ENV CARGO_HOME=${RUST_HOME}/cargo
+ENV RUSTUP_HOME=${RUST_HOME}/rustup
 ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${GOROOT}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -88,6 +90,15 @@ COPY Makefile .
 COPY .git .
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
 COPY . .
+COPY --from=docker /usr/local/bin/dind /usr/bin/dind
+COPY --from=docker /usr/local/bin/docker /usr/bin/docker
+COPY --from=docker /usr/local/bin/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
+COPY --from=docker /usr/local/bin/docker-init /usr/bin/docker-init
+COPY --from=docker /usr/local/bin/docker-proxy /usr/bin/docker-proxy
+COPY --from=docker /usr/local/bin/dockerd /usr/bin/dockerd
+COPY --from=docker /usr/local/bin/dockerd-entrypoint.sh /usr/bin/dockerd-entrypoint.sh
+COPY --from=docker /usr/local/bin/modprobe /usr/bin/modprobe
+COPY --from=docker /usr/local/bin/runc /usr/bin/runc
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
 #skipcq: DOK-W1001, DOK-SC2086
@@ -121,4 +132,4 @@ RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \
     && make telepresence/install \
     && make ngt/install \
     && make faiss/install \
-    && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
+    && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile
@@ -124,4 +124,4 @@ RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \
     && make staticcheck/install \
     && make ngt/install \
     && make faiss/install \
-    && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
+    && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
diff --git a/dockers/discoverer/k8s/Dockerfile b/dockers/discoverer/k8s/Dockerfile
@@ -97,4 +97,4 @@ ENV APP_NAME=discoverer
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/discoverer"]
+ENTRYPOINT ["/usr/bin/discoverer"]
diff --git a/dockers/gateway/filter/Dockerfile b/dockers/gateway/filter/Dockerfile
@@ -97,4 +97,4 @@ ENV APP_NAME=filter
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/filter"]
+ENTRYPOINT ["/usr/bin/filter"]
diff --git a/dockers/gateway/lb/Dockerfile b/dockers/gateway/lb/Dockerfile
@@ -97,4 +97,4 @@ ENV APP_NAME=lb
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/lb"]
+ENTRYPOINT ["/usr/bin/lb"]
diff --git a/dockers/gateway/mirror/Dockerfile b/dockers/gateway/mirror/Dockerfile
@@ -97,4 +97,4 @@ ENV APP_NAME=mirror
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/mirror"]
+ENTRYPOINT ["/usr/bin/mirror"]
diff --git a/dockers/index/job/correction/Dockerfile b/dockers/index/job/correction/Dockerfile
@@ -97,4 +97,4 @@ ENV APP_NAME=index-correction
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-correction"]
+ENTRYPOINT ["/usr/bin/index-correction"]
diff --git a/dockers/index/job/creation/Dockerfile b/dockers/index/job/creation/Dockerfile
@@ -97,4 +97,4 @@ ENV APP_NAME=index-creation
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-creation"]
+ENTRYPOINT ["/usr/bin/index-creation"]
diff --git a/dockers/index/job/readreplica/rotate/Dockerfile b/dockers/index/job/readreplica/rotate/Dockerfile
@@ -97,4 +97,4 @@ ENV APP_NAME=readreplica-rotate
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/readreplica-rotate"]
+ENTRYPOINT ["/usr/bin/readreplica-rotate"]
diff --git a/dockers/index/job/save/Dockerfile b/dockers/index/job/save/Dockerfile
@@ -97,4 +97,4 @@ ENV APP_NAME=index-save
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-save"]
+ENTRYPOINT ["/usr/bin/index-save"]
diff --git a/dockers/index/operator/Dockerfile b/dockers/index/operator/Dockerfile
@@ -97,4 +97,4 @@ ENV APP_NAME=index-operator
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-operator"]
+ENTRYPOINT ["/usr/bin/index-operator"]
diff --git a/dockers/manager/index/Dockerfile b/dockers/manager/index/Dockerfile
@@ -97,4 +97,4 @@ ENV APP_NAME=index
 COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
 COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index"]
+ENTRYPOINT ["/usr/bin/index"]