[BUGFIX] index correction process

.gitfiles

@@ -50,6 +50,7 @@
 .github/helm/values/vald-mirror-target.yaml
 .github/helm/values/values-agent-sidecar.yaml
 .github/helm/values/values-chaos.yaml
+.github/helm/values/values-correction.yaml
 .github/helm/values/values-index-management-jobs.yaml
 .github/helm/values/values-lb.yaml
 .github/helm/values/values-max-dim.yaml
@@ -81,7 +82,9 @@
 .github/workflows/dockers-agent-sidecar-image.yml
 .github/workflows/dockers-benchmark-job-image.yml
 .github/workflows/dockers-benchmark-operator-image.yaml
+.github/workflows/dockers-binfmt-image.yaml
 .github/workflows/dockers-buildbase-image.yml
+.github/workflows/dockers-buildkit-image.yaml
 .github/workflows/dockers-ci-container-image.yml
 .github/workflows/dockers-dev-container-image.yml
 .github/workflows/dockers-discoverer-k8s-image.yml
@@ -525,7 +528,9 @@ dockers/agent/core/ngt/Dockerfile
 dockers/agent/core/ngt/README.md
 dockers/agent/sidecar/Dockerfile
 dockers/agent/sidecar/README.md
+dockers/binfmt/Dockerfile
 dockers/buildbase/Dockerfile
+dockers/buildkit/Dockerfile
 dockers/ci/base/Dockerfile
 dockers/ci/base/README.md
 dockers/dev/Dockerfile

.github/helm/values/values-correction.yaml

@@ -0,0 +1,73 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+defaults:
+  logging:
+    level: info
+  networkPolicy:
+    enabled: true
+gateway:
+  lb:
+    enabled: true
+    minReplicas: 1
+    hpa:
+      enabled: false
+    resources:
+      requests:
+        cpu: 100m
+        memory: 50Mi
+    gateway_config:
+      index_replica: 2
+agent:
+  minReplicas: 10
+  maxReplicas: 10
+  podManagementPolicy: Parallel
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  ngt:
+    auto_index_duration_limit: 2m
+    auto_index_check_duration: 30s
+    auto_index_length: 500
+    dimension: 784
+discoverer:
+  minReplicas: 1
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+manager:
+  index:
+    replicas: 1
+    resources:
+      requests:
+        cpu: 100m
+        memory: 30Mi
+    indexer:
+      auto_index_duration_limit: 2m
+      auto_index_check_duration: 30s
+      auto_index_length: 1000
+    corrector:
+      enabled: true
+      # suspend because you do not want corrector to start automatically in CI
+      # instead run it manually
+      suspend: true
+      schedule: "1 2 3 4 5"

.github/workflows/dockers-binfmt-image.yaml

@@ -0,0 +1,48 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Build docker image: binfmt"
+on:
+  schedule:
+    - cron: "0 * * * *"
+  push:
+    branches:
+      - "main"
+      - "release/v*.*"
+      - "!release/v*.*.*"
+    tags:
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
+  pull_request:
+    paths:
+      - ".github/actions/docker-build/action.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-binfmt-image.yml"
+      - "dockers/binfmt/Dockerfile"
+  pull_request_target:
+    paths:
+      - ".github/actions/docker-build/action.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-binfmt-image.yml"
+      - "dockers/binfmt/Dockerfile"
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: binfmt
+      platforms: linux/amd64,linux/arm64
+    secrets: inherit

.github/workflows/dockers-buildkit-image.yaml

@@ -0,0 +1,48 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Build docker image: buildkit"
+on:
+  schedule:
+    - cron: "0 * * * *"
+  push:
+    branches:
+      - "main"
+      - "release/v*.*"
+      - "!release/v*.*.*"
+    tags:
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
+  pull_request:
+    paths:
+      - ".github/actions/docker-build/action.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-buildkit-image.yml"
+      - "dockers/buildkit/Dockerfile"
+  pull_request_target:
+    paths:
+      - ".github/actions/docker-build/action.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-buildkit-image.yml"
+      - "dockers/buildkit/Dockerfile"
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: buildkit
+      platforms: linux/amd64,linux/arm64
+    secrets: inherit

Makefile

@@ -17,32 +17,35 @@
 SHELL                           = bash
 ORG                             ?= vdaas
 NAME                            = vald
-GOPKG                           = github.com/$(ORG)/$(NAME)
+REPO                            = $(ORG)/$(NAME)
+GOPKG                           = github.com/$(REPO)
 DATETIME                        = $(eval DATETIME := $(shell date -u +%Y/%m/%d_%H:%M:%S%z))$(DATETIME)
 TAG                            ?= latest
 CRORG                          ?= $(ORG)
-GHCRORG                         = ghcr.io/$(ORG)/$(NAME)
+GHCRORG                         = ghcr.io/$(REPO)
+AGENT_IMAGE                     = $(NAME)-agent
 AGENT_NGT_IMAGE                 = $(NAME)-agent-ngt
 AGENT_FAISS_IMAGE               = $(NAME)-agent-faiss
 AGENT_SIDECAR_IMAGE             = $(NAME)-agent-sidecar
-AGENT_IMAGE                     = $(NAME)-agent
+BENCHMARK_JOB_IMAGE             = $(NAME)-benchmark-job
+BENCHMARK_OPERATOR_IMAGE        = $(NAME)-benchmark-operator
+BINFMT_IMAGE                    = $(NAME)-binfmt
 BUILDBASE_IMAGE                 = $(NAME)-buildbase
+BUILDKIT_IMAGE                  = $(NAME)-buildkit
 CI_CONTAINER_IMAGE              = $(NAME)-ci-container
 DEV_CONTAINER_IMAGE             = $(NAME)-dev-container
 DISCOVERER_IMAGE                = $(NAME)-discoverer-k8s
 FILTER_GATEWAY_IMAGE            = $(NAME)-filter-gateway
-MIRROR_GATEWAY_IMAGE            = $(NAME)-mirror-gateway
 HELM_OPERATOR_IMAGE             = $(NAME)-helm-operator
-LB_GATEWAY_IMAGE                = $(NAME)-lb-gateway
-LOADTEST_IMAGE                  = $(NAME)-loadtest
 INDEX_CORRECTION_IMAGE          = $(NAME)-index-correction
 INDEX_CREATION_IMAGE            = $(NAME)-index-creation
-INDEX_SAVE_IMAGE                = $(NAME)-index-save
 INDEX_OPERATOR_IMAGE            = $(NAME)-index-operator
-READREPLICA_ROTATE_IMAGE        = $(NAME)-readreplica-rotate
+INDEX_SAVE_IMAGE                = $(NAME)-index-save
+LB_GATEWAY_IMAGE                = $(NAME)-lb-gateway
+LOADTEST_IMAGE                  = $(NAME)-loadtest
 MANAGER_INDEX_IMAGE             = $(NAME)-manager-index
-BENCHMARK_JOB_IMAGE             = $(NAME)-benchmark-job
-BENCHMARK_OPERATOR_IMAGE        = $(NAME)-benchmark-operator
+MIRROR_GATEWAY_IMAGE            = $(NAME)-mirror-gateway
+READREPLICA_ROTATE_IMAGE        = $(NAME)-readreplica-rotate
 MAINTAINER                      = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>"
 
 VERSION ?= $(eval VERSION := $(shell cat versions/VALD_VERSION))$(VERSION)
@@ -351,6 +354,13 @@ E2E_UPDATE_COUNT                   ?= 10
 E2E_UPSERT_COUNT                   ?= 10
 E2E_WAIT_FOR_CREATE_INDEX_DURATION ?= 8m
 E2E_WAIT_FOR_START_TIMEOUT         ?= 10m
+E2E_SEARCH_FROM                    ?= 0
+E2E_SEARCH_BY_ID_FROM              ?= 0
+E2E_INSERT_FROM                    ?= 0
+E2E_UPDATE_FROM                    ?= 0
+E2E_UPSERT_FROM                    ?= 0
+E2E_GET_OBJECT_FROM                ?= 0
+E2E_REMOVE_FROM                    ?= 0
 
 TEST_RESULT_DIR ?= /tmp
 

Makefile.d/dependencies.mk

@@ -225,12 +225,12 @@ update/hdf5:
 .PHONY: update/vald
 ## update vald it's self version
 update/vald:
-	curl -fsSL https://api.github.com/repos/vdaas/vald/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALD_VERSION
+	curl -fsSL https://api.github.com/repos/$(REPO)/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALD_VERSION
 
 .PHONY: update/valdcli
 ## update vald client library made by clojure self version
 update/valdcli:
-	curl -fsSL https://api.github.com/repos/vdaas/vald-client-clj/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALDCLI_VERSION
+	curl -fsSL https://api.github.com/repos/$(REPO)-client-clj/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALDCLI_VERSION
 
 .PHONY: update/template
 ## update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE

Makefile.d/docker.mk

@@ -16,16 +16,29 @@
 .PHONY: docker/build
 ## build all docker images
 docker/build: \
-	docker/build/agent-ngt \
+	docker/build/agent \
 	docker/build/agent-faiss \
+	docker/build/agent-ngt \
 	docker/build/agent-sidecar \
+	docker/build/benchmark-job \
+	docker/build/benchmark-operator \
+	docker/build/binfmt \
+	docker/build/buildbase \
+	docker/build/buildkit \
+	docker/build/ci-container \
+	docker/build/dev-container \
 	docker/build/discoverer-k8s \
-	docker/build/gateway-lb \
 	docker/build/gateway-filter \
+	docker/build/gateway-lb \
+	docker/build/gateway-mirror \
+	docker/build/index-correction \
+	docker/build/index-creation \
+	docker/build/index-operator \
+	docker/build/index-save \
+	docker/build/loadtest \
 	docker/build/manager-index \
-	docker/build/benchmark-job \
-	docker/build/benchmark-operator \
-	docker/build/operator/helm
+	docker/build/operator/helm \
+	docker/build/readreplica-rotate
 
 .PHONY: docker/name/org
 docker/name/org:
@@ -184,6 +197,28 @@ docker/build/buildbase:
 		IMAGE=$(BUILDBASE_IMAGE) \
 		docker/build/image
 
+.PHONY: docker/name/buildkit
+docker/name/buildkit:
+	@echo "$(ORG)/$(BUILDKIT_IMAGE)"
+
+.PHONY: docker/build/buildkit
+## build buildkit image
+docker/build/buildkit:
+	@make DOCKERFILE="$(ROOTDIR)/dockers/buildkit/Dockerfile" \
+		IMAGE=$(BUILDKIT_IMAGE) \
+		docker/build/image
+
+.PHONY: docker/name/binfmt
+docker/name/binfmt:
+	@echo "$(ORG)/$(BINFMT_IMAGE)"
+
+.PHONY: docker/build/binfmt
+## build binfmt image
+docker/build/binfmt:
+	@make DOCKERFILE="$(ROOTDIR)/dockers/binfmt/Dockerfile" \
+		IMAGE=$(BINFMT_IMAGE) \
+		docker/build/image
+
 .PHONY: docker/name/ci-container
 docker/name/ci-container:
 	@echo "$(ORG)/$(CI_CONTAINER_IMAGE)"

Makefile.d/e2e.mk

@@ -130,7 +130,7 @@ e2e/actions/run/job: \
 	kubectl wait -n kube-system --for=condition=Ready pod -l k8s-app=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT)
 	kubectl wait -n kube-system --for=condition=ContainersReady pod -l k8s-app=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT)
 	make k8s/vald/deploy \
-		HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-lb.yaml
+		HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-correction.yaml
 	sleep 3
 	kubectl wait --for=condition=Ready pod -l "app=$(LB_GATEWAY_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT)
 	kubectl wait --for=condition=ContainersReady pod -l "app=$(LB_GATEWAY_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT)

Makefile.d/functions.mk

@@ -118,12 +118,21 @@ define run-e2e-crud-test
 	    -port=$(E2E_BIND_PORT) \
 	    -dataset=$(ROOTDIR)/hack/benchmark/assets/dataset/$(E2E_DATASET_NAME) \
 	    -insert-num=$(E2E_INSERT_COUNT) \
+	    -correction-insert-num=$(E2E_INSERT_COUNT) \
+	    -insert-num=$(E2E_INSERT_COUNT) \
 	    -search-num=$(E2E_SEARCH_COUNT) \
 	    -search-by-id-num=$(E2E_SEARCH_BY_ID_COUNT) \
 	    -get-object-num=$(E2E_GET_OBJECT_COUNT) \
 	    -update-num=$(E2E_UPDATE_COUNT) \
 	    -upsert-num=$(E2E_UPSERT_COUNT) \
 	    -remove-num=$(E2E_REMOVE_COUNT) \
+	    -insert-from=$(E2E_INSERT_FROM) \
+	    -update-from=$(E2E_UPDATE_FROM) \
+	    -upsert-from=$(E2E_UPSERT_FROM) \
+	    -remove-from=$(E2E_REMOVE_FROM) \
+	    -search-from=$(E2E_SEARCH_FROM) \
+	    -search-by-id-from=$(E2E_SEARCH_BY_ID_FROM) \
+	    -get-object-from=$(E2E_GET_OBJECT_FROM) \
 	    -wait-after-insert=$(E2E_WAIT_FOR_CREATE_INDEX_DURATION) \
 	    -portforward=$(E2E_PORTFORWARD_ENABLED) \
 	    -portforward-pod-name=$(E2E_TARGET_POD_NAME) \