feat: Implement delete expired index job

.cspell.json

@@ -41,6 +41,7 @@
     "**/cmd/gateway/mirror/mirror",
     "**/cmd/index/job/correction/index-correction",
     "**/cmd/index/job/creation/index-creation",
+    "**/cmd/index/job/deletion/index-deletion",
     "**/cmd/index/job/readreplica/rotate/readreplica-rotate",
     "**/cmd/index/job/save/index-save",
     "**/cmd/index/operator/index-operator",

.gitfiles

@@ -96,6 +96,7 @@
 .github/workflows/dockers-image-scan.yaml
 .github/workflows/dockers-index-correction-image.yaml
 .github/workflows/dockers-index-creation-image.yaml
+.github/workflows/dockers-index-deletion-image.yaml
 .github/workflows/dockers-index-operator-image.yaml
 .github/workflows/dockers-index-save-image.yaml
 .github/workflows/dockers-loadtest-image.yaml

.github/actions/detect-docker-image-tags/action.yaml

@@ -48,6 +48,7 @@ runs:
           ["vdaas/vald-mirror-gateway"]="gateway.mirror.image.tag"
           ["vdaas/vald-manager-index"]="manager.index.image.tag"
           ["vdaas/vald-index-creation"]="manager.index.creator.image.tag"
+          ["vdaas/vald-index-deletion"]="manager.index.delete.image.tag"
           ["vdaas/vald-index-save"]="manager.index.saver.image.tag"
           ["vdaas/vald-readreplica-rotate"]="manager.index.readreplica.rotator.image.tag"
           ["vdaas/vald-helm-operator"]="image.tag"

.github/workflows/dockers-image-scan.yaml

@@ -70,6 +70,10 @@ jobs:
     uses: ./.github/workflows/_docker-image-scan.yaml
     with:
       target: index-creation
+  index-deletion:
+    uses: ./.github/workflows/_docker-image-scan.yaml
+    with:
+      target: index-deletion
   index-save:
     uses: ./.github/workflows/_docker-image-scan.yaml
     with:

.github/workflows/dockers-index-deletion-image.yaml

@@ -0,0 +1,80 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go
+
+name: 'Build docker image: index-deletion'
+on:
+  push:
+    branches:
+    - main
+    - release/v*.*
+    - '!release/v*.*.*'
+    tags:
+    - '*.*.*'
+    - v*.*.*
+    - '*.*.*-*'
+    - v*.*.*-*
+  pull_request:
+    paths:
+    - hack/docker/gen/main.go
+    - dockers/index/job/deletion/Dockerfile
+    - hack/actions/gen/main.go
+    - .github/workflows/dockers-index-deletion-image.yaml
+    - .github/actions/docker-build/action.yaml
+    - .github/workflows/_docker-image.yaml
+    - cmd/index/job/deletion/**
+    - pkg/index/job/deletion/**
+    - apis/grpc/**
+    - apis/proto/**
+    - go.mod
+    - go.sum
+    - versions/GO_VERSION
+    - internal/**
+    - '!internal/**/*_test.go'
+    - '!internal/**/*_mock.go'
+    - '!internal/db/**'
+    - '!internal/k8s/**'
+    - Makefile
+    - Makefile.d/**
+  pull_request_target:
+    paths:
+    - hack/docker/gen/main.go
+    - dockers/index/job/deletion/Dockerfile
+    - hack/actions/gen/main.go
+    - .github/workflows/dockers-index-deletion-image.yaml
+    - .github/actions/docker-build/action.yaml
+    - .github/workflows/_docker-image.yaml
+    - cmd/index/job/deletion/**
+    - pkg/index/job/deletion/**
+    - apis/grpc/**
+    - apis/proto/**
+    - go.mod
+    - go.sum
+    - versions/GO_VERSION
+    - internal/**
+    - '!internal/**/*_test.go'
+    - '!internal/**/*_mock.go'
+    - '!internal/db/**'
+    - '!internal/k8s/**'
+    - Makefile
+    - Makefile.d/**
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: index-deletion
+    secrets: inherit

.github/workflows/dockers-index-deletion.yaml

@@ -0,0 +1,80 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go
+
+name: "Build docker image: index-deletion"
+on:
+  push:
+    branches:
+      - main
+      - release/v*.*
+      - "!release/v*.*.*"
+    tags:
+      - "*.*.*"
+      - v*.*.*
+      - "*.*.*-*"
+      - v*.*.*-*
+  pull_request:
+    paths:
+      - hack/docker/gen/main.go
+      - dockers/index/job/deletion/Dockerfile
+      - hack/actions/gen/main.go
+      - .github/workflows/dockers-index-deletion-image.yaml
+      - .github/actions/docker-build/action.yaml
+      - .github/workflows/_docker-image.yaml
+      - cmd/index/job/deletion/**
+      - pkg/index/job/deletion/**
+      - apis/grpc/**
+      - apis/proto/**
+      - go.mod
+      - go.sum
+      - versions/GO_VERSION
+      - internal/**
+      - "!internal/**/*_test.go"
+      - "!internal/**/*_mock.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
+      - Makefile
+      - Makefile.d/**
+  pull_request_target:
+    paths:
+      - hack/docker/gen/main.go
+      - dockers/index/job/deletion/Dockerfile
+      - hack/actions/gen/main.go
+      - .github/workflows/dockers-index-deletion-image.yaml
+      - .github/actions/docker-build/action.yaml
+      - .github/workflows/_docker-image.yaml
+      - cmd/index/job/deletion/**
+      - pkg/index/job/deletion/**
+      - apis/grpc/**
+      - apis/proto/**
+      - go.mod
+      - go.sum
+      - versions/GO_VERSION
+      - internal/**
+      - "!internal/**/*_test.go"
+      - "!internal/**/*_mock.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
+      - Makefile
+      - Makefile.d/**
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: index-deletion
+    secrets: inherit

.github/workflows/dockers-release-branch-image.yaml

@@ -88,6 +88,12 @@ jobs:
     with:
       target: index-creation
     secrets: inherit
+  index-deletion:
+    needs: [dump-contexts-to-log]
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: index-deletion
+    secrets: inherit
   index-save:
     needs: [dump-contexts-to-log]
     uses: ./.github/workflows/_docker-image.yaml

Makefile

@@ -40,6 +40,7 @@ FILTER_GATEWAY_IMAGE            = $(NAME)-filter-gateway
 HELM_OPERATOR_IMAGE             = $(NAME)-helm-operator
 INDEX_CORRECTION_IMAGE          = $(NAME)-index-correction
 INDEX_CREATION_IMAGE            = $(NAME)-index-creation
+INDEX_DELETION_IMAGE            = $(NAME)-index-deletion
 INDEX_OPERATOR_IMAGE            = $(NAME)-index-operator
 INDEX_SAVE_IMAGE                = $(NAME)-index-save
 LB_GATEWAY_IMAGE                = $(NAME)-lb-gateway

Makefile.d/build.mk

@@ -24,6 +24,7 @@ binary/build: \
 	cmd/gateway/mirror/mirror \
 	cmd/index/job/correction/index-correction \
 	cmd/index/job/creation/index-creation \
+	cmd/index/job/deletion/index-deletion \
 	cmd/index/job/readreplica/rotate/readreplica-rotate \
 	cmd/index/job/save/index-save \
 	cmd/index/operator/index-operator \
@@ -79,6 +80,10 @@ cmd/index/job/creation/index-creation:
 	$(eval CGO_ENABLED = 0)
 	$(call go-build,index/job/creation,,-static,,,$@)
 
+cmd/index/job/deletion/index-deletion:
+	$(eval CGO_ENABLED = 0)
+	$(call go-build,index/job/deletion,,-static,,,$@)
+
 cmd/index/job/save/index-save:
 	$(eval CGO_ENABLED = 0)
 	$(call go-build,index/job/save,,-static,,,$@)
@@ -122,6 +127,7 @@ binary/build/zip: \
 	artifacts/vald-filter-gateway-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-index-correction-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-index-creation-$(GOOS)-$(GOARCH).zip \
+	artifacts/vald-index-deletion-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-index-operator-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-index-save-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-lb-gateway-$(GOOS)-$(GOARCH).zip \
@@ -181,6 +187,10 @@ artifacts/vald-index-creation-$(GOOS)-$(GOARCH).zip: cmd/index/job/creation/inde
 	$(call mkdir, $(dir $@))
 	zip --junk-paths $@ $<
 
+artifacts/vald-index-deletion-$(GOOS)-$(GOARCH).zip: cmd/index/job/deletion/index-deletion
+	$(call mkdir, $(dir $@))
+	zip --junk-paths $@ $<
+
 artifacts/vald-index-save-$(GOOS)-$(GOARCH).zip: cmd/index/job/save/index-save
 	$(call mkdir, $(dir $@))
 	zip --junk-paths $@ $<

Makefile.d/docker.mk

@@ -34,6 +34,7 @@ docker/build: \
 	docker/build/gateway-mirror \
 	docker/build/index-correction \
 	docker/build/index-creation \
+	docker/build/index-deletion \
 	docker/build/index-operator \
 	docker/build/index-save \
 	docker/build/loadtest \
@@ -61,6 +62,7 @@ docker/xpanes/build:
 		docker/build/gateway-mirror \
 		docker/build/index-correction \
 		docker/build/index-creation \
+		docker/build/index-deletion \
 		docker/build/index-operator \
 		docker/build/index-save \
 		docker/build/loadtest \
@@ -339,6 +341,17 @@ docker/build/index-save:
 		IMAGE=$(INDEX_SAVE_IMAGE) \
 		docker/build/image
 
+.PHONY: docker/name/index-deletion
+docker/name/index-deletion:
+	@echo "$(ORG)/$(INDEX_DELETION_IMAGE)"
+
+.PHONY: docker/build/index-deletion
+## build index-deletion image
+docker/build/index-deletion:
+	@make DOCKERFILE="$(ROOTDIR)/dockers/index/job/deletion/Dockerfile" \
+		IMAGE=$(INDEX_DELETION_IMAGE) \
+		docker/build/image
+
 .PHONY: docker/name/index-operator
 docker/name/index-operator:
 	@echo "$(ORG)/$(INDEX_OPERATOR_IMAGE)"

apis/grpc/v1/agent/core/agent.go

@@ -19,6 +19,7 @@ package core
 
 const (
 	CreateIndexRPCName        = "CreateIndex"
+	DeleteIndexRPCName        = "DeleteIndex"
 	SaveIndexRPCName          = "SaveIndex"
 	CreateAndSaveIndexRPCName = "CreateAndSaveIndex"
 )

apis/grpc/v1/agent/core/agent_vtproto.pb.go

@@ -44,6 +44,8 @@ const _ = grpc.SupportPackageIsVersion7
 type AgentClient interface {
 	// Represent the creating index RPC.
 	CreateIndex(ctx context.Context, in *payload.Control_CreateIndexRequest, opts ...grpc.CallOption) (*payload.Empty, error)
+	// Represent the deleting index RPC.
+	DeleteIndex(ctx context.Context, in *payload.Remove_Request, opts ...grpc.CallOption) (*payload.Empty, error)
 	// Represent the saving index RPC.
 	SaveIndex(ctx context.Context, in *payload.Empty, opts ...grpc.CallOption) (*payload.Empty, error)
 	// Represent the creating and saving index RPC.
@@ -69,6 +71,17 @@ func (c *agentClient) CreateIndex(
 	return out, nil
 }
 
+func (c *agentClient) DeleteIndex(
+	ctx context.Context, in *payload.Remove_Request, opts ...grpc.CallOption,
+) (*payload.Empty, error) {
+	out := new(payload.Empty)
+	err := c.cc.Invoke(ctx, "core.v1.Agent/DeleteIndex", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *agentClient) SaveIndex(
 	ctx context.Context, in *payload.Empty, opts ...grpc.CallOption,
 ) (*payload.Empty, error) {
@@ -97,6 +110,8 @@ func (c *agentClient) CreateAndSaveIndex(
 type AgentServer interface {
 	// Represent the creating index RPC.
 	CreateIndex(context.Context, *payload.Control_CreateIndexRequest) (*payload.Empty, error)
+	// Represent the deleting index RPC.
+	DeleteIndex(context.Context, *payload.Remove_Request) (*payload.Empty, error)
 	// Represent the saving index RPC.
 	SaveIndex(context.Context, *payload.Empty) (*payload.Empty, error)
 	// Represent the creating and saving index RPC.
@@ -113,6 +128,12 @@ func (UnimplementedAgentServer) CreateIndex(
 	return nil, status.Errorf(codes.Unimplemented, "method CreateIndex not implemented")
 }
 
+func (UnimplementedAgentServer) DeleteIndex(
+	context.Context, *payload.Remove_Request,
+) (*payload.Empty, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method DeleteIndex not implemented")
+}
+
 func (UnimplementedAgentServer) SaveIndex(context.Context, *payload.Empty) (*payload.Empty, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method SaveIndex not implemented")
 }
@@ -155,6 +176,26 @@ func _Agent_CreateIndex_Handler(
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Agent_DeleteIndex_Handler(
+	srv any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor,
+) (any, error) {
+	in := new(payload.Remove_Request)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(AgentServer).DeleteIndex(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/core.v1.Agent/CreateIndex",
+	}
+	handler := func(ctx context.Context, req any) (any, error) {
+		return srv.(AgentServer).DeleteIndex(ctx, req.(*payload.Remove_Request))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _Agent_SaveIndex_Handler(
 	srv any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor,
 ) (any, error) {
@@ -206,6 +247,10 @@ var Agent_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "CreateIndex",
 			Handler:    _Agent_CreateIndex_Handler,
 		},
+		{
+			MethodName: "DeleteIndex",
+			Handler:    _Agent_DeleteIndex_Handler,
+		},
 		{
 			MethodName: "SaveIndex",
 			Handler:    _Agent_SaveIndex_Handler,

apis/grpc/v1/vald/remove_vtproto.pb.go

@@ -40,7 +40,7 @@ const _ = grpc.SupportPackageIsVersion7
 
 // RemoveClient is the client API for Remove service.
 //
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+// Foaaar semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
 type RemoveClient interface {
 	// A method to remove an indexed vector.
 	Remove(ctx context.Context, in *payload.Remove_Request, opts ...grpc.CallOption) (*payload.Object_Location, error)