From 8b257c6f2655fb8d928e5cd2ae2868d584f61ad6 Mon Sep 17 00:00:00 2001 From: kpango Date: Thu, 3 Oct 2024 03:41:00 +0900 Subject: [PATCH] Refactor use Absolute path for Makefile Signed-off-by: kpango --- Makefile | 12 +-- Makefile.d/docker.mk | 4 +- Makefile.d/functions.mk | 1 - Makefile.d/git.mk | 2 +- Makefile.d/helm.mk | 78 ++++++++-------- Makefile.d/k8s.mk | 90 +++++++++---------- Makefile.d/proto.mk | 6 +- example/client/mirror/main.go | 2 +- hack/actions/gen/main.go | 2 +- hack/cspell/main.go | 5 +- hack/docker/gen/main.go | 5 +- hack/tools/deadlink/main.go | 2 +- internal/core/algorithm/faiss/faiss.go | 2 +- internal/core/algorithm/faiss/option.go | 3 +- internal/core/algorithm/usearch/option.go | 2 +- internal/core/algorithm/usearch/usearch.go | 2 +- internal/info/info.go | 62 ++++++++++--- internal/k8s/job/job.go | 4 +- internal/net/grpc/errdetails/errdetails.go | 7 +- internal/observability/metrics/mem/mem.go | 2 +- pkg/gateway/lb/handler/grpc/handler.go | 2 +- pkg/gateway/mirror/handler/grpc/handler.go | 2 +- pkg/gateway/mirror/service/gateway.go | 2 +- pkg/index/job/correction/service/corrector.go | 2 +- .../job/readreplica/rotate/service/rotator.go | 2 +- tests/performance/max_vector_dim_test.go | 14 ++- 26 files changed, 170 insertions(+), 147 deletions(-) diff --git a/Makefile b/Makefile index 1eb5851603..6480b303a3 100644 --- a/Makefile +++ b/Makefile @@ -139,18 +139,18 @@ GIT_COMMIT := $(eval GIT_COMMIT := $(shell git rev-list -1 HEAD))$(GIT_COMMIT) MAKELISTS := Makefile $(shell find Makefile.d -type f -regex ".*\.mk") ROOTDIR = $(eval ROOTDIR := $(or $(shell git rev-parse --show-toplevel), $(PWD)))$(ROOTDIR) -PROTODIRS := $(eval PROTODIRS := $(shell find apis/proto -type d | sed -e "s%apis/proto/%%g" | grep -v "apis/proto"))$(PROTODIRS) +PROTODIRS := $(eval PROTODIRS := $(shell find $(ROOTDIR)/apis/proto -type d | sed -e "s%apis/proto/%%g" | grep -v "apis/proto"))$(PROTODIRS) BENCH_DATASET_BASE_DIR = hack/benchmark/assets BENCH_DATASET_MD5_DIR_NAME = checksum BENCH_DATASET_HDF5_DIR_NAME = dataset BENCH_DATASET_MD5_DIR = $(BENCH_DATASET_BASE_DIR)/$(BENCH_DATASET_MD5_DIR_NAME) BENCH_DATASET_HDF5_DIR = $(BENCH_DATASET_BASE_DIR)/$(BENCH_DATASET_HDF5_DIR_NAME) -PROTOS := $(eval PROTOS := $(shell find apis/proto -type f -regex ".*\.proto"))$(PROTOS) +PROTOS := $(eval PROTOS := $(shell find $(ROOTDIR)/apis/proto -type f -regex ".*\.proto"))$(PROTOS) PROTOS_V1 := $(eval PROTOS_V1 := $(filter apis/proto/v1/%.proto,$(PROTOS)))$(PROTOS_V1) PBGOS = $(PROTOS:apis/proto/%.proto=apis/grpc/%.pb.go) SWAGGERS = $(PROTOS:apis/proto/%.proto=apis/swagger/%.swagger.json) -PBDOCS = apis/docs/v1/docs.md +PBDOCS = $(ROOTDIR)/apis/docs/v1/docs.md LDFLAGS = -static -fPIC -pthread -std=gnu++23 -lstdc++ -lm -z relro -z now -flto=auto -march=native -mtune=native -fno-plt -Ofast -fvisibility=hidden -ffp-contract=fast -fomit-frame-pointer -fmerge-all-constants -funroll-loops -falign-functions=32 -ffunction-sections -fdata-sections @@ -795,13 +795,13 @@ changelog/update: echo "" >> $(TEMP_DIR)/CHANGELOG.md $(MAKE) -s changelog/next/print >> $(TEMP_DIR)/CHANGELOG.md echo "" >> $(TEMP_DIR)/CHANGELOG.md - tail -n +2 CHANGELOG.md >> $(TEMP_DIR)/CHANGELOG.md - mv -f $(TEMP_DIR)/CHANGELOG.md CHANGELOG.md + tail -n +2 $(ROOTDIR)/CHANGELOG.md >> $(TEMP_DIR)/CHANGELOG.md + mv -f $(TEMP_DIR)/CHANGELOG.md $(ROOTDIR)/CHANGELOG.md .PHONY: changelog/next/print ## print next changelog entry changelog/next/print: - @cat hack/CHANGELOG.template.md | \ + @cat $(ROOTDIR)/hack/CHANGELOG.template.md | \ sed -e 's/{{ version }}/$(VERSION)/g' @echo "$$BODY" diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index b653757ffa..d9ef1ac5f4 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -101,7 +101,7 @@ ifeq ($(REMOTE),true) -t $(GHCRORG)/$(IMAGE):$(TAG) \ $(EXTRA_ARGS) \ --output type=registry,oci-mediatypes=true,compression=zstd,compression-level=5,force-compression=true,push=true \ - -f $(DOCKERFILE) . + -f $(DOCKERFILE) $(ROOTDIR) else @echo "starting local build for $(IMAGE):$(TAG)" DOCKER_BUILDKIT=1 $(DOCKER) build \ @@ -113,7 +113,7 @@ else $(EXTRA_ARGS) \ -t $(CRORG)/$(IMAGE):$(TAG) \ -t $(GHCRORG)/$(IMAGE):$(TAG) \ - -f $(DOCKERFILE) . + -f $(DOCKERFILE) $(ROOTDIR) endif .PHONY: docker/name/agent-ngt diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk index 9710a92f4a..7faaa10961 100644 --- a/Makefile.d/functions.mk +++ b/Makefile.d/functions.mk @@ -434,4 +434,3 @@ define gen-deadlink-checker $$BIN_PATH -path $3 -ignore-path $4 -format $5 $1; \ rm -rf $$BIN_PATH endef - diff --git a/Makefile.d/git.mk b/Makefile.d/git.mk index ac858a1842..766f0362fe 100644 --- a/Makefile.d/git.mk +++ b/Makefile.d/git.mk @@ -16,7 +16,7 @@ .PHONY: git/config/init ## add git configs required for development git/config/init: - git config commit.template ".commit_template" + git config commit.template "$(ROOTDIR)/.commit_template" git config core.fileMode false .PHONY: git/hooks/init diff --git a/Makefile.d/helm.mk b/Makefile.d/helm.mk index 06d49345c9..c468abe6c7 100644 --- a/Makefile.d/helm.mk +++ b/Makefile.d/helm.mk @@ -37,14 +37,14 @@ $(BINDIR)/helm-docs: .PHONY: helm/package/vald ## packaging Helm chart for Vald helm/package/vald: - helm package charts/vald + helm package $(ROOTDIR)/charts/vald .PHONY: helm/package/vald-helm-operator ## packaging Helm chart for vald-helm-operator helm/package/vald-helm-operator: \ helm/schema/crd/vald \ helm/schema/crd/vald-helm-operator - helm package charts/vald-helm-operator + helm package $(ROOTDIR)/charts/vald-helm-operator .PHONY: helm/package/vald-benchmark-operator ## packaging Helm chart for vald-helm-operator @@ -52,11 +52,11 @@ helm/package/vald-benchmark-operator: \ helm/schema/crd/vald-benchmark-job \ helm/schema/crd/vald-benchmark-scenario \ helm/schema/crd/vald-benchmark-operator - helm package charts/vald-benchmark-operator + helm package $(ROOTDIR)/charts/vald-benchmark-operator .PHONY: helm/package/vald-readreplica helm/package/vald-readreplica: - helm package charts/vald-readreplica + helm package $(ROOTDIR)/charts/vald-readreplica .PHONY: helm/repo/add ## add Helm chart repository @@ -64,42 +64,42 @@ helm/repo/add: helm repo add vald https://vald.vdaas.org/charts .PHONY: helm/docs/vald -helm/docs/vald: charts/vald/README.md +helm/docs/vald: $(ROOTDIR)/charts/vald/README.md # force to rebuild -.PHONY: charts/vald/README.md -charts/vald/README.md: \ - charts/vald/README.md.gotmpl \ - charts/vald/values.yaml +.PHONY: $(ROOTDIR)/charts/vald/README.md +$(ROOTDIR)/charts/vald/README.md: \ + $(ROOTDIR)/charts/vald/README.md.gotmpl \ + $(ROOTDIR)/charts/vald/values.yaml helm-docs .PHONY: helm/docs/vald-helm-operator -helm/docs/vald-helm-operator: charts/vald-helm-operator/README.md +helm/docs/vald-helm-operator: $(ROOTDIR)/charts/vald-helm-operator/README.md # force to rebuild -.PHONY: charts/vald-helm-operator/README.md -charts/vald-helm-operator/README.md: \ - charts/vald-helm-operator/README.md.gotmpl \ - charts/vald-helm-operator/values.yaml +.PHONY: $(ROOTDIR)/charts/vald-helm-operator/README.md +$(ROOTDIR)/charts/vald-helm-operator/README.md: \ + $(ROOTDIR)/charts/vald-helm-operator/README.md.gotmpl \ + $(ROOTDIR)/charts/vald-helm-operator/values.yaml helm-docs .PHONY: helm/docs/vald-readreplica -helm/docs/vald-readreplica: charts/vald-readreplica/README.md +helm/docs/vald-readreplica: $(ROOTDIR)/charts/vald-readreplica/README.md .PHONY: helm/docs/vald-benchmark-operator -helm/docs/vald-benchmark-operator: charts/vald-benchmark-operator/README.md +helm/docs/vald-benchmark-operator: $(ROOTDIR)/charts/vald-benchmark-operator/README.md -.PHONY: charts/vald-benchmark-operator/README.md -charts/vald-benchmark-operator/README.md: \ - charts/vald-benchmark-operator/README.md.gotmpl \ - charts/vald-benchmark-operator/values.yaml +.PHONY: $(ROOTDIR)/charts/vald-benchmark-operator/README.md +$(ROOTDIR)/charts/vald-benchmark-operator/README.md: \ + $(ROOTDIR)/charts/vald-benchmark-operator/README.md.gotmpl \ + $(ROOTDIR)/charts/vald-benchmark-operator/values.yaml helm-docs # force to rebuild -.PHONY: charts/vald-readreplica/README.md -charts/vald-readreplica/README.md: \ - charts/vald-readreplica/README.md.gotmpl \ - charts/vald-readreplica/values.yaml +.PHONY: $(ROOTDIR)/charts/vald-readreplica/README.md +$(ROOTDIR)/charts/vald-readreplica/README.md: \ + $(ROOTDIR)/charts/vald-readreplica/README.md.gotmpl \ + $(ROOTDIR)/charts/vald-readreplica/values.yaml helm-docs .PHONY: helm/schema/all @@ -112,42 +112,42 @@ helm/schema/all: \ .PHONY: helm/schema/vald ## generate json schema for Vald Helm Chart -helm/schema/vald: charts/vald/values.schema.json +helm/schema/vald: $(ROOTDIR)/charts/vald/values.schema.json -charts/vald/values.schema.json: \ - charts/vald/values.yaml +$(ROOTDIR)/charts/vald/values.schema.json: \ + $(ROOTDIR)/charts/vald/values.yaml $(call gen-vald-helm-schema,vald/values) .PHONY: helm/schema/vald-helm-operator ## generate json schema for Vald Helm Operator Chart -helm/schema/vald-helm-operator: charts/vald-helm-operator/values.schema.json +helm/schema/vald-helm-operator: $(ROOTDIR)/charts/vald-helm-operator/values.schema.json -charts/vald-helm-operator/values.schema.json: \ - charts/vald-helm-operator/values.yaml +$(ROOTDIR)/charts/vald-helm-operator/values.schema.json: \ + $(ROOTDIR)/charts/vald-helm-operator/values.yaml $(call gen-vald-helm-schema,vald-helm-operator/values) .PHONY: helm/schema/vald-benchmark-job ## generate json schema for Vald Benchmark Job Chart -helm/schema/vald-benchmark-job: charts/vald-benchmark-operator/job-values.schema.json +helm/schema/vald-benchmark-job: $(ROOTDIR)/charts/vald-benchmark-operator/job-values.schema.json -charts/vald-benchmark-operator/job-values.schema.json: \ - charts/vald-benchmark-operator/schemas/job-values.yaml +$(ROOTDIR)/charts/vald-benchmark-operator/job-values.schema.json: \ + $(ROOTDIR)/charts/vald-benchmark-operator/schemas/job-values.yaml $(call gen-vald-helm-schema,vald-benchmark-operator/schemas/job-values) .PHONY: helm/schema/vald-benchmark-scenario ## generate json schema for Vald Benchmark Job Chart -helm/schema/vald-benchmark-scenario: charts/vald-benchmark-operator/scenario-values.schema.json +helm/schema/vald-benchmark-scenario: $(ROOTDIR)/charts/vald-benchmark-operator/scenario-values.schema.json -charts/vald-benchmark-operator/scenario-values.schema.json: \ - charts/vald-benchmark-operator/schemas/scenario-values.yaml +$(ROOTDIR)/charts/vald-benchmark-operator/scenario-values.schema.json: \ + $(ROOTDIR)/charts/vald-benchmark-operator/schemas/scenario-values.yaml $(call gen-vald-helm-schema,vald-benchmark-operator/schemas/scenario-values) .PHONY: helm/schema/vald-benchmark-operator ## generate json schema for Vald Benchmark Operator Chart -helm/schema/vald-benchmark-operator: charts/vald-benchmark-operator/values.schema.json +helm/schema/vald-benchmark-operator: $(ROOTDIR)/charts/vald-benchmark-operator/values.schema.json -charts/vald-benchmark-operator/values.schema.json: \ - charts/vald-benchmark-operator/values.yaml +$(ROOTDIR)/charts/vald-benchmark-operator/values.schema.json: \ + $(ROOTDIR)/charts/vald-benchmark-operator/values.yaml $(call gen-vald-helm-schema,vald-benchmark-operator/values) .PHONY: yq/install diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk index 5da9210129..8693f9d2fe 100644 --- a/Makefile.d/k8s.mk +++ b/Makefile.d/k8s.mk @@ -44,19 +44,19 @@ k8s/manifest/update: \ --set gateway.mirror.enabled=true \ --output-dir $(TEMP_DIR) \ charts/vald - mkdir -p k8s/gateway - mkdir -p k8s/manager - mkdir -p k8s/index/job - mkdir -p k8s/index/job/readreplica - mv $(TEMP_DIR)/vald/templates/agent k8s/agent - mv $(TEMP_DIR)/vald/templates/discoverer k8s/discoverer - mv $(TEMP_DIR)/vald/templates/gateway k8s/gateway - mv $(TEMP_DIR)/vald/templates/manager/index k8s/manager/index - mv $(TEMP_DIR)/vald/templates/index/operator k8s/index/operator - mv $(TEMP_DIR)/vald/templates/index/job/correction k8s/index/job/correction - mv $(TEMP_DIR)/vald/templates/index/job/creation k8s/index/job/creation - mv $(TEMP_DIR)/vald/templates/index/job/save k8s/index/job/save - mv $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate k8s/index/job/readreplica/rotate + mkdir -p $(ROOTDIR)/k8s/gateway + mkdir -p $(ROOTDIR)/k8s/manager + mkdir -p $(ROOTDIR)/k8s/index/job + mkdir -p $(ROOTDIR)/k8s/index/job/readreplica + mv $(TEMP_DIR)/vald/templates/agent $(ROOTDIR)/k8s/agent + mv $(TEMP_DIR)/vald/templates/discoverer $(ROOTDIR)/k8s/discoverer + mv $(TEMP_DIR)/vald/templates/gateway $(ROOTDIR)/k8s/gateway + mv $(TEMP_DIR)/vald/templates/manager/index $(ROOTDIR)/k8s/manager/index + mv $(TEMP_DIR)/vald/templates/index/operator $(ROOTDIR)/k8s/index/operator + mv $(TEMP_DIR)/vald/templates/index/job/correction $(ROOTDIR)/k8s/index/job/correction + mv $(TEMP_DIR)/vald/templates/index/job/creation $(ROOTDIR)/k8s/index/job/creation + mv $(TEMP_DIR)/vald/templates/index/job/save $(ROOTDIR)/k8s/index/job/save + mv $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate $(ROOTDIR)/k8s/index/job/readreplica/rotate rm -rf $(TEMP_DIR) .PHONY: k8s/manifest/helm-operator/clean @@ -72,10 +72,10 @@ k8s/manifest/helm-operator/update: \ helm template \ --output-dir $(TEMP_DIR) \ charts/vald-helm-operator - mkdir -p k8s/operator - mv $(TEMP_DIR)/vald-helm-operator/templates k8s/operator/helm + mkdir -p $(ROOTDIR)/k8s/operator + mv $(TEMP_DIR)/vald-helm-operator/templates $(ROOTDIR)/k8s/operator/helm rm -rf $(TEMP_DIR) - cp -r charts/vald-helm-operator/crds k8s/operator/helm/crds + cp -r $(ROOTDIR)/charts/vald-helm-operator/crds $(ROOTDIR)/k8s/operator/helm/crds .PHONY: k8s/manifest/benchmark-operator/clean ## clean k8s manifests for benchmark-operator @@ -90,10 +90,10 @@ k8s/manifest/benchmark-operator/update: \ helm template \ --output-dir $(TEMP_DIR) \ charts/vald-benchmark-operator - mkdir -p k8s/tools/benchmark - mv $(TEMP_DIR)/vald-benchmark-operator/templates k8s/tools/benchmark/operator + mkdir -p $(ROOTDIR)/k8s/tools/benchmark + mv $(TEMP_DIR)/vald-benchmark-operator/templates $(ROOTDIR)/k8s/tools/benchmark/operator rm -rf $(TEMP_DIR) - cp -r charts/vald-benchmark-operator/crds k8s/tools/benchmark/operator/crds + cp -r $(ROOTDIR)/charts/vald-benchmark-operator/crds $(ROOTDIR)/k8s/tools/benchmark/operator/crds .PHONY: k8s/manifest/readreplica/clean ## clean k8s manifests for readreplica @@ -108,7 +108,7 @@ k8s/manifest/readreplica/update: \ helm template \ --output-dir $(TEMP_DIR) \ charts/vald-readreplica - mv $(TEMP_DIR)/vald-readreplica/templates k8s/readreplica + mv $(TEMP_DIR)/vald-readreplica/templates $(ROOTDIR)/k8s/readreplica rm -rf $(TEMP_DIR) .PHONY: k8s/vald/deploy @@ -190,15 +190,15 @@ k8s/multi/vald/deploy: -@kubectl create ns $(MIRROR01_NAMESPACE) -@kubectl create ns $(MIRROR02_NAMESPACE) -@kubectl create ns $(MIRROR03_NAMESPACE) - helm install vald-cluster-01 charts/vald \ + helm install vald-cluster-01 $(ROOTDIR)/charts/vald \ -f $(ROOTDIR)/charts/vald/values/multi-vald/dev-vald-with-mirror.yaml \ -f $(ROOTDIR)/charts/vald/values/multi-vald/dev-vald-01.yaml \ -n $(MIRROR01_NAMESPACE) - helm install vald-cluster-02 charts/vald \ + helm install vald-cluster-02 $(ROOTDIR)/charts/vald \ -f $(ROOTDIR)/charts/vald/values/multi-vald/dev-vald-with-mirror.yaml \ -f $(ROOTDIR)/charts/vald/values/multi-vald/dev-vald-02.yaml \ -n $(MIRROR02_NAMESPACE) - helm install vald-cluster-03 charts/vald \ + helm install vald-cluster-03 $(ROOTDIR)/charts/vald \ -f $(ROOTDIR)/charts/vald/values/multi-vald/dev-vald-with-mirror.yaml \ -f $(ROOTDIR)/charts/vald/values/multi-vald/dev-vald-03.yaml \ -n $(MIRROR03_NAMESPACE) @@ -346,18 +346,18 @@ k8s/external/cert-manager/delete: .PHONY: k8s/external/minio/deploy ## deploy minio k8s/external/minio/deploy: - kubectl apply -f k8s/external/minio/deployment.yaml - kubectl apply -f k8s/external/minio/svc.yaml + kubectl apply -f $(ROOTDIR)/k8s/external/minio/deployment.yaml + kubectl apply -f $(ROOTDIR)/k8s/external/minio/svc.yaml sleep $(K8S_SLEEP_DURATION_FOR_WAIT_COMMAND) kubectl wait --for=condition=ready pod -l app=minio --timeout=600s - kubectl apply -f k8s/external/minio/mb-job.yaml + kubectl apply -f $(ROOTDIR)/k8s/external/minio/mb-job.yaml sleep $(K8S_SLEEP_DURATION_FOR_WAIT_COMMAND) kubectl wait --for=condition=complete job/minio-make-bucket --timeout=600s .PHONY: k8s/external/minio/delete ## delete minio k8s/external/minio/delete: - kubectl delete -f k8s/external/minio + kubectl delete -f $(ROOTDIR)/k8s/external/minio .PHONY: k8s/metrics/metrics-server/deploy ## deploy metrics-serrver @@ -374,12 +374,12 @@ k8s/metrics/metrics-server/delete: .PHONY: k8s/metrics/prometheus/deploy ## deploy prometheus k8s/metrics/prometheus/deploy: - kubectl apply -f k8s/metrics/prometheus + kubectl apply -f $(ROOTDIR)/k8s/metrics/prometheus .PHONY: k8s/metrics/prometheus/delete ## delete prometheus k8s/metrics/prometheus/delete: - kubectl delete -f k8s/metrics/prometheus + kubectl delete -f $(ROOTDIR)/k8s/metrics/prometheus .PHONY: k8s/metrics/prometheus/operator/deploy ## deploy prometheus operator @@ -395,14 +395,14 @@ k8s/metrics/prometheus/operator/delete: .PHONY: k8s/metrics/grafana/deploy ## deploy grafana k8s/metrics/grafana/deploy: - kubectl apply -f k8s/metrics/grafana/dashboards - kubectl apply -f k8s/metrics/grafana + kubectl apply -f $(ROOTDIR)/k8s/metrics/grafana/dashboards + kubectl apply -f $(ROOTDIR)/k8s/metrics/grafana .PHONY: k8s/metrics/grafana/delete ## delete grafana k8s/metrics/grafana/delete: - kubectl delete -f k8s/metrics/grafana/dashboards - kubectl delete -f k8s/metrics/grafana + kubectl delete -f $(ROOTDIR)/k8s/metrics/grafana/dashboards + kubectl delete -f $(ROOTDIR)/k8s/metrics/grafana .PHONY: k8s/metrics/jaeger/deploy ## deploy jaeger @@ -412,63 +412,63 @@ k8s/metrics/jaeger/deploy: kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=jaeger-operator --timeout=60s kubectl wait --for=condition=available deployment/jaeger-jaeger-operator --timeout=60s sleep $(JAEGER_OPERATOR_WAIT_DURATION) - kubectl apply -f k8s/metrics/jaeger/jaeger.yaml + kubectl apply -f $(ROOTDIR)/k8s/metrics/jaeger/jaeger.yaml .PHONY: k8s/metrics/jaeger/delete ## delete jaeger k8s/metrics/jaeger/delete: - kubectl delete -f k8s/metrics/jaeger + kubectl delete -f $(ROOTDIR)/k8s/metrics/jaeger helm uninstall jaeger .PHONY: k8s/metrics/loki/deploy ## deploy loki and promtail k8s/metrics/loki/deploy: - kubectl apply -f k8s/metrics/loki + kubectl apply -f $(ROOTDIR)/k8s/metrics/loki .PHONY: k8s/metrics/loki/delete ## delete loki and promtail k8s/metrics/loki/delete: - kubectl delete -f k8s/metrics/loki + kubectl delete -f $(ROOTDIR)/k8s/metrics/loki .PHONY: k8s/metrics/tempo/deploy ## deploy tempo and jaeger-agent k8s/metrics/tempo/deploy: - kubectl apply -f k8s/metrics/tempo + kubectl apply -f $(ROOTDIR)/k8s/metrics/tempo .PHONY: k8s/metrics/tempo/delete ## delete tempo and jaeger-agent k8s/metrics/tempo/delete: - kubectl delete -f k8s/metrics/tempo + kubectl delete -f $(ROOTDIR)/k8s/metrics/tempo .PHONY: k8s/metrics/profefe/deploy ## deploy profefe k8s/metrics/profefe/deploy: - kubectl apply -f k8s/metrics/profefe + kubectl apply -f $(ROOTDIR)/k8s/metrics/profefe .PHONY: k8s/metrics/profefe/delete ## delete profefe k8s/metrics/profefe/delete: - kubectl delete -f k8s/metrics/profefe + kubectl delete -f $(ROOTDIR)/k8s/metrics/profefe .PHONY: k8s/metrics/pyroscope/deploy ## deploy pyroscope k8s/metrics/pyroscope/deploy: - kubectl apply -k k8s/metrics/pyroscope/base + kubectl apply -k $(ROOTDIR)/k8s/metrics/pyroscope/base .PHONY: k8s/metrics/pyroscope/delete ## delete pyroscope k8s/metrics/pyroscope/delete: - kubectl delete -k k8s/metrics/pyroscope/base + kubectl delete -k $(ROOTDIR)/k8s/metrics/pyroscope/base .PHONY: k8s/metrics/pyroscope/pv/deploy ## deploy pyroscope on persistent volume k8s/metrics/pyroscope/pv/deploy: - kubectl apply -k k8s/metrics/pyroscope/overlay + kubectl apply -k $(ROOTDIR)/k8s/metrics/pyroscope/overlay .PHONY: k8s/metrics/pyroscope/pv/delete ## delete pyroscope on persistent volume k8s/metrics/pyroscope/pv/delete: - kubectl delete -k k8s/metrics/pyroscope/overlay + kubectl delete -k $(ROOTDIR)/k8s/metrics/pyroscope/overlay .PHONY: k8s/linkerd/deploy ## deploy linkerd to k8s diff --git a/Makefile.d/proto.mk b/Makefile.d/proto.mk index 2eef443f91..1870f8aa72 100644 --- a/Makefile.d/proto.mk +++ b/Makefile.d/proto.mk @@ -22,9 +22,9 @@ proto/all: \ .PHONY: proto/clean ## clean proto artifacts proto/clean: - find apis/grpc -name "*.pb.go" | xargs -P$(CORES) rm -f - find apis/grpc -name "*.pb.json.go" | xargs -P$(CORES) rm -f - rm -rf apis/swagger apis/docs + find $(ROOTDIR)/apis/grpc -name "*.pb.go" | xargs -P$(CORES) rm -f + find $(ROOTDIR)/apis/grpc -name "*.pb.json.go" | xargs -P$(CORES) rm -f + rm -rf $(ROOTDIR)/apis/swagger $(ROOTDIR)/apis/docs .PHONY: proto/paths/print ## print proto paths diff --git a/example/client/mirror/main.go b/example/client/mirror/main.go index 3bd0318166..4c8d5be4c3 100644 --- a/example/client/mirror/main.go +++ b/example/client/mirror/main.go @@ -18,13 +18,13 @@ import ( "encoding/json" "flag" "log" - "strings" "time" "github.com/kpango/fuid" "github.com/kpango/glg" "github.com/vdaas/vald-client-go/v1/payload" "github.com/vdaas/vald-client-go/v1/vald" + "github.com/vdaas/vald/internal/strings" "gonum.org/v1/hdf5" "google.golang.org/grpc" ) diff --git a/hack/actions/gen/main.go b/hack/actions/gen/main.go index 63298521c1..3461b86d9f 100644 --- a/hack/actions/gen/main.go +++ b/hack/actions/gen/main.go @@ -24,7 +24,6 @@ import ( "io/fs" "os" "os/signal" - "strings" "syscall" "text/template" "time" @@ -32,6 +31,7 @@ import ( "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/sync/errgroup" "gopkg.in/yaml.v2" ) diff --git a/hack/cspell/main.go b/hack/cspell/main.go index 0595983918..075bb26076 100644 --- a/hack/cspell/main.go +++ b/hack/cspell/main.go @@ -22,8 +22,9 @@ import ( "os" "regexp" "slices" - "strings" - "sync" + + "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" ) type CSpellConfig struct { diff --git a/hack/docker/gen/main.go b/hack/docker/gen/main.go index 3cd50d9f81..bb17b44e65 100644 --- a/hack/docker/gen/main.go +++ b/hack/docker/gen/main.go @@ -805,10 +805,7 @@ func main() { data.Environments["HOME"] = "/" + rootUser data.Environments["USER"] = rootUser } else { - user := data.BuildUser - if strings.Contains(user, ":") { - user = strings.SplitN(user, ":", 2)[0] - } + user, _, _ := strings.Cut(data.BuildUser, ":") data.Environments["HOME"] = "/home/" + user data.Environments["USER"] = user } diff --git a/hack/tools/deadlink/main.go b/hack/tools/deadlink/main.go index 2579dbe72e..098523acf7 100644 --- a/hack/tools/deadlink/main.go +++ b/hack/tools/deadlink/main.go @@ -23,13 +23,13 @@ import ( "os" "path/filepath" "regexp" - "strings" "sync/atomic" "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/http/client" + "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/internal/sync/errgroup" ) diff --git a/internal/core/algorithm/faiss/faiss.go b/internal/core/algorithm/faiss/faiss.go index 1a43dbc83b..5e0d19021d 100644 --- a/internal/core/algorithm/faiss/faiss.go +++ b/internal/core/algorithm/faiss/faiss.go @@ -24,11 +24,11 @@ package faiss import "C" import ( - "sync" "unsafe" "github.com/vdaas/vald/internal/core/algorithm" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync" ) type ( diff --git a/internal/core/algorithm/faiss/option.go b/internal/core/algorithm/faiss/option.go index d426722b27..89ea6ffad2 100644 --- a/internal/core/algorithm/faiss/option.go +++ b/internal/core/algorithm/faiss/option.go @@ -24,10 +24,9 @@ package faiss import "C" import ( - "strings" - "github.com/vdaas/vald/internal/core/algorithm" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/strings" ) // Option represents the functional option for faiss. diff --git a/internal/core/algorithm/usearch/option.go b/internal/core/algorithm/usearch/option.go index d4bf0061c8..71dd10a0ee 100644 --- a/internal/core/algorithm/usearch/option.go +++ b/internal/core/algorithm/usearch/option.go @@ -19,12 +19,12 @@ package usearch import ( "strconv" - "strings" "github.com/kpango/fastime" core "github.com/unum-cloud/usearch/golang" "github.com/vdaas/vald/internal/core/algorithm" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/strings" ) // Option represents the functional option for usearch. diff --git a/internal/core/algorithm/usearch/usearch.go b/internal/core/algorithm/usearch/usearch.go index 0d8c647f63..0aa50f8260 100644 --- a/internal/core/algorithm/usearch/usearch.go +++ b/internal/core/algorithm/usearch/usearch.go @@ -19,11 +19,11 @@ package usearch import ( "strconv" - "sync" core "github.com/unum-cloud/usearch/golang" "github.com/vdaas/vald/internal/core/algorithm" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync" ) type ( diff --git a/internal/info/info.go b/internal/info/info.go index 223a507b07..e5cd9cd447 100644 --- a/internal/info/info.go +++ b/internal/info/info.go @@ -37,9 +37,10 @@ type Info interface { } type info struct { - baseURL string // e.g https://github.com/vdaas/vald/tree/main - detail Detail - prepOnce sync.Once + baseURL string // e.g https://github.com/vdaas/vald/tree/main + detail Detail + prepOnce sync.Once + valdReplacer *strings.Replacer // runtime functions rtCaller func(skip int) (pc uintptr, file string, line int, ok bool) @@ -115,13 +116,14 @@ var ( ) const ( - goSrc = "go/src/" - goSrcLen = len(goSrc) - goMod = "go/pkg/mod/" - goModLen = len(goMod) - cgoTrue = "true" - cgoFalse = "false" - cgoUnknown = "unknown" + goSrc = "go/src/" + goSrcLen = len(goSrc) + goMod = "go/pkg/mod/" + goModLen = len(goMod) + cgoTrue = "true" + cgoFalse = "false" + cgoUnknown = "unknown" + googleGolang = "google.golang.org" ) // Init initializes Detail object only once. @@ -298,11 +300,35 @@ func (i info) getDetail() Detail { if funcName == "runtime.main" { break } + index := strings.LastIndex(funcName, "/") + if index != -1 { + funcName = funcName[index+1:] + } url := i.baseURL var idx int switch { case strings.HasPrefix(file, i.detail.GoRoot+"/src"): - url = "https://github.com/golang/go/blob/" + i.detail.GoVersion + strings.TrimPrefix(file, i.detail.GoRoot) + "#L" + strconv.Itoa(line) + url = "https://github.com/golang/go/blob/" + i.detail.GoVersion + strings.TrimPrefix(file, i.detail.GoRoot) + case strings.HasPrefix(file, "runtime"): + url = "https://github.com/golang/go/blob/master/src/" + file + case strings.HasPrefix(file, googleGolang+"/grpc"): + // google.golang.org/grpc@v1.65.0/server.go to https://github.com/grpc/grpc-go/blob/v1.65.0/server.go + url = "https://github.com/grpc/grpc-go/blob/" + _, versionSource, ok := strings.Cut(file, "@") + if ok && versionSource != "" { + url += versionSource + } else { + url = strings.ReplaceAll(file, googleGolang+"/grpc@", url) + } + case strings.HasPrefix(file, googleGolang+"/protobuf"): + // google.golang.org/protobuf@v1.34.0/proto/decode.go to https://github.com/protocolbuffers/protobuf-go/blob/v1.34.0/proto/decode.go + url = "https://github.com/protocolbuffers/protobuf-go/blob/" + _, versionSource, ok := strings.Cut(file, "@") + if ok && versionSource != "" { + url += versionSource + } else { + url = strings.ReplaceAll(file, googleGolang+"/protobuf@", url) + } case func() bool { idx = strings.Index(file, goMod) return idx >= 0 @@ -319,15 +345,16 @@ func (i info) getDetail() Detail { } url += "/" + path } - url += "#L" + strconv.Itoa(line) case func() bool { idx = strings.Index(file, goSrc) return idx >= 0 && strings.Index(file, valdRepo) >= 0 }(): - url = strings.Replace(file[idx+goSrcLen:]+"#L"+strconv.Itoa(line), valdRepo, "https://"+valdRepo+"/blob/"+i.detail.GitCommit, -1) + url = i.valdReplacer.Replace(file[idx+goSrcLen:]) case strings.HasPrefix(file, valdRepo): - url = fmt.Sprintf("%s#L%d", strings.Replace(file, valdRepo, "https://"+valdRepo+"/blob/"+i.detail.GitCommit, -1), line) + url = i.valdReplacer.Replace(file) } + url += "#L" + strconv.Itoa(line) + i.detail.StackTrace = append(i.detail.StackTrace, StackTrace{ FuncName: funcName, File: file, @@ -390,9 +417,16 @@ func (i *info) prepare() { if len(i.detail.GoroutineCount) == 0 { i.detail.GoroutineCount = strconv.Itoa(runtime.NumGoroutine()) } + if i.valdReplacer == nil { + i.valdReplacer = strings.NewReplacer(valdRepo, "https://"+valdRepo+"/blob/"+i.detail.GitCommit) + } }) } func (s StackTrace) String() string { return "URL: " + s.URL + "\tFile: " + s.File + "\tLine: #" + strconv.Itoa(s.Line) + "\tFuncName: " + s.FuncName } + +func (s StackTrace) ShortString() string { + return s.URL + " " + s.FuncName +} diff --git a/internal/k8s/job/job.go b/internal/k8s/job/job.go index db984ef28c..bd29584969 100644 --- a/internal/k8s/job/job.go +++ b/internal/k8s/job/job.go @@ -16,13 +16,13 @@ package job import ( "context" "reflect" - "strings" - "sync" "time" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/k8s" "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" batchv1 "k8s.io/api/batch/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/builder" diff --git a/internal/net/grpc/errdetails/errdetails.go b/internal/net/grpc/errdetails/errdetails.go index a8e016c1d7..b7443978d7 100644 --- a/internal/net/grpc/errdetails/errdetails.go +++ b/internal/net/grpc/errdetails/errdetails.go @@ -411,12 +411,7 @@ func DebugInfoFromInfoDetail(v *info.Detail) (debug *DebugInfo) { if v.StackTrace != nil { debug.StackEntries = make([]string, 0, len(v.StackTrace)) for i, stack := range v.StackTrace { - debug.StackEntries = append(debug.GetStackEntries(), strings.Join([]string{ - "id:", - strconv.Itoa(i), - "stack_trace:", - stack.String(), - }, " ")) + debug.StackEntries = append(debug.GetStackEntries(), "id: "+strconv.Itoa(i)+" stack_trace: "+stack.ShortString()) } v.StackTrace = nil } diff --git a/internal/observability/metrics/mem/mem.go b/internal/observability/metrics/mem/mem.go index ece3236388..75cb3095a1 100644 --- a/internal/observability/metrics/mem/mem.go +++ b/internal/observability/metrics/mem/mem.go @@ -19,11 +19,11 @@ import ( "os" "runtime" "strconv" - "strings" "time" "github.com/vdaas/vald/internal/conv" "github.com/vdaas/vald/internal/observability/metrics" + "github.com/vdaas/vald/internal/strings" api "go.opentelemetry.io/otel/metric" view "go.opentelemetry.io/otel/sdk/metric" ) diff --git a/pkg/gateway/lb/handler/grpc/handler.go b/pkg/gateway/lb/handler/grpc/handler.go index f6ef4fdb42..10021e53e4 100644 --- a/pkg/gateway/lb/handler/grpc/handler.go +++ b/pkg/gateway/lb/handler/grpc/handler.go @@ -20,7 +20,6 @@ package grpc import ( "context" "fmt" - "io" "math" "slices" "strconv" @@ -33,6 +32,7 @@ import ( "github.com/vdaas/vald/internal/core/algorithm" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/grpc" diff --git a/pkg/gateway/mirror/handler/grpc/handler.go b/pkg/gateway/mirror/handler/grpc/handler.go index 7065b1e177..c424ac6a65 100644 --- a/pkg/gateway/mirror/handler/grpc/handler.go +++ b/pkg/gateway/mirror/handler/grpc/handler.go @@ -16,7 +16,6 @@ package grpc import ( "context" "fmt" - "io" "reflect" "sync/atomic" @@ -24,6 +23,7 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/apis/grpc/v1/vald" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/net/grpc/codes" diff --git a/pkg/gateway/mirror/service/gateway.go b/pkg/gateway/mirror/service/gateway.go index 51b4a7ad2a..6d708b91ad 100644 --- a/pkg/gateway/mirror/service/gateway.go +++ b/pkg/gateway/mirror/service/gateway.go @@ -16,13 +16,13 @@ package service import ( "context" "reflect" - "strings" "github.com/vdaas/vald/internal/client/v1/client/mirror" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/observability/trace" + "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/sync/errgroup" ) diff --git a/pkg/index/job/correction/service/corrector.go b/pkg/index/job/correction/service/corrector.go index 5bcf7a6e96..d403551c5e 100644 --- a/pkg/index/job/correction/service/corrector.go +++ b/pkg/index/job/correction/service/corrector.go @@ -17,7 +17,6 @@ import ( "cmp" "context" "fmt" - "io" "os" "reflect" "slices" @@ -31,6 +30,7 @@ import ( "github.com/vdaas/vald/internal/db/kvs/pogreb" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/io" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/net/grpc/codes" diff --git a/pkg/index/job/readreplica/rotate/service/rotator.go b/pkg/index/job/readreplica/rotate/service/rotator.go index ef24e67889..a57c7ecbfa 100644 --- a/pkg/index/job/readreplica/rotate/service/rotator.go +++ b/pkg/index/job/readreplica/rotate/service/rotator.go @@ -17,7 +17,6 @@ import ( "context" "fmt" "reflect" - "strings" "time" "github.com/vdaas/vald/internal/errors" @@ -27,6 +26,7 @@ import ( "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/sync/errgroup" "k8s.io/utils/ptr" ) diff --git a/tests/performance/max_vector_dim_test.go b/tests/performance/max_vector_dim_test.go index 1eb434ce49..d39db3e5f4 100644 --- a/tests/performance/max_vector_dim_test.go +++ b/tests/performance/max_vector_dim_test.go @@ -68,17 +68,15 @@ func init_ngt_service(dim int) (service.NGT, error) { } func parse(raw string) (key string, value int) { - text := strings.ReplaceAll(raw[:len(raw)-2], " ", "") - keyValue := strings.Split(text, ":") - val := 0 - if keyValue[1] != "" { - val, err := strconv.Atoi(keyValue[1]) + k, v, ok := strings.Cut(strings.ReplaceAll(raw[:len(raw)-2], " ", ""), ":") + if ok { + val, err := strconv.Atoi(v) if err != nil { - panic(err) + return k, 0 } - return keyValue[0], val + return k, val } - return keyValue[0], val + return k, 0 } func TestMain(m *testing.M) {