[do not merge] modified apiversion capabilities check #2073

Workflow file for this run

.github/workflows/e2e-deploy.yml at 6f233b4

	#
	# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	#
	name: "Run E2E deploy and integration test"
	on:
	push:
	tags:
	- "..*"
	- "v..*"
	- "..-"
	- "v..-"
	pull_request:
	types:
	- "labeled"
	jobs:
	dump_contexts_to_log:
	runs-on: ubuntu-latest
	steps:
	- name: Dump GitHub context
	id: github_context_step
	run: echo $JSON
	env:
	JSON: ${{ toJSON(github) }}
	- name: Dump job context
	run: echo $JSON
	env:
	JSON: ${{ toJSON(job) }}
	- name: Dump steps context
	run: echo $JSON
	env:
	JSON: ${{ toJSON(steps) }}
	- name: Dump runner context
	run: echo $JSON
	env:
	JSON: ${{ toJSON(runner) }}
	- name: Dump strategy context
	run: echo $JSON
	env:
	JSON: ${{ toJSON(strategy) }}
	- name: Dump matrix context
	run: echo $JSON
	env:
	JSON: ${{ toJSON(matrix) }}
	e2e-stream-crud:
	name: "E2E test (Stream CRUD)"
	runs-on: ubuntu-latest
	timeout-minutes: 60
	if: startsWith( github.ref, 'refs/tags/') \|\| github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	steps:
	- uses: actions/checkout@v3
	- name: set git config
	run: \|
	git config --global --add safe.directory ${GITHUB_WORKSPACE}
	- name: Wait for Docker images
	if: startsWith( github.ref, 'refs/tags/')
	id: wait_for_docker_images
	uses: ./.github/actions/wait-for-docker-image
	- name: Get PR number
	id: get_pr_number
	if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	run: \|
	pr_num=`cat $GITHUB_EVENT_PATH \| jq -r ".number"`
	echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT
	- name: Specify container versions
	if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	id: specify_container_versions
	uses: ./.github/actions/detect-docker-image-tags
	with:
	tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }}
	- name: Fetch Helm version
	run: \|
	HELM_VERSION=`make version/helm`
	echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT
	id: version
	- uses: rinx/[email protected]
	with:
	version: latest
	name: vald
	agents: 3
	- name: check k3d
	run: \|
	kubectl cluster-info
	- uses: azure/setup-helm@v3
	with:
	version: ${{ steps.version.outputs.helm }}
	- name: Helm version
	run: \|
	helm version
	- name: deploy Vald
	id: deploy_vald
	uses: ./.github/actions/e2e-deploy-vald
	with:
	require_libhdf5: "true"
	helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }}
	values: .github/helm/values/values-lb.yaml
	wait_for_selector: app=vald-lb-gateway
	- name: Fetch golang version
	run: \|
	GO_VERSION=`make version/go`
	echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT
	id: golang_version
	- uses: actions/setup-go@v3
	with:
	go-version: ${{ steps.golang_version.outputs.version }}
	- name: run E2E CRUD
	run: \|
	make hack/benchmark/assets/dataset/${DATASET}
	go version
	make E2E_BIND_PORT=8081 \
	E2E_DATASET_NAME=${DATASET} \
	E2E_INSERT_COUNT=60000 \
	E2E_SEARCH_COUNT=10000 \
	E2E_SEARCH_BY_ID_COUNT=10000 \
	E2E_GET_OBJECT_COUNT=100 \
	E2E_UPDATE_COUNT=100 \
	E2E_UPSERT_COUNT=100 \
	E2E_REMOVE_COUNT=100 \
	E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \
	E2E_TARGET_POD_NAME=${POD_NAME} \
	E2E_TARGET_NAMESPACE=default \
	e2e
	env:
	DATASET: fashion-mnist-784-euclidean.hdf5
	POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
	e2e-stream-crud-for-operator:
	name: "E2E test (Stream CRUD) for operator"
	runs-on: ubuntu-latest
	timeout-minutes: 60
	if: startsWith( github.ref, 'refs/tags/') \|\| github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	steps:
	- uses: actions/checkout@v3
	- name: set git config
	run: \|
	git config --global --add safe.directory ${GITHUB_WORKSPACE}
	- name: Wait for Docker images
	if: startsWith( github.ref, 'refs/tags/')
	id: wait_for_docker_images
	uses: ./.github/actions/wait-for-docker-image
	- name: Get PR number
	id: get_pr_number
	if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	run: \|
	pr_num=`cat $GITHUB_EVENT_PATH \| jq -r ".number"`
	echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT
	- name: Specify container versions
	if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	id: specify_container_versions
	uses: ./.github/actions/detect-docker-image-tags
	with:
	tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }}
	- name: Fetch Helm version
	run: \|
	HELM_VERSION=`make version/helm`
	echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT
	id: version
	- uses: rinx/[email protected]
	with:
	version: latest
	name: vald
	agents: 3
	- name: check k3d
	run: \|
	kubectl cluster-info
	- uses: azure/setup-helm@v3
	with:
	version: ${{ steps.version.outputs.helm }}
	- name: Helm version
	run: \|
	helm version
	- name: Merge image tag
	run: \|
	IMAGE_TAGS=(${{ steps.specify_container_versions.outputs.IMAGE_TAGS }})

	for IMAGE_TAG in "${IMAGE_TAGS[@]}"
	do
	echo "IMAGE_TAG:${IMAGE_TAG}"

	ARR=(${IMAGE_TAG//=/ })
	FIELD=${ARR[0]}
	TAG="\"${ARR[1]}\""
	yq e ".spec.$FIELD=$TAG" -i ./.github/valdrelease/valdrelease.yaml
	done
	- name: deploy Vald
	id: deploy_vald
	uses: ./.github/actions/e2e-deploy-vald-helm-operator
	with:
	require_libhdf5: "true"
	valdrelease: ./.github/valdrelease/valdrelease.yaml
	wait_for_selector: app=vald-lb-gateway
	- name: Fetch golang version
	run: \|
	GO_VERSION=`make version/go`
	echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT
	id: golang_version
	- uses: actions/setup-go@v3
	with:
	go-version: ${{ steps.golang_version.outputs.version }}
	- name: run E2E CRUD
	run: \|
	make hack/benchmark/assets/dataset/${DATASET}
	go version
	make E2E_BIND_PORT=8081 \
	E2E_DATASET_NAME=${DATASET} \
	E2E_INSERT_COUNT=60000 \
	E2E_SEARCH_COUNT=10000 \
	E2E_SEARCH_BY_ID_COUNT=10000 \
	E2E_GET_OBJECT_COUNT=100 \
	E2E_UPDATE_COUNT=100 \
	E2E_UPSERT_COUNT=100 \
	E2E_REMOVE_COUNT=100 \
	E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \
	E2E_TARGET_POD_NAME=${POD_NAME} \
	E2E_TARGET_NAMESPACE=default \
	e2e
	env:
	DATASET: fashion-mnist-784-euclidean.hdf5
	POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}

	e2e-stream-crud-skip-exist-check:
	name: "E2E test (Stream CRUD: skip strict exist check)"
	runs-on: ubuntu-latest
	timeout-minutes: 60
	if: startsWith( github.ref, 'refs/tags/') \|\| github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	steps:
	- uses: actions/checkout@v3
	- name: set git config
	run: \|
	git config --global --add safe.directory ${GITHUB_WORKSPACE}
	- name: Wait for Docker images
	if: startsWith( github.ref, 'refs/tags/')
	id: wait_for_docker_images
	uses: ./.github/actions/wait-for-docker-image
	- name: Get PR number
	id: get_pr_number
	if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	run: \|
	pr_num=`cat $GITHUB_EVENT_PATH \| jq -r ".number"`
	echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT
	- name: Specify container versions
	if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	id: specify_container_versions
	uses: ./.github/actions/detect-docker-image-tags
	with:
	tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }}
	- name: Fetch Helm version
	run: \|
	HELM_VERSION=`make version/helm`
	echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT
	id: version
	- uses: rinx/[email protected]
	with:
	version: latest
	name: vald
	agents: 3
	- name: check k3d
	run: \|
	kubectl cluster-info
	- uses: azure/setup-helm@v3
	with:
	version: ${{ steps.version.outputs.helm }}
	- name: Helm version
	run: \|
	helm version
	- name: deploy Vald
	id: deploy_vald
	uses: ./.github/actions/e2e-deploy-vald
	with:
	require_libhdf5: "true"
	helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }}
	values: .github/helm/values/values-lb.yaml
	wait_for_selector: app=vald-lb-gateway
	- name: Fetch golang version
	run: \|
	GO_VERSION=`make version/go`
	echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT
	id: golang_version
	- uses: actions/setup-go@v3
	with:
	go-version: ${{ steps.golang_version.outputs.version }}
	- name: run E2E CRUD
	run: \|
	make hack/benchmark/assets/dataset/${DATASET}
	go version
	make E2E_BIND_PORT=8081 \
	E2E_DATASET_NAME=${DATASET} \
	E2E_INSERT_COUNT=10 \
	E2E_SEARCH_COUNT=10 \
	E2E_SEARCH_BY_ID_COUNT=10 \
	E2E_GET_OBJECT_COUNT=10 \
	E2E_UPDATE_COUNT=10 \
	E2E_UPSERT_COUNT=10 \
	E2E_REMOVE_COUNT=10 \
	E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \
	E2E_TARGET_POD_NAME=${POD_NAME} \
	E2E_TARGET_NAMESPACE=default \
	e2e/skip
	env:
	DATASET: fashion-mnist-784-euclidean.hdf5
	POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
	e2e-multiapis-crud:
	name: "E2E test (Multi-APIs CRUD)"
	runs-on: ubuntu-latest
	timeout-minutes: 60
	if: startsWith( github.ref, 'refs/tags/') \|\| github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	steps:
	- uses: actions/checkout@v3
	- name: set git config
	run: \|
	git config --global --add safe.directory ${GITHUB_WORKSPACE}
	- name: Wait for Docker images
	if: startsWith( github.ref, 'refs/tags/')
	id: wait_for_docker_images
	uses: ./.github/actions/wait-for-docker-image
	- name: Get PR number
	id: get_pr_number
	if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	run: \|
	pr_num=`cat $GITHUB_EVENT_PATH \| jq -r ".number"`
	echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT
	- name: Specify container versions
	if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	id: specify_container_versions
	uses: ./.github/actions/detect-docker-image-tags
	with:
	tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }}
	- name: Fetch Helm version
	run: \|
	HELM_VERSION=`make version/helm`
	echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT
	id: version
	- uses: rinx/[email protected]
	with:
	version: latest
	name: vald
	agents: 3
	- name: check k3d
	run: \|
	kubectl cluster-info
	- uses: azure/setup-helm@v3
	with:
	version: ${{ steps.version.outputs.helm }}
	- name: Helm version
	run: \|
	helm version
	- name: deploy Vald
	id: deploy_vald
	uses: ./.github/actions/e2e-deploy-vald
	with:
	require_libhdf5: "true"
	helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }}
	values: .github/helm/values/values-lb.yaml
	wait_for_selector: app=vald-lb-gateway
	- name: Fetch golang version
	run: \|
	GO_VERSION=`make version/go`
	echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT
	id: golang_version
	- uses: actions/setup-go@v3
	with:
	go-version: ${{ steps.golang_version.outputs.version }}
	- name: run E2E CRUD
	run: \|
	make hack/benchmark/assets/dataset/${DATASET}
	go version
	make E2E_BIND_PORT=8081 \
	E2E_DATASET_NAME=${DATASET} \
	E2E_INSERT_COUNT=100 \
	E2E_SEARCH_COUNT=10 \
	E2E_SEARCH_BY_ID_COUNT=10 \
	E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \
	E2E_TARGET_POD_NAME=${POD_NAME} \
	E2E_TARGET_NAMESPACE=default \
	e2e/multi
	env:
	DATASET: fashion-mnist-784-euclidean.hdf5
	POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
	e2e-agent-and-sidecar:
	name: "E2E Agent & Sidecar test"
	runs-on: ubuntu-latest
	timeout-minutes: 60
	if: startsWith( github.ref, 'refs/tags/') \|\| github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	steps:
	- uses: actions/checkout@v3
	- name: set git config
	run: \|
	git config --global --add safe.directory ${GITHUB_WORKSPACE}
	- name: Wait for Docker images
	if: startsWith( github.ref, 'refs/tags/')
	id: wait_for_docker_images
	uses: ./.github/actions/wait-for-docker-image
	with:
	images: vdaas/vald-agent-ngt vdaas/vald-agent-sidecar
	- name: Get PR number
	id: get_pr_number
	if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	run: \|
	pr_num=`cat $GITHUB_EVENT_PATH \| jq -r ".number"`
	echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT
	- name: Specify container versions
	if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
	id: specify_container_versions
	uses: ./.github/actions/detect-docker-image-tags
	with:
	tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }}
	images: vdaas/vald-agent-ngt vdaas/vald-agent-sidecar
	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y libhdf5-dev
	- name: Fetch Helm version
	run: \|
	HELM_VERSION=`make version/helm`
	echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT
	id: version
	- uses: rinx/[email protected]
	with:
	version: latest
	name: vald
	agents: 3
	- name: check k3d
	run: \|
	kubectl cluster-info
	- uses: azure/setup-helm@v3
	with:
	version: ${{ steps.version.outputs.helm }}
	- name: Helm version
	run: \|
	helm version
	- name: deploy Vald
	id: deploy_vald
	uses: ./.github/actions/e2e-deploy-vald
	with:
	require_libhdf5: "true"
	require_minio: "true"
	helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }}
	values: .github/helm/values/values-agent-sidecar.yaml
	wait_for_selector: app=vald-agent-ngt
	wait_for_timeout: 29m
	- name: Fetch golang version
	run: \|
	GO_VERSION=`make version/go`
	echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT
	id: golang_version
	- uses: actions/setup-go@v3
	with:
	go-version: ${{ steps.golang_version.outputs.version }}
	- name: run E2E Agent & Sidecar
	run: \|
	make hack/benchmark/assets/dataset/${DATASET}
	go version
	make E2E_BIND_PORT=8081 \
	E2E_DATASET_NAME=${DATASET} \
	E2E_INSERT_COUNT=10000 \
	E2E_SEARCH_COUNT=4000 \
	E2E_WAIT_FOR_CREATE_INDEX_DURATION=8m \
	E2E_TARGET_POD_NAME=${POD_NAME} \
	E2E_TARGET_NAMESPACE=default \
	e2e/sidecar
	env:
	DATASET: fashion-mnist-784-euclidean.hdf5
	POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
	slack-notification:
	name: "Slack notification"
	needs:
	- e2e-agent-and-sidecar
	- e2e-multiapis-crud
	- e2e-stream-crud
	- e2e-stream-crud-for-operator
	- e2e-stream-crud-skip-exist-check
	runs-on: ubuntu-latest
	if: startsWith( github.ref, 'refs/tags/')
	steps:
	- uses: technote-space/workflow-conclusion-action@v2
	with:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- uses: 8398a7/action-slack@v3
	with:
	author_name: "E2E deploy test"
	status: ${{ env.WORKFLOW_CONCLUSION }}
	only_mention_fail: channel
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[do not merge] modified apiversion capabilities check #2073

Workflow file

[do not merge] modified apiversion capabilities check #2073

Jobs

Run details

Workflow file for this run