e2e-chaos.yaml

#
# Copyright (C) 2019-2023 vdaas.org vald team <vald@vdaas.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: "Run E2E chaos test"
on:
  push:
    tags:
      - "*.*.*"
      - "v*.*.*"
      - "*.*.*-*"
      - "v*.*.*-*"
  pull_request:
    types:
      - "labeled"

jobs:
  dump_contexts_to_log:
    runs-on: ubuntu-latest
    steps:
      - name: Dump GitHub context
        id: github_context_step
        run: echo $JSON
        env:
          JSON: ${{ toJSON(github) }}
      - name: Dump job context
        run: echo $JSON
        env:
          JSON: ${{ toJSON(job) }}
      - name: Dump steps context
        run: echo $JSON
        env:
          JSON: ${{ toJSON(steps) }}
      - name: Dump runner context
        run: echo $JSON
        env:
          JSON: ${{ toJSON(runner) }}
      - name: Dump strategy context
        run: echo $JSON
        env:
          JSON: ${{ toJSON(strategy) }}
      - name: Dump matrix context
        run: echo $JSON
        env:
          JSON: ${{ toJSON(matrix) }}
  agent-failure:
    name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)"
    runs-on: ubuntu-latest
    timeout-minutes: 60
    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos'
    steps:
      - uses: actions/checkout@v3
      - name: set git config
        run: |
          git config --global --add safe.directory ${GITHUB_WORKSPACE}
      - name: Wait for Docker images
        if: startsWith( github.ref, 'refs/tags/')
        id: wait_for_docker_images
        uses: ./.github/actions/wait-for-docker-image
      - name: Get PR number
        id: get_pr_number
        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
        run: |
          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
          echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT
      - name: Specify container versions
        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
        id: specify_container_versions
        uses: ./.github/actions/detect-docker-image-tags
        with:
          tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }}
      - name: Fetch Helm version
        run: |
          HELM_VERSION=`make version/helm`
          echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT
        id: version
      - uses: rinx/setup-k3d@v0.0.4
        with:
          version: latest
          name: vald
          agents: 3
          options: "--image docker.io/rancher/k3s:latest"
      - name: check k3d
        run: |
          kubectl cluster-info
      - uses: azure/setup-helm@v3
        with:
          version: ${{ steps.version.outputs.helm }}
      - name: Helm version
        run: |
          helm version
      - name: deploy Vald
        id: deploy_vald
        uses: ./.github/actions/e2e-deploy-vald
        with:
          require_libhdf5: "true"
          helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }}
          values: .github/helm/values/values-chaos.yaml
          wait_for_selector: app=vald-lb-gateway
      - name: deploy Chaos Mesh
        run: |
          make kubectl/install
          curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s
          helm install \
            --set podChaos.failure.enabled=true \
            vald-chaos-test tests/chaos/chart
      - name: Fetch golang version
        run: |
          GO_VERSION=`make version/go`
          echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT
        id: golang_version
      - uses: actions/setup-go@v3
        with:
          go-version: ${{ steps.golang_version.outputs.version }}
      - name: run Insert and Search jobs
        run: |
          make hack/benchmark/assets/dataset/${DATASET}
          go version
          make E2E_BIND_PORT=8081 \
            E2E_DATASET_NAME=${DATASET} \
            E2E_TIMEOUT=15m \
            E2E_INSERT_COUNT=10000 \
            E2E_SEARCH_COUNT=10000 \
            E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \
            E2E_TARGET_POD_NAME=${POD_NAME} \
            E2E_TARGET_NAMESPACE=default \
            e2e/insert/search
        env:
          DATASET: fashion-mnist-784-euclidean.hdf5
          POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
  random-pod-failure:
    name: "E2E chaos test (random Pod failure: to test redundancy)"
    runs-on: ubuntu-latest
    timeout-minutes: 60
    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos'
    steps:
      - uses: actions/checkout@v3
      - name: set git config
        run: |
          git config --global --add safe.directory ${GITHUB_WORKSPACE}
      - name: Wait for Docker images
        if: startsWith( github.ref, 'refs/tags/')
        id: wait_for_docker_images
        uses: ./.github/actions/wait-for-docker-image
      - name: Get PR number
        id: get_pr_number
        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
        run: |
          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
          echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT
      - name: Specify container versions
        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
        id: specify_container_versions
        uses: ./.github/actions/detect-docker-image-tags
        with:
          tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }}
      - name: Fetch Helm version
        run: |
          HELM_VERSION=`make version/helm`
          echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT
        id: version
      - uses: rinx/setup-k3d@v0.0.4
        with:
          version: latest
          name: vald
          agents: 3
          options: "--image docker.io/rancher/k3s:latest"
      - name: check k3d
        run: |
          kubectl cluster-info
      - uses: azure/setup-helm@v3
        with:
          version: ${{ steps.version.outputs.helm }}
      - name: Helm version
        run: |
          helm version
      - name: deploy Vald
        id: deploy_vald
        uses: ./.github/actions/e2e-deploy-vald
        with:
          require_libhdf5: "true"
          helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }}
          values: .github/helm/values/values-chaos.yaml
          wait_for_selector: app=vald-lb-gateway
      - name: deploy Chaos Mesh
        run: |
          make kubectl/install
          curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s
          helm install \
            --set podChaos.kill.enabled=true \
            vald-chaos-test tests/chaos/chart
      - name: Fetch golang version
        run: |
          GO_VERSION=`make version/go`
          echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT
        id: golang_version
      - uses: actions/setup-go@v3
        with:
          go-version: ${{ steps.golang_version.outputs.version }}
      - name: run Insert and Search jobs
        run: |
          make hack/benchmark/assets/dataset/${DATASET}
          podname=`kubectl get pods --selector=app=vald-lb-gateway | tail -1 | awk '{print $1}'`
          go version
          make E2E_BIND_PORT=8081 \
            E2E_DATASET_NAME=${DATASET} \
            E2E_TIMEOUT=15m \
            E2E_INSERT_COUNT=10000 \
            E2E_SEARCH_COUNT=10000 \
            E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \
            E2E_TARGET_POD_NAME=${podname} \
            E2E_TARGET_NAMESPACE=default \
            e2e/insert/search
        env:
          DATASET: fashion-mnist-784-euclidean.hdf5
  agent-network-partition:
    name: "E2E chaos test (agent network partition: to test retries)"
    runs-on: ubuntu-latest
    timeout-minutes: 60
    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos'
    steps:
      - uses: actions/checkout@v3
      - name: set git config
        run: |
          git config --global --add safe.directory ${GITHUB_WORKSPACE}
      - name: Wait for Docker images
        if: startsWith( github.ref, 'refs/tags/')
        id: wait_for_docker_images
        uses: ./.github/actions/wait-for-docker-image
      - name: Get PR number
        id: get_pr_number
        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
        run: |
          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
          echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT
      - name: Specify container versions
        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
        id: specify_container_versions
        uses: ./.github/actions/detect-docker-image-tags
        with:
          tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }}
      - name: Fetch Helm version
        run: |
          HELM_VERSION=`make version/helm`
          echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT
        id: version
      - uses: rinx/setup-k3d@v0.0.4
        with:
          version: latest
          name: vald
          agents: 3
          options: "--image docker.io/rancher/k3s:latest"
      - name: check k3d
        run: |
          kubectl cluster-info
      - uses: azure/setup-helm@v3
        with:
          version: ${{ steps.version.outputs.helm }}
      - name: Helm version
        run: |
          helm version
      - name: deploy Vald
        id: deploy_vald
        uses: ./.github/actions/e2e-deploy-vald
        with:
          require_libhdf5: "true"
          helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }}
          values: .github/helm/values/values-chaos.yaml
          wait_for_selector: app=vald-lb-gateway
      - name: deploy Chaos Mesh
        run: |
          make kubectl/install
          curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s
          helm install \
            --set networkChaos.partition.enabled=true \
            vald-chaos-test tests/chaos/chart
      - name: Fetch golang version
        run: |
          GO_VERSION=`make version/go`
          echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT
        id: golang_version
      - uses: actions/setup-go@v3
        with:
          go-version: ${{ steps.golang_version.outputs.version }}
      - name: run Insert and Search jobs
        run: |
          make hack/benchmark/assets/dataset/${DATASET}
          go version
          make E2E_BIND_PORT=8081 \
            E2E_DATASET_NAME=${DATASET} \
            E2E_TIMEOUT=15m \
            E2E_INSERT_COUNT=10000 \
            E2E_SEARCH_COUNT=10000 \
            E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \
            E2E_TARGET_POD_NAME=${POD_NAME} \
            E2E_TARGET_NAMESPACE=default \
            e2e/insert/search
        env:
          DATASET: fashion-mnist-784-euclidean.hdf5
          POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
  clusterwide-network-bandwidth:
    name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)"
    runs-on: ubuntu-latest
    timeout-minutes: 60
    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos'
    steps:
      - uses: actions/checkout@v3
      - name: set git config
        run: |
          git config --global --add safe.directory ${GITHUB_WORKSPACE}
      - name: Wait for Docker images
        if: startsWith( github.ref, 'refs/tags/')
        id: wait_for_docker_images
        uses: ./.github/actions/wait-for-docker-image
      - name: Get PR number
        id: get_pr_number
        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
        run: |
          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
          echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT
      - name: Specify container versions
        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
        id: specify_container_versions
        uses: ./.github/actions/detect-docker-image-tags
        with:
          tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }}
      - name: Fetch Helm version
        run: |
          HELM_VERSION=`make version/helm`
          echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT
        id: version
      - uses: rinx/setup-k3d@v0.0.4
        with:
          version: latest
          name: vald
          agents: 3
          options: "--image docker.io/rancher/k3s:latest"
      - name: check k3d
        run: |
          kubectl cluster-info
      - uses: azure/setup-helm@v3
        with:
          version: ${{ steps.version.outputs.helm }}
      - name: Helm version
        run: |
          helm version
      - name: deploy Vald
        id: deploy_vald
        uses: ./.github/actions/e2e-deploy-vald
        with:
          require_libhdf5: "true"
          helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }}
          values: .github/helm/values/values-lb.yaml
          wait_for_selector: app=vald-lb-gateway
      - name: deploy Chaos Mesh
        run: |
          make kubectl/install
          curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s
          helm install \
            --set networkChaos.bandwidth.enabled=true \
            vald-chaos-test tests/chaos/chart
      - name: Fetch golang version
        run: |
          GO_VERSION=`make version/go`
          echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT
        id: golang_version
      - uses: actions/setup-go@v3
        with:
          go-version: ${{ steps.golang_version.outputs.version }}
      - name: run Insert jobs
        run: |
          make hack/benchmark/assets/dataset/${DATASET}
          go version
          make E2E_BIND_PORT=8081 \
            E2E_DATASET_NAME=${DATASET} \
            E2E_TIMEOUT=15m \
            E2E_INSERT_COUNT=10000 \
            E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \
            E2E_TARGET_POD_NAME=${POD_NAME} \
            E2E_TARGET_NAMESPACE=default \
            e2e/insert
        env:
          DATASET: fashion-mnist-784-euclidean.hdf5
          POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
      - name: run Search jobs
        run: |
          make hack/benchmark/assets/dataset/${DATASET}
          go version
          make E2E_BIND_PORT=8081 \
            E2E_DATASET_NAME=${DATASET} \
            E2E_TIMEOUT=15m \
            E2E_SEARCH_COUNT=10000 \
            E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \
            E2E_TARGET_POD_NAME=${POD_NAME} \
            E2E_TARGET_NAMESPACE=default \
            e2e/search
        env:
          DATASET: fashion-mnist-784-euclidean.hdf5
          POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
  slack-notification:
    name: "Slack notification"
    needs:
      - agent-failure
      - random-pod-failure
      - agent-network-partition
      - clusterwide-network-bandwidth
    runs-on: ubuntu-latest
    if: startsWith( github.ref, 'refs/tags/')
    steps:
      - uses: technote-space/workflow-conclusion-action@v2
        with:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - uses: 8398a7/action-slack@v3
        with:
          author_name: "E2E chaos test"
          status: ${{ env.WORKFLOW_CONCLUSION }}
          only_mention_fail: channel
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }}