e2e-chaos.yaml

#
# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: "Run E2E chaos test"
on:
  push:
    tags:
      - "*.*.*"
      - "v*.*.*"
      - "*.*.*-*"
      - "v*.*.*-*"
  pull_request:
    types:
      - "labeled"
env:
  VALUES: .github/helm/values/values-chaos.yaml
  DATASET: fashion-mnist-784-euclidean.hdf5
  INGRESS_PORT: 8081
jobs:
  dump-contexts-to-log:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/dump-context
  detect-ci-container:
    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos'
    uses: ./.github/workflows/_detect-ci-container.yaml
  agent-failure:
    name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)"
    needs: [detect-ci-container]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    container:
      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
      options: "--add-host host.docker.internal:host-gateway"
    steps:
      - uses: actions/checkout@v4
      - name: Set Git config
        run: |
          git config --global --add safe.directory ${GITHUB_WORKSPACE}
      - name: Setup E2E environment
        id: setup_e2e
        uses: ./.github/actions/setup-e2e
        with:
          ingress_port: ${{ env.INGRESS_PORT }}
      - name: Deploy Vald
        id: deploy_vald
        uses: ./.github/actions/e2e-deploy-vald
        with:
          helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}
          values: ${{ env.VALUES }}
          wait_for_selector: app=vald-lb-gateway
      - name: Deploy Chaos Mesh
        uses: ./.github/actions/deploy-chaos-mesh
        with:
          helm_extra_options: "--set podChaos.failure.enabled=true"
      - name: Run Insert and Search operations
        run: |
          make hack/benchmark/assets/dataset/${{ env.DATASET }}
          make E2E_BIND_HOST=host.docker.internal \
            E2E_BIND_PORT=8081 \
            E2E_PORTFORWARD_ENABLED=false \
            E2E_DATASET_NAME=${{ env.DATASET }} \
            E2E_TIMEOUT=15m \
            E2E_INSERT_COUNT=5000 \
            E2E_SEARCH_COUNT=5000 \
            E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \
            e2e/insert/search
  random-pod-failure:
    name: "E2E chaos test (random Pod failure: to test redundancy)"
    needs: [detect-ci-container]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    container:
      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
      options: "--add-host host.docker.internal:host-gateway"
    steps:
      - uses: actions/checkout@v4
      - name: Set Git config
        run: |
          git config --global --add safe.directory ${GITHUB_WORKSPACE}
      - name: Setup E2E environment
        id: setup_e2e
        uses: ./.github/actions/setup-e2e
        with:
          ingress_port: ${{ env.INGRESS_PORT }}
      - name: Deploy Vald
        id: deploy_vald
        uses: ./.github/actions/e2e-deploy-vald
        with:
          helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}
          values: ${{ env.VALUES }}
          wait_for_selector: app=vald-lb-gateway
      - name: Deploy Chaos Mesh
        uses: ./.github/actions/deploy-chaos-mesh
        with:
          helm_extra_options: "--set podChaos.kill.enabled=true"
      - name: Run Insert and Search operations
        run: |
          make hack/benchmark/assets/dataset/${{ env.DATASET }}
          make E2E_BIND_HOST=host.docker.internal \
            E2E_BIND_PORT=8081 \
            E2E_PORTFORWARD_ENABLED=false \
            E2E_DATASET_NAME=${{ env.DATASET }} \
            E2E_TIMEOUT=15m \
            E2E_INSERT_COUNT=5000 \
            E2E_SEARCH_COUNT=5000 \
            E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \
            e2e/insert/search
  agent-network-partition:
    name: "E2E chaos test (agent network partition: to test retries)"
    needs: [detect-ci-container]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    container:
      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
      options: "--add-host host.docker.internal:host-gateway"
    steps:
      - uses: actions/checkout@v4
      - name: Set Git config
        run: |
          git config --global --add safe.directory ${GITHUB_WORKSPACE}
      - name: Setup E2E environment
        id: setup_e2e
        uses: ./.github/actions/setup-e2e
        with:
          ingress_port: ${{ env.INGRESS_PORT }}
      - name: Deploy Vald
        id: deploy_vald
        uses: ./.github/actions/e2e-deploy-vald
        with:
          helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}
          values: ${{ env.VALUES }}
          wait_for_selector: app=vald-lb-gateway
      - name: Deploy Chaos Mesh
        uses: ./.github/actions/deploy-chaos-mesh
        with:
          helm_extra_options: "--set networkChaos.partition.enabled=true"
      - name: Run Insert and Search operations
        run: |
          make hack/benchmark/assets/dataset/${{ env.DATASET }}
          make E2E_BIND_HOST=host.docker.internal \
            E2E_BIND_PORT=8081 \
            E2E_PORTFORWARD_ENABLED=false \
            E2E_DATASET_NAME=${{ env.DATASET }} \
            E2E_TIMEOUT=15m \
            E2E_INSERT_COUNT=10000 \
            E2E_SEARCH_COUNT=10000 \
            E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \
            e2e/insert/search
  clusterwide-network-bandwidth:
    name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)"
    needs: [detect-ci-container]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    container:
      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
      options: "--add-host host.docker.internal:host-gateway"
    steps:
      - uses: actions/checkout@v4
      - name: Set Git config
        run: |
          git config --global --add safe.directory ${GITHUB_WORKSPACE}
      - name: Setup E2E environment
        id: setup_e2e
        uses: ./.github/actions/setup-e2e
        with:
          ingress_port: ${{ env.INGRESS_PORT }}
      - name: Deploy Vald
        id: deploy_vald
        uses: ./.github/actions/e2e-deploy-vald
        with:
          helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}
          values: ${{ env.VALUES }}
          wait_for_selector: app=vald-lb-gateway
      - name: Deploy Chaos Mesh
        uses: ./.github/actions/deploy-chaos-mesh
        with:
          helm_extra_options: "--set networkChaos.bandwidth.enabled=true"
      - name: Run Insert and Search operations
        run: |
          make hack/benchmark/assets/dataset/${{ env.DATASET }}
          make E2E_BIND_HOST=host.docker.internal \
            E2E_BIND_PORT=8081 \
            E2E_PORTFORWARD_ENABLED=false \
            E2E_DATASET_NAME=${{ env.DATASET }} \
            E2E_TIMEOUT=15m \
            E2E_INSERT_COUNT=5000 \
            E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \
            e2e/insert/search
  slack-notification:
    name: "Slack notification"
    if: startsWith( github.ref, 'refs/tags/')
    runs-on: ubuntu-latest
    needs:
      - agent-failure
      - agent-network-partition
      - clusterwide-network-bandwidth
      - random-pod-failure
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/notify-slack
        with:
          author_name: E2E Chaos test
          slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }}