From 34d49f1010a70d7452bb42bf454f3508daa36caa Mon Sep 17 00:00:00 2001 From: Sergey Kudasov Date: Thu, 22 Aug 2024 10:22:09 +0200 Subject: [PATCH] Merge WASP library (#1082) * init * change mod name, add CI * try to trigger * commit go.sum * fix lint * move wasp to the root folder * fix lint * fix lint again * link readmes * change workflow to paths * move back to dorny for required checks * fix readme lint * try trigger CI --- .github/workflows/wasp-lint.yml | 28 + .github/workflows/wasp-test-e2e.yml | 31 + .github/workflows/wasp-test.yml | 27 + .golangci.yaml | 2 + .prettierignore | 1 + README.md | 12 +- _typos.toml | 10 +- wasp/.dockerignore | 1 + wasp/.env | 7 + wasp/.gitignore | 7 + wasp/.golangci.yaml | 15 + wasp/CODEOWNERS | 1 + wasp/DockerfileWasp | 22 + wasp/DockerfileWasp.dockerignore | 31 + wasp/HOW_IT_WORKS.md | 235 ++++ wasp/LICENSE | 21 + wasp/Makefile | 53 + wasp/README.md | 171 +++ wasp/SECURITY.md | 11 + wasp/alert.go | 104 ++ wasp/buffer.go | 26 + wasp/charts/wasp/.helmignore | 23 + wasp/charts/wasp/Chart.yaml | 6 + wasp/charts/wasp/namespace_setup/setup.yaml | 25 + wasp/charts/wasp/templates/_helpers.tpl | 62 + wasp/charts/wasp/templates/job.yaml | 69 + wasp/charts/wasp/values.yaml | 35 + wasp/charts/wasp/wasp-0.1.8.tgz | Bin 0 -> 2135 bytes wasp/cluster.go | 235 ++++ wasp/cmd.go | 49 + wasp/compose/conf/defaults.ini | 1238 +++++++++++++++++ wasp/compose/conf/grafana.ini | 1160 +++++++++++++++ .../provisioning/access-control/sample.yaml | 68 + .../conf/provisioning/dashboards/sample.yaml | 10 + .../conf/provisioning/datasources/loki.yaml | 10 + .../conf/provisioning/notifiers/sample.yaml | 24 + .../conf/provisioning/plugins/sample.yaml | 10 + .../compose/conf/provisioning/rules/rules.yml | 9 + wasp/compose/docker-compose.yaml | 31 + wasp/compose/loki-config.yaml | 47 + wasp/compose/pyroscope-compose.yaml | 9 + wasp/dashboard/cmd/main.go | 23 + wasp/dashboard/dashboard.go | 467 +++++++ wasp/dashboard/grafanasdk/panels.go | 381 +++++ wasp/dashboard/panels.go | 80 ++ wasp/docs/dashboard_basic.png | Bin 0 -> 651751 bytes wasp/docs/how-it-works.png | Bin 0 -> 130430 bytes wasp/docs/wasp-2.png | Bin 0 -> 36767 bytes wasp/examples/README.md | 122 ++ wasp/examples/alerts/gun.go | 35 + wasp/examples/alerts/main_test.go | 195 +++ wasp/examples/go.mod | 193 +++ wasp/examples/go.sum | 1186 ++++++++++++++++ wasp/examples/profiles/gun.go | 35 + wasp/examples/profiles/node_rps_test.go | 40 + wasp/examples/profiles/node_vu_test.go | 40 + wasp/examples/profiles/vu.go | 74 + wasp/examples/scenario/main_test.go | 32 + wasp/examples/scenario/vu.go | 79 ++ wasp/examples/simple_rps/gun.go | 35 + wasp/examples/simple_rps/main.go | 38 + wasp/examples/simple_vu/main_test.go | 46 + wasp/examples/simple_vu/vu.go | 61 + wasp/examples/zcluster/Dockerfile | 22 + wasp/examples/zcluster/build.sh | 38 + wasp/examples/zcluster/cluster_test.go | 34 + wasp/flake.lock | 61 + wasp/flake.nix | 15 + wasp/go.mod | 212 +++ wasp/go.sum | 1234 ++++++++++++++++ wasp/gun_http_mock.go | 39 + wasp/gun_sleep_mock.go | 67 + wasp/http_server_mock.go | 56 + wasp/k8s.go | 150 ++ wasp/log.go | 47 + wasp/loki_client.go | 226 +++ wasp/loki_client_test.go | 72 + wasp/perf_test.go | 250 ++++ wasp/profile.go | 223 +++ wasp/responses.go | 37 + wasp/sampler.go | 44 + wasp/schedule.go | 59 + wasp/schedule_test.go | 114 ++ wasp/shell.nix | 18 + wasp/stat.go | 46 + wasp/vu_sleep_mock.go | 81 ++ wasp/vu_ws_mock.go | 66 + wasp/wasp.go | 823 +++++++++++ wasp/wasp_bench_test.go | 29 + wasp/wasp_test.go | 1041 ++++++++++++++ wasp/ws_server_mock.go | 47 + 91 files changed, 12247 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/wasp-lint.yml create mode 100644 .github/workflows/wasp-test-e2e.yml create mode 100644 .github/workflows/wasp-test.yml create mode 100644 wasp/.dockerignore create mode 100644 wasp/.env create mode 100644 wasp/.gitignore create mode 100644 wasp/.golangci.yaml create mode 100644 wasp/CODEOWNERS create mode 100644 wasp/DockerfileWasp create mode 100644 wasp/DockerfileWasp.dockerignore create mode 100644 wasp/HOW_IT_WORKS.md create mode 100644 wasp/LICENSE create mode 100644 wasp/Makefile create mode 100644 wasp/README.md create mode 100644 wasp/SECURITY.md create mode 100644 wasp/alert.go create mode 100644 wasp/buffer.go create mode 100644 wasp/charts/wasp/.helmignore create mode 100644 wasp/charts/wasp/Chart.yaml create mode 100644 wasp/charts/wasp/namespace_setup/setup.yaml create mode 100644 wasp/charts/wasp/templates/_helpers.tpl create mode 100644 wasp/charts/wasp/templates/job.yaml create mode 100644 wasp/charts/wasp/values.yaml create mode 100644 wasp/charts/wasp/wasp-0.1.8.tgz create mode 100644 wasp/cluster.go create mode 100644 wasp/cmd.go create mode 100644 wasp/compose/conf/defaults.ini create mode 100644 wasp/compose/conf/grafana.ini create mode 100644 wasp/compose/conf/provisioning/access-control/sample.yaml create mode 100644 wasp/compose/conf/provisioning/dashboards/sample.yaml create mode 100644 wasp/compose/conf/provisioning/datasources/loki.yaml create mode 100644 wasp/compose/conf/provisioning/notifiers/sample.yaml create mode 100644 wasp/compose/conf/provisioning/plugins/sample.yaml create mode 100644 wasp/compose/conf/provisioning/rules/rules.yml create mode 100644 wasp/compose/docker-compose.yaml create mode 100644 wasp/compose/loki-config.yaml create mode 100644 wasp/compose/pyroscope-compose.yaml create mode 100644 wasp/dashboard/cmd/main.go create mode 100644 wasp/dashboard/dashboard.go create mode 100644 wasp/dashboard/grafanasdk/panels.go create mode 100644 wasp/dashboard/panels.go create mode 100644 wasp/docs/dashboard_basic.png create mode 100644 wasp/docs/how-it-works.png create mode 100644 wasp/docs/wasp-2.png create mode 100644 wasp/examples/README.md create mode 100644 wasp/examples/alerts/gun.go create mode 100644 wasp/examples/alerts/main_test.go create mode 100644 wasp/examples/go.mod create mode 100644 wasp/examples/go.sum create mode 100644 wasp/examples/profiles/gun.go create mode 100644 wasp/examples/profiles/node_rps_test.go create mode 100644 wasp/examples/profiles/node_vu_test.go create mode 100644 wasp/examples/profiles/vu.go create mode 100644 wasp/examples/scenario/main_test.go create mode 100644 wasp/examples/scenario/vu.go create mode 100644 wasp/examples/simple_rps/gun.go create mode 100644 wasp/examples/simple_rps/main.go create mode 100644 wasp/examples/simple_vu/main_test.go create mode 100644 wasp/examples/simple_vu/vu.go create mode 100755 wasp/examples/zcluster/Dockerfile create mode 100755 wasp/examples/zcluster/build.sh create mode 100644 wasp/examples/zcluster/cluster_test.go create mode 100644 wasp/flake.lock create mode 100644 wasp/flake.nix create mode 100644 wasp/go.mod create mode 100644 wasp/go.sum create mode 100644 wasp/gun_http_mock.go create mode 100644 wasp/gun_sleep_mock.go create mode 100644 wasp/http_server_mock.go create mode 100644 wasp/k8s.go create mode 100644 wasp/log.go create mode 100644 wasp/loki_client.go create mode 100644 wasp/loki_client_test.go create mode 100644 wasp/perf_test.go create mode 100644 wasp/profile.go create mode 100644 wasp/responses.go create mode 100644 wasp/sampler.go create mode 100644 wasp/schedule.go create mode 100644 wasp/schedule_test.go create mode 100644 wasp/shell.nix create mode 100644 wasp/stat.go create mode 100644 wasp/vu_sleep_mock.go create mode 100644 wasp/vu_ws_mock.go create mode 100644 wasp/wasp.go create mode 100644 wasp/wasp_bench_test.go create mode 100644 wasp/wasp_test.go create mode 100644 wasp/ws_server_mock.go diff --git a/.github/workflows/wasp-lint.yml b/.github/workflows/wasp-lint.yml new file mode 100644 index 000000000..effc9a8b2 --- /dev/null +++ b/.github/workflows/wasp-lint.yml @@ -0,0 +1,28 @@ +name: WASP Lint +on: + push: +permissions: + contents: read +jobs: + golangci: + defaults: + run: + working-directory: wasp + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + src: + - 'wasp/**' + - uses: cachix/install-nix-action@v18 + if: steps.changes.outputs.src == 'true' + with: + nix_path: nixpkgs=channel:nixos-unstable + - name: Run lint + if: steps.changes.outputs.src == 'true' + run: |- + nix develop -c make lint diff --git a/.github/workflows/wasp-test-e2e.yml b/.github/workflows/wasp-test-e2e.yml new file mode 100644 index 000000000..dc297c34d --- /dev/null +++ b/.github/workflows/wasp-test-e2e.yml @@ -0,0 +1,31 @@ +name: WASP E2E tests +on: [push] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + test: + defaults: + run: + working-directory: wasp + env: + LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }} + LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} + LOKI_URL: ${{ secrets.LOKI_URL }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + src: + - 'wasp/**' + - uses: cachix/install-nix-action@v18 + if: steps.changes.outputs.src == 'true' + with: + nix_path: nixpkgs=channel:nixos-unstable + - name: Run tests + if: steps.changes.outputs.src == 'true' + run: |- + nix develop -c make test_loki diff --git a/.github/workflows/wasp-test.yml b/.github/workflows/wasp-test.yml new file mode 100644 index 000000000..15dae11ee --- /dev/null +++ b/.github/workflows/wasp-test.yml @@ -0,0 +1,27 @@ +name: WASP Go Tests +on: [push] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + test: + defaults: + run: + working-directory: wasp + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + src: + - 'wasp/**' + - uses: cachix/install-nix-action@v18 + if: steps.changes.outputs.src == 'true' + with: + nix_path: nixpkgs=channel:nixos-unstable + - name: Run tests + if: steps.changes.outputs.src == 'true' + run: |- + nix develop -c make test_race diff --git a/.golangci.yaml b/.golangci.yaml index d24f15561..97a940f47 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -84,3 +84,5 @@ issues: - contracts/ethereum - examples - imports + - wasp/examples/* + - k8s diff --git a/.prettierignore b/.prettierignore index 6e486a834..5c08ba825 100644 --- a/.prettierignore +++ b/.prettierignore @@ -6,3 +6,4 @@ charts/**/README.md k8s-test-runner/chart/**/*.yaml node_modules/ index.yaml +wasp/** diff --git a/README.md b/README.md index ea1b7cc2a..2616bedb7 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,20 @@ -The Chainlink Testing Framework is a blockchain development framework written in Go. Its primary purpose is to help chainlink developers create extensive integration, e2e, performance, and chaos tests to ensure the stability of the chainlink project. It can also be helpful to those who just want to use chainlink oracles in their projects to help test their contracts, or even for those that aren't using chainlink. +The Chainlink Testing Framework (CTF) is a blockchain development framework written in Go. Its primary purpose is to help chainlink developers create extensive integration, e2e, performance, and chaos tests to ensure the stability of the chainlink project. It can also be helpful to those who just want to use chainlink oracles in their projects to help test their contracts, or even for those that aren't using chainlink. If you're looking to implement a new chain integration for the testing framework, head over to the [blockchain](./blockchain/) directory for more info. +# Content + +1. [Libraries](#libraries) + +## Libraries + +CTF contains a set of useful libraries: + +- [WASP](wasp/README.md) - Scalable protocol-agnostic load testing library for `Go` + ## k8s package We have a k8s package we are using in tests, it provides: diff --git a/_typos.toml b/_typos.toml index 590f56770..1d04c9b9b 100644 --- a/_typos.toml +++ b/_typos.toml @@ -1,2 +1,10 @@ [files] -extend-exclude = ["**/go.mod", "**/go.sum", "charts", "**/*.tgz", "**/*.png"] +extend-exclude = [ + "**/go.mod", + "**/go.sum", + "charts", + "**/*.tgz", + "**/*.png", + "wasp/HOW_IT_WORKS.md", + "wasp/dashboard/**" +] diff --git a/wasp/.dockerignore b/wasp/.dockerignore new file mode 100644 index 000000000..2d2ecd68d --- /dev/null +++ b/wasp/.dockerignore @@ -0,0 +1 @@ +.git/ diff --git a/wasp/.env b/wasp/.env new file mode 100644 index 000000000..cc4331cca --- /dev/null +++ b/wasp/.env @@ -0,0 +1,7 @@ +export LOKI_TOKEN= +export LOKI_URL=http://localhost:3030/loki/api/v1/push +export GRAFANA_URL=http://localhost:3000 +export GRAFANA_TOKEN= +export DATA_SOURCE_NAME=Loki +export DASHBOARD_FOLDER=LoadTests +export DASHBOARD_NAME=Wasp diff --git a/wasp/.gitignore b/wasp/.gitignore new file mode 100644 index 000000000..dd187f467 --- /dev/null +++ b/wasp/.gitignore @@ -0,0 +1,7 @@ +bin/ +.vscode/ +.idea/ +.direnv/ + +k3dvolume/ +.private.env diff --git a/wasp/.golangci.yaml b/wasp/.golangci.yaml new file mode 100644 index 000000000..4aaeac0bf --- /dev/null +++ b/wasp/.golangci.yaml @@ -0,0 +1,15 @@ +run: + timeout: 5m +issues: + exclude-use-default: false + exclude-dirs: + - bin + - imports + - examples/* +linters-settings: + revive: + rules: + - name: exported + severity: warning + - name: dot-imports + disabled: true diff --git a/wasp/CODEOWNERS b/wasp/CODEOWNERS new file mode 100644 index 000000000..cfb4f2262 --- /dev/null +++ b/wasp/CODEOWNERS @@ -0,0 +1 @@ +* @smartcontractkit/test-tooling-team diff --git a/wasp/DockerfileWasp b/wasp/DockerfileWasp new file mode 100644 index 000000000..17c68900f --- /dev/null +++ b/wasp/DockerfileWasp @@ -0,0 +1,22 @@ +# Example DockerfileWasp for k8s run +# Builds all the tests in some directory that must have go.mod +# All tests are built as separate binaries with name "module.test" +FROM golang:1.21 as build +ARG TESTS_ROOT + +WORKDIR /go/src +COPY . . + +RUN echo $(pwd) +RUN ls -lah +WORKDIR /go/src/${TESTS_ROOT} +RUN echo $(pwd) +RUN ls -lah +RUN cd /go/src/${TESTS_ROOT} && CGO_ENABLED=0 go test -c ./... + +FROM debian +ARG TESTS_ROOT + +COPY --from=build /go/src/${TESTS_ROOT} . +RUN apt-get update && apt-get install -y ca-certificates +ENTRYPOINT /bin/bash diff --git a/wasp/DockerfileWasp.dockerignore b/wasp/DockerfileWasp.dockerignore new file mode 100644 index 000000000..4eefe522f --- /dev/null +++ b/wasp/DockerfileWasp.dockerignore @@ -0,0 +1,31 @@ +.git-together +.DS_Store +.envrc +*.log +node_modules/ +**/node_modules/ +vendor/ +tmp/ + +contracts/node_modules +examples/ + +integration/ +integration-scripts/ + +tools/gethnet/datadir/geth +tools/clroot/db.bolt +tools/clroot/*.log +tools/clroot/tempkeys + +core/sgx/target/ + +core/*.Dockerfile +chainlink + +# codeship +codeship-*.yml +*.aes +dockercfg +credentials.env +gcr_creds.env diff --git a/wasp/HOW_IT_WORKS.md b/wasp/HOW_IT_WORKS.md new file mode 100644 index 000000000..1cd491359 --- /dev/null +++ b/wasp/HOW_IT_WORKS.md @@ -0,0 +1,235 @@ +# How it works + +## Overview +General idea is to be able to compose load tests programmatically by combining different `Generators` + +- `Generator` is an entity that can execute some workload using some `Gun` or `VU` definition, each `Generator` may have only one `Gun` or `VU` implementation used + +- `Gun` can be an implementation of single or multiple sequential requests workload for stateless protocols + +- `VU` is a stateful implementation that's more suitable for stateful protocols or when your client have some logic/simulating real users + +- Each `Generator` have a `Schedule` that control workload params throughout the test (increase/decrease RPS or VUs) + +- `Generators` can be combined to run multiple workload units in parallel or sequentially + +- `Profiles` are wrappers that allow you to run multiple generators with different `Schedules` and wait for all of them to finish + +- `ClusterProfiles` are high-level wrappers that create multiple profile parts and scale your test in `k8s` + +- `VU` implementations can also include sequential and parallel requests to simulate users behaviour + +- `AlertChecker` can be used in tests to check if any specific alerts with label and dashboardUUID was triggered and update test status + +Example `Syntetic/RPS` test diagram: + +```mermaid +--- +title: Syntetic/RPS test +--- +sequenceDiagram + participant Profile(Test) + participant Scheduler + participant Generator(Gun) + participant Promtail + participant Loki + participant Grafana + loop Test Execution + Profile(Test) ->> Generator(Gun): Start with (API, TestData) + loop Schedule + Scheduler ->> Scheduler: Process schedule segment + Scheduler ->> Generator(Gun): Set new RPS target + loop RPS load + Generator(Gun) ->> Generator(Gun): Execute Call() in parallel + Generator(Gun) ->> Promtail: Save CallResult + end + Promtail ->> Loki: Send batch
when ready or timeout + end + Scheduler ->> Scheduler: All segments done
wait all responses
test ends + Profile(Test) ->> Grafana: Check alert groups
FAIL or PASS the test + end +``` + +Example `VUs` test diagram: + +```mermaid +--- +title: VUs test +--- +sequenceDiagram + participant Profile(Test) + participant Scheduler + participant Generator(VUs) + participant VU1 + participant VU2 + participant Promtail + participant Loki + participant Grafana + loop Test Execution + Profile(Test) ->> Generator(VUs): Start with (API, TestData) + loop Schedule + Scheduler ->> Scheduler: Process schedule segment + Scheduler ->> Generator(VUs): Set new VUs target + loop VUs load + Generator(VUs) ->> Generator(VUs): Add/remove VUs + Generator(VUs) ->> VU1: Start/end + Generator(VUs) ->> VU2: Start/end + VU1 ->> VU1: Run loop, execute multiple calls + VU1 ->> Promtail: Save []CallResult + VU2 ->> VU2: Run loop, execute multiple calls + VU2 ->> Promtail: Save []CallResult + Promtail ->> Loki: Send batch
when ready or timeout + end + end + Scheduler ->> Scheduler: All segments done
wait all responses
test ends + Profile(Test) ->> Grafana: Check alert groups
FAIL or PASS the test + end +``` + +Load workflow testing diagram: +```mermaid +--- +title: Load testing workflow +--- +sequenceDiagram + participant Product repo + participant Runner + participant K8s + participant Loki + participant Grafana + participant Devs + Product repo->>Product repo: Define NFR for different workloads
Define application dashboard
Define dashboard alerts
Define load tests + Product repo->>Grafana: Upload app dashboard
Alerts has "requirement_name" label
Each "requirement_name" groups is based on some NFR + loop CI runs + Product repo->>Runner: CI Runs small load test + Runner->>Runner: Execute load test logic
Run multiple generators + Runner->>Loki: Stream load test data + Runner->>Grafana: Checking "requirement_name": "baseline" alerts + Grafana->>Devs: Notify devs (Dashboard URL/Alert groups) + Product repo->>Runner: CI Runs huge load test + Runner->>K8s: Split workload into multiple jobs
Monitor jobs statuses + K8s->>Loki: Stream load test data + Runner->>Grafana: Checking "requirement_name": "stress" alerts + Grafana->>Devs: Notify devs (Dashboard URL/Alert groups) + end +``` + +Example cluster component diagram: +```mermaid +--- +title: Workload execution. P - Profile, G - Generator, VU - VirtualUser +--- +flowchart TB + ClusterProfile-- generate k8s manifests/deploy/await jobs completion -->P1 + ClusterProfile-->PN + ClusterProfile-- check NFRs -->Grafana + subgraph Pod1 + P1-->P1-G1 + P1-->P1-GN + P1-G1-->P1-G1-VU1 + P1-G1-->P1-G1-VUN + P1-GN-->P1-GN-VU1 + P1-GN--->P1-GN-VUN + + P1-G1-VU1-->P1-Batch + P1-G1-VUN-->P1-Batch + P1-GN-VU1-->P1-Batch + P1-GN-VUN-->P1-Batch + end + subgraph PodN + PN-->PN-G1 + PN-->PN-GN + PN-G1-->PN-G1-VU1 + PN-G1-->PN-G1-VUN + PN-GN-->PN-GN-VU1 + PN-GN--->PN-GN-VUN + + PN-G1-VU1-->PN-Batch + PN-G1-VUN-->PN-Batch + PN-GN-VU1-->PN-Batch + PN-GN-VUN-->PN-Batch + + end + P1-Batch-->Loki + PN-Batch-->Loki + + Loki-->Grafana + +``` + +## Defining NFRs and checking alerts +You can define different non-functional requirements groups +In this example we have 2 groups: +- `baseline` - checking both 99th latencies per `Generator` and errors +- `stress` - checking only errors + +`WaspAlerts` can be defined on default `Generators` metrics, for each alert additional row is generated + +`CustomAlerts` can be defined as [timeseries.Alert](https://pkg.go.dev/github.com/K-Phoen/grabana@v0.21.18/timeseries#Alert) but timeseries won't be included, though `AlertChecker` will check them + +Run 2 tests, change mock latency/status codes to see how it works + +Alert definitions usually defined with your `dashboard` and then constantly updated on each Git commit by your CI + +After each run `AlertChecker` will fail the test if any alert from selected group was raised +- [definitions](https://github.com/smartcontractkit/wasp/blob/master/examples/alerts/main_test.go#L37) +- [wasp alerts](https://github.com/smartcontractkit/wasp/blob/master/examples/alerts/main_test.go#L73) +- [custom alerts](https://github.com/smartcontractkit/wasp/blob/master/examples/alerts/main_test.go#L82) +- [baseline NFR group test](https://github.com/smartcontractkit/wasp/blob/master/examples/alerts/main_test.go#L115) +- [stress NFR group test](https://github.com/smartcontractkit/wasp/blob/master/examples/alerts/main_test.go#L143) +``` +cd examples/alerts +go test -v -count 1 -run TestBaselineRequirements +go test -v -count 1 -run TestStressRequirements +``` +Open [alert groups](http://localhost:3000/alerting/groups) + +Check [dashboard](http://localhost:3000/d/wasp/wasp-load-generator?orgId=1&refresh=5s&var-go_test_name=All&var-gen_name=All&var-branch=generator_healthcheck&var-commit=generator_healthcheck&from=now-5m&to=now), you can see per alert timeseries in the bottom + +## Cluster test with k8s +`Warning`: we don't have Loki + Grafana k8s setup yet, if you have them in your `k8s` set up you can run this test + +Cluster mode [overview](examples/CLUSTER.md) + +You may also need to set your `LOKI_TOKEN` env var, depends on your authorization + +Your `k8s context` should be set up to work with `kubectl` + +Set up your namespace with role/rolebinding to be able to run tests: +``` +cd charts/wasp +kubectl create ns wasp +kubectl -n wasp apply -f setup.yaml +``` +You can build your tests like in example `Dockerfile` in the root dir +``` +docker build -f Dockerfile.test --build-arg BUILD_ROOT=/go/src/examples/cluster -t wasp_test . +docker tag wasp_test:latest ${registry}/wasp_test:latest +docker push ${registry}/wasp_test:latest +``` + +Then run an example test: +``` +cd examples/cluster +go test -v -count 1 -run TestClusterScenario . +``` + +- [cluster test](https://github.com/smartcontractkit/wasp/blob/master/examples/cluster/cluster_test.go#L11) +- [test](https://github.com/smartcontractkit/wasp/blob/master/examples/cluster/node_test.go#L14) +- [vu](https://github.com/smartcontractkit/wasp/blob/master/examples/cluster/vu.go#L70) + +Open [dashboard](http://localhost:3000/d/wasp/wasp-load-generator?orgId=1&refresh=5s) + +## How to choose RPS vs VU workload +Pick `Gun` if: +- You need to figure out if system can respond to some limited workload +- You have a stateless protocol + +Pick `VU` if: +- You need to simulate some client behaviour with user wait time +- You need to execute more than 1 request in a `Call` +- Your protocol is stateful, and you need to test connections or keep some state + +Differences between `Gun` and `VU` entities: +- `Gun` should perform 1 call, elapsed time is measured automatically, RPS is limited +- `VU` can perform multiple calls, elapsed time is **not measured** automatically, implementation of `VU` should care about time measurement and rate limiting diff --git a/wasp/LICENSE b/wasp/LICENSE new file mode 100644 index 000000000..3682fb80f --- /dev/null +++ b/wasp/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2023 SmartContract ChainLink, Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/wasp/Makefile b/wasp/Makefile new file mode 100644 index 000000000..b4833b8b2 --- /dev/null +++ b/wasp/Makefile @@ -0,0 +1,53 @@ +.PHONY: test +test: + go test -v -count 1 `go list ./... | grep -v examples` -run TestSmoke + +.PHONY: test_race +test_race: + go test -v -race -count 1 `go list ./... | grep -v examples` -run TestSmoke + +.PHONY: test_bench +test_bench: + go test -bench=. -benchmem -count 1 -run=^# + +.PHONY: test+cover +test_cover: + go test -v -coverprofile cover.out -count 1 `go list ./... | grep -v examples` -run TestSmoke + go tool cover -html cover.out + +.PHONY: test +test_loki: + go test -v -count 1 `go list ./... | grep -v examples` -run TestPerfRenderLoki + +.PHONY: test +test_pyro_rps: + go test -v -run TestPyroscopeLocalTraceRPSCalls -trace trace.out + +.PHONY: test +test_pyro_vu: + go test -v -run TestPyroscopeLocalTraceVUCalls -trace trace.out + +.PHONY: dashboard +dashboard: + go run dashboard/cmd/main.go + +.PHONY: start +start: + docker compose -f compose/docker-compose.yaml up -d + sleep 5 && curl -X POST -H "Content-Type: application/json" -d '{"name":"test", "role": "Admin"}' http://localhost:3000/api/auth/keys | jq .key + +.PHONY: stop +stop: + docker compose -f compose/docker-compose.yaml down -v + +.PHONY: pyro_start +pyro_start: + docker compose -f compose/pyroscope-compose.yaml up -d + +.PHONY: pyro_stop +pyro_stop: + docker compose -f compose/pyroscope-compose.yaml down -v + +.PHONY: lint +lint: + golangci-lint --color=always run -v diff --git a/wasp/README.md b/wasp/README.md new file mode 100644 index 000000000..948943d75 --- /dev/null +++ b/wasp/README.md @@ -0,0 +1,171 @@ +

+ wasp +

+
+ +[![Go Report Card](https://goreportcard.com/badge/github.com/smartcontractkit/wasp)](https://goreportcard.com/report/github.com/smartcontractkit/wasp) +[![Component Tests](https://github.com/smartcontractkit/chainlink-testing-framework/actions/workflows/wasp-test.yml/badge.svg)](https://github.com/smartcontractkit/chainlink-testing-framework/actions/workflows/wasp-test.yml) +[![E2E tests](https://github.com/smartcontractkit/chainlink-testing-framework/actions/workflows/wasp-test-e2e.yml/badge.svg)](https://github.com/smartcontractkit/chainlink-testing-framework/actions/workflows/wasp-test-e2e.yml) +![gopherbadger-tag-do-not-edit](https://img.shields.io/badge/Go%20Coverage-80%25-brightgreen.svg?longCache=true&style=flat) + +Scalable protocol-agnostic load testing library for `Go` + +
+ +## Goals +- Easy to reuse any custom client `Go` code +- Easy to grasp +- Have a slim codebase (500-1k loc) +- No test harness or CLI, easy to integrate and run with plain `go test` +- Have a predictable performance footprint +- Easy to create synthetic or user-based scenarios +- Scalable in `k8s` without complicated configuration or vendored UI interfaces +- Non-opinionated reporting, push any data to `Loki` + +## Setup +We are using `nix` for deps, see [installation](https://nixos.org/manual/nix/stable/installation/installation.html) guide +```bash +nix develop +``` + + +## Run example tests with Grafana + Loki +```bash +make start +``` +Insert `GRAFANA_TOKEN` created in previous command +```bash +export LOKI_TOKEN= +export LOKI_URL=http://localhost:3030/loki/api/v1/push +export GRAFANA_URL=http://localhost:3000 +export GRAFANA_TOKEN= +export DATA_SOURCE_NAME=Loki +export DASHBOARD_FOLDER=LoadTests +export DASHBOARD_NAME=Wasp + +make dashboard +``` +Run some tests: +``` +make test_loki +``` +Open your [Grafana dashboard](http://localhost:3000/d/wasp/wasp-load-generator?orgId=1&refresh=5s) + +In case you deploy to your own Grafana check `DASHBOARD_FOLDER` and `DASHBOARD_NAME`, defaults are `LoadTests` dir and dashboard is called `Wasp` + +Remove environment: +```bash +make stop +``` + +## Test Layout and examples +Check [examples](examples/README.md) to understand what is the easiest way to structure your tests, run them both locally and remotely, at scale, inside `k8s` + +## Run pyroscope test +``` +make pyro_start +make test_pyro_rps +make test_pyro_vu +make pyro_stop +``` +Open [pyroscope](http://localhost:4040/) + +You can also use `trace.out` in the root folder with `Go` default tracing UI + +## How it works +![img.png](docs/how-it-works.png) + +Check this [doc](./HOW_IT_WORKS.md) for more examples and project overview + +## Loki debug +You can check all the messages the tool sends with env var `WASP_LOG_LEVEL=trace` + +If Loki client fail to deliver a batch test will proceed, if you experience Loki issues, consider setting `Timeout` in `LokiConfig` or set `MaxErrors: 10` to return an error after N Loki errors + +`MaxErrors: -1` can be used to ignore all the errors + +Default Promtail settings are: +``` +&LokiConfig{ + TenantID: os.Getenv("LOKI_TENANT_ID"), + URL: os.Getenv("LOKI_URL"), + Token: os.Getenv("LOKI_TOKEN"), + BasicAuth: os.Getenv("LOKI_BASIC_AUTH"), + MaxErrors: 10, + BatchWait: 5 * time.Second, + BatchSize: 500 * 1024, + Timeout: 20 * time.Second, + DropRateLimitedBatches: false, + ExposePrometheusMetrics: false, + MaxStreams: 600, + MaxLineSize: 999999, + MaxLineSizeTruncate: false, +} +``` +If you see errors like +``` +ERR Malformed promtail log message, skipping Line=["level",{},"component","client","host","...","msg","batch add err","tenant","","error",{}] +``` +Try to increase `MaxStreams` even more or check your `Loki` configuration + + +## WASP Dashboard + +Basic [dashboard](dashboard/dashboard.go): + +![dashboard_img](./docs/dashboard_basic.png) + +### Reusing Dashboard Components + +You can integrate components from the WASP dashboard into your custom dashboards. + +Example: + +``` +import ( + waspdashboard "github.com/smartcontractkit/wasp/dashboard" +) + +func BuildCustomLoadTestDashboard(dashboardName string) (dashboard.Builder, error) { + // Custom key,value used to query for panels + panelQuery := map[string]string{ + "branch": `=~"${branch:pipe}"`, + "commit": `=~"${commit:pipe}"`, + "network_type": `="testnet"`, + } + + return dashboard.New( + dashboardName, + waspdashboard.WASPLoadStatsRow("Loki", panelQuery), + waspdashboard.WASPDebugDataRow("Loki", panelQuery, true), + # other options + ) +} +``` + +## Annotate Dashboards and Monitor Alerts + +To enable dashboard annotations and alert monitoring, utilize the `WithGrafana()` function in conjunction with `wasp.Profile`. This approach allows for the integration of dashboard annotations and the evaluation of dashboard alerts. + +Example: + +``` +_, err = wasp.NewProfile(). + WithGrafana(grafanaOpts). + Add(wasp.NewGenerator(getLatestReportByTimestampCfg)). + Run(true) +require.NoError(t, err) +``` + +Where: + +``` +type GrafanaOpts struct { + GrafanaURL string `toml:"grafana_url"` + GrafanaToken string `toml:"grafana_token_secret"` + WaitBeforeAlertCheck time.Duration `toml:"grafana_wait_before_alert_check"` // Cooldown period to wait before checking for alerts + AnnotateDashboardUIDs []string `toml:"grafana_annotate_dashboard_uids"` // Grafana dashboardUIDs to annotate start and end of the run + CheckDashboardAlertsAfterRun []string `toml:"grafana_check_alerts_after_run_on_dashboard_uids"` // Grafana dashboardIds to check for alerts after run +} + +``` diff --git a/wasp/SECURITY.md b/wasp/SECURITY.md new file mode 100644 index 000000000..b0ee4c04d --- /dev/null +++ b/wasp/SECURITY.md @@ -0,0 +1,11 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 0.1.x | :white_check_mark: | + +## Reporting a Vulnerability + +Open an issue, we'll review it and get back to you promptly. diff --git a/wasp/alert.go b/wasp/alert.go new file mode 100644 index 000000000..04bbd5bfe --- /dev/null +++ b/wasp/alert.go @@ -0,0 +1,104 @@ +package wasp + +import ( + "fmt" + "os" + "sort" + "strings" + "testing" + "time" + + "github.com/pkg/errors" + "github.com/smartcontractkit/chainlink-testing-framework/grafana" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +// AlertChecker is checking alerts according to dashboardUUID and requirements labels +type AlertChecker struct { + RequirementLabelKey string + T *testing.T + l zerolog.Logger + grafanaClient *grafana.Client +} + +func NewAlertChecker(t *testing.T) *AlertChecker { + url := os.Getenv("GRAFANA_URL") + if url == "" { + panic(fmt.Errorf("GRAFANA_URL env var must be defined")) + } + apiKey := os.Getenv("GRAFANA_TOKEN") + if apiKey == "" { + panic(fmt.Errorf("GRAFANA_TOKEN env var must be defined")) + } + + grafanaClient := grafana.NewGrafanaClient(url, apiKey) + + return &AlertChecker{ + RequirementLabelKey: "requirement_name", + T: t, + grafanaClient: grafanaClient, + l: GetLogger(t, "AlertChecker"), + } +} + +// AnyAlerts check if any alerts with dashboardUUID have been raised +func (m *AlertChecker) AnyAlerts(dashboardUUID, requirementLabelValue string) ([]grafana.AlertGroupsResponse, error) { + raised := false + defer func() { + if m.T != nil && raised { + m.T.Fail() + } + }() + alertGroups, _, err := m.grafanaClient.AlertManager.GetAlertGroups() + if err != nil { + return alertGroups, fmt.Errorf("failed to get alert groups: %s", err) + } + for _, a := range alertGroups { + for _, aa := range a.Alerts { + log.Debug().Interface("Alert", aa).Msg("Scanning alert") + if aa.Annotations.DashboardUID == dashboardUUID && aa.Labels[m.RequirementLabelKey] == requirementLabelValue { + log.Warn(). + Str("Summary", aa.Annotations.Summary). + Str("Description", aa.Annotations.Description). + Str("URL", aa.GeneratorURL). + Interface("Labels", aa.Labels). + Time("StartsAt", aa.StartsAt). + Time("UpdatedAt", aa.UpdatedAt). + Str("State", aa.Status.State). + Msg("Alert fired") + raised = true + } + } + } + return alertGroups, nil +} + +// CheckDashobardAlerts checks for alerts in the given dashboardUUIDs between from and to times +func CheckDashboardAlerts(grafanaClient *grafana.Client, from, to time.Time, dashboardUID string) ([]grafana.Annotation, error) { + annotationType := "alert" + alerts, _, err := grafanaClient.GetAnnotations(grafana.AnnotationsQueryParams{ + DashboardUID: &dashboardUID, + From: &from, + To: &to, + Type: &annotationType, + }) + if err != nil { + return alerts, fmt.Errorf("could not check for alerts: %s", err) + } + + // Sort the annotations by time oldest to newest + sort.Slice(alerts, func(i, j int) bool { + return alerts[i].Time.Before(alerts[j].Time.Time) + }) + + // Check if any alerts are in alerting state + for _, a := range alerts { + if strings.ToLower(a.NewState) == "alerting" { + return alerts, errors.New("at least one alert was firing") + } + } + + return alerts, nil +} diff --git a/wasp/buffer.go b/wasp/buffer.go new file mode 100644 index 000000000..ae9ccdc8c --- /dev/null +++ b/wasp/buffer.go @@ -0,0 +1,26 @@ +package wasp + +// SliceBuffer keeps Capacity of type T, after len => cap overrides old data +type SliceBuffer[T any] struct { + Idx int + Capacity int + Data []T +} + +// NewSliceBuffer creates new limited capacity slice +func NewSliceBuffer[T any](cap int) *SliceBuffer[T] { + return &SliceBuffer[T]{Capacity: cap, Data: make([]T, 0)} +} + +// Append appends T if len <= cap, overrides old data otherwise +func (m *SliceBuffer[T]) Append(s T) { + if m.Idx >= m.Capacity { + m.Idx = 0 + } + if len(m.Data) <= m.Capacity { + m.Data = append(m.Data, s) + } else { + m.Data[m.Idx] = s + } + m.Idx++ +} diff --git a/wasp/charts/wasp/.helmignore b/wasp/charts/wasp/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/wasp/charts/wasp/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/wasp/charts/wasp/Chart.yaml b/wasp/charts/wasp/Chart.yaml new file mode 100644 index 000000000..2154f204f --- /dev/null +++ b/wasp/charts/wasp/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: wasp +description: Wasp cluster test +type: application +version: 0.1.8 +appVersion: "0.1.8" diff --git a/wasp/charts/wasp/namespace_setup/setup.yaml b/wasp/charts/wasp/namespace_setup/setup.yaml new file mode 100644 index 000000000..20f3f6c39 --- /dev/null +++ b/wasp/charts/wasp/namespace_setup/setup.yaml @@ -0,0 +1,25 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: wasp-role +rules: + - apiGroups: + - "" + - "apps" + - "batch" + resources: + - "*" + verbs: + - "*" +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: wasp-role +subjects: + - kind: ServiceAccount + name: default +roleRef: + kind: Role + name: wasp-role + apiGroup: rbac.authorization.k8s.io diff --git a/wasp/charts/wasp/templates/_helpers.tpl b/wasp/charts/wasp/templates/_helpers.tpl new file mode 100644 index 000000000..0ecdf99d2 --- /dev/null +++ b/wasp/charts/wasp/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "wasp.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "wasp.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "wasp.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "wasp.labels" -}} +helm.sh/chart: {{ include "wasp.chart" . }} +{{ include "wasp.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "wasp.selectorLabels" -}} +app.kubernetes.io/name: {{ include "wasp.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "wasp.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "wasp.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/wasp/charts/wasp/templates/job.yaml b/wasp/charts/wasp/templates/job.yaml new file mode 100644 index 000000000..77ca98e8c --- /dev/null +++ b/wasp/charts/wasp/templates/job.yaml @@ -0,0 +1,69 @@ +{{- range $i, $e := until (int .Values.jobs) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: wasp-{{ $.Release.Name }}-{{ $i }} + labels: + sync: "{{ $.Values.sync }}" +spec: + backoffLimit: 0 + template: + metadata: + name: wasp-{{ $.Release.Name }}-{{ $i }} + {{- with $.Values.labels }} + labels: + {{- toYaml . | nindent 8 }} + {{- end }} + sync: {{ $.Values.sync }} + {{- with $.Values.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + restartPolicy: Never + containers: + - name: wasp + image: {{ $.Values.image }} + command: + - ./{{ $.Values.test.binaryName }} + - -test.v + - -test.run + - {{ $.Values.test.name }} + - -test.timeout + - {{ $.Values.test.timeout }} + imagePullPolicy: {{ $.Values.imagePullPolicy }} + {{- with $.Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: LOKI_URL + value: {{ $.Values.env.loki.url }} + - name: LOKI_TOKEN + value: {{ $.Values.env.loki.token }} + - name: LOKI_BASIC_AUTH + value: {{ $.Values.env.loki.basic_auth }} + - name: LOKI_TENANT_ID + value: {{ $.Values.env.loki.tenant_id }} + - name: WASP_LOG_LEVEL + value: {{ $.Values.env.wasp.log_level }} + - name: WASP_NODE_ID + value: {{ $i | quote }} + - name: WASP_NAMESPACE + value: {{ $.Values.namespace }} + - name: WASP_SYNC + value: {{ $.Values.sync }} + - name: WASP_JOBS + value: {{ $.Values.jobs | quote }} + {{- range $key, $value := $.Values.test }} + {{- if $value }} + - name: {{ $key | upper}} + {{- if kindIs "string" $value}} + value: {{ $value | quote}} + {{- else }} + value: {{ $value }} + {{- end }} + {{- end }} + {{- end }} +--- +{{- end }} diff --git a/wasp/charts/wasp/values.yaml b/wasp/charts/wasp/values.yaml new file mode 100644 index 000000000..4bb7e288d --- /dev/null +++ b/wasp/charts/wasp/values.yaml @@ -0,0 +1,35 @@ +namespace: wasp +# amount of jobs to spin up +jobs: 1 +# a label jobs will use to sync before starting, a random 5-digit number by default +sync: +# Go test name and timeout +test: + name: "" + timeout: "24h" + binaryName: "" + +# image + tag string - ${IMAGE}:${TAG} format +image: public.ecr.aws/chainlink/wasp-test:latest +imagePullPolicy: Always +labels: + app: wasp +annotations: {} +env: + wasp: + log_level: info + loki: + basic_auth: "" + tenant_id: "" + token: "" + url: "" +resources: + requests: + cpu: 1000m + memory: 512Mi + limits: + cpu: 1000m + memory: 512Mi +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/wasp/charts/wasp/wasp-0.1.8.tgz b/wasp/charts/wasp/wasp-0.1.8.tgz new file mode 100644 index 0000000000000000000000000000000000000000..818d21a9f8e6ca7c6f71d5d856d0f738c28f31ad GIT binary patch literal 2135 zcmV-d2&nfTiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PI>{Z`(K$@3TI|oRbCGY%SY~lPv_?gPR7uY}2HuyDb)ngF#6n zn>9tMB<0lW=05v@q-0C7oj-T|gC0B&j!X`JoEdS3Gviznm32-hL~4IdQugRuP`BId zp7i_nZ@1fR{_XaT`;U74F>i*ArHI@5Gh-CCV zN<{?^V0P${EL(o<`aS=d7o&ty|ZMp%yOn+1S7{J+;b={5bo-|OxC|9g<}UzL%lbmBcAsmQqok-$e0 zD$oK{Mmgk}XTAr}Gd#dZh|FRM7KoTB{6pq_59a9ZF zPeYUt&LPHx~BQ zFG~(@4`@op2)}|RV^CUBK87~zFV0^NUY^|q`-^vjmp70InG)?;-T<;ZWHj-Wc1`BWv$j!2SdwLhT?ywx+o-4lSloEX zXNJMdnZE!`j7JP-$O51|5uO>DQa2Y8MWYeP^`y`oKqDtyk7&Fa5mV&LZ@FalS)vlT zjL?cB@l%dUD~B3oc>uj`x0~9pDW*cs132y-zNTh-n$ml*auMSY8Ae)2tD6=KWvS~w z|K*V+p`7Zuo&3=Uen&O3@n2(_*)!c4O^{_Mm9Mj}cNN?b|BsH2x{dhXKklFG;{W%c z#iDcIoqf)1sGXpTyCSjAku3|ngU-#3w^+1Itfm~HWzLLm7+cU@O_?wbzAGh62nPSX zODhcL)eNPiF~S$nGUpMT9N9;j4)Y|TpP|)mL4k!lHjkba?NlPs2xPfKk~21kPdQ;U zp%|MGYlZf`59qKhzcxD<1r&rB5pz8$ks`dxLzEmfDyM2fk;Mv#L>KTdHrJS*Pz8^} zxi!O!%c0>lcZbR_dg6QM2}oo}RAH}3a7`#z&SGwiYIiY)Xzs=shyr4TFQnaURl`uX zEKQb%EGL)7ELgVA-^bWYEphg(;KAST|29`LE=}M-{I||NZfz-NZSaA@Ws-88Kq%pRLW0wG&r)mBu%UQpUVS`#OuW;kfLDzwMP;meu=g>QObZdoBtv_yQ6$ zMk0jPuPtbeS`T*~k*0z>@zfUa+@zClf-Lpbq+>-6V6gzoBbLVvReZNuV^S3rvpym0b3Osk|?q{ycXuLZdJxI z4;=61wxCmI^CP!&BkVt}tGbF}{*!G$ysBHp&r8Tw@zA;#z?v`r=zH~Np+^4qQG{Q= z4RA;P*Y6!3H}b!ulau3J{`Wn|xQrxxjId9i!oI!!bFL{fKCPF&XA1o%E_AK>g(1<= zq%-SzQ_AB2eixyaq9!rXWL2r#s(G=1{bp3NgVbOGtSV-(ErbACc1h{^a~{FXP0KU+ zm*EOYG!;p5VKTM=x}K?r-4PCo0(B*Zp$CX!$Qb3C>d8t1C*X>sDsZ_%i$6)qz=tou zDUXqBcvi?=j#@C?vEudus z>JEs4t&XzRAzCxWYO=m^{oOA+Id#TbgQvcJ!~5?Yu5Z&gl>kL=s;aJg!53Gr&PRW| zy{JtB*!FQ&)i7EAS$<_^9L3?-g>g@6Xc(%)RE8_FP@ceW%_~YF_9ss-Qu-z{0 z+2!E!-RS(qLnJOcb=#0X42Ew;7gsMw7iaI!?rB?>x|c1(TahoXUYy-3F|B6S+hGQ; z&xUUXr)PHyxa=w43OM}p^7L-3dPLj+^ZV8F;oUH{QN2;k$^|&Z`BT`ptIai7^Y5k% zMV4G}ZxPT)FvU51fjrAlHn^4`=K4HW&~gpxRsp)kT~W*JTIjP8woIw-fY`jY84$J% zR|ebdb|w4%(MipY?oBrT_f6qX$6QGK7JJ-6uED$gHN-Laoee9sU6k{$b54ot=!^HF|PGu~MNnPMW(AfXJEci^k*%scQyGW8C8 zotkGM>DW8?x3>rHiKHS|aQ@;<%K;` zsPn92Kfhid;4c3^Ir^>P|Hu8KUH$)E$d&GI1;bitFGCXfCLI=%{$(B3pFVRv+TOMz zUm6gZGgpeVfn@Zh6nVC+Oxop-)Ao^MnW}uX%>?&-t)yu~>%dLTP=<|RlX|SQcuu(q zT3<&Qm50W`3w_)s!&(>8O5&E4|H)uhzQx3>sTg-19F_*UMZzC_Ja)9B9sM8Ee*ypi N|NrN%8>av)001 0") +) + +// ClusterConfig defines k8s jobs settings +type ClusterConfig struct { + ChartPath string + Namespace string + KeepJobs bool + UpdateImage bool + DockerCmdExecPath string + DockerfilePath string + DockerIgnoreFilePath string + BuildScriptPath string + BuildCtxPath string + ImageTag string + RegistryName string + RepoName string + HelmDeployTimeoutSec string + HelmValues map[string]string + // generated values + tmpHelmFilePath string +} + +func (m *ClusterConfig) Defaults() error { + // TODO: will it be more clear if we move Helm values to a struct + // TODO: or should it be like that for extensibility of a chart without reflection? + m.HelmValues["namespace"] = m.Namespace + // nolint + m.HelmValues["sync"] = fmt.Sprintf("a%s", uuid.NewString()[0:5]) + if m.HelmDeployTimeoutSec == "" { + m.HelmDeployTimeoutSec = defaultHelmDeployTimeoutSec + } + if m.HelmValues["test.timeout"] == "" { + m.HelmValues["test.timeout"] = "12h" + } + if m.HelmValues["resources.requests.cpu"] == "" { + m.HelmValues["resources.requests.cpu"] = DefaultRequestsCPU + } + if m.HelmValues["resources.requests.memory"] == "" { + m.HelmValues["resources.requests.memory"] = DefaultRequestsMemory + } + if m.HelmValues["resources.limits.cpu"] == "" { + m.HelmValues["resources.limits.cpu"] = DefaultLimitsCPU + } + if m.HelmValues["resources.limits.memory"] == "" { + m.HelmValues["resources.limits.memory"] = DefaultLimitsMemory + } + if m.ChartPath == "" { + log.Info().Msg("Using default embedded chart") + if err := os.WriteFile(defaultArchiveName, defaultChart, os.ModePerm); err != nil { + return err + } + m.tmpHelmFilePath, m.ChartPath = defaultArchiveName, defaultArchiveName + } + if m.DockerfilePath == "" { + log.Info().Msg("Using default embedded DockerfileWasp") + if err := os.WriteFile(defaultDockerfilePath, DefaultDockerfile, os.ModePerm); err != nil { + return err + } + p, err := filepath.Abs(defaultDockerfilePath) + if err != nil { + return err + } + m.DockerfilePath = p + } + if m.DockerIgnoreFilePath == "" { + log.Info().Msg("Using default embedded DockerfileWasp.dockerignore") + if err := os.WriteFile(defaultDockerfileIgnorePath, DefaultDockerIgnorefile, os.ModePerm); err != nil { + return err + } + p, err := filepath.Abs(defaultDockerfileIgnorePath) + if err != nil { + return err + } + m.DockerIgnoreFilePath = p + } + if m.BuildScriptPath == "" { + log.Info().Msg("Using default build script") + fname := strings.Replace(defaultBuildScriptPath, "./", "", -1) + if err := os.WriteFile(fname, DefaultBuildScript, os.ModePerm); err != nil { + return err + } + m.BuildScriptPath = defaultBuildScriptPath + } + return nil +} + +func (m *ClusterConfig) Validate() (err error) { + if m.Namespace == "" { + err = errors.Join(err, ErrNoNamespace) + } + if m.HelmValues["jobs"] == "" { + err = errors.Join(err, ErrNoJobs) + } + return +} + +// parseECRImageURI parses the ECR image URI and returns its components +func parseECRImageURI(uri string) (registry, repo, tag string, err error) { + re := regexp.MustCompile(`^([^/]+)/([^:]+):(.+)$`) + matches := re.FindStringSubmatch(uri) + if len(matches) != 4 { + return "", "", "", fmt.Errorf("invalid ECR image URI format, must be ${registry}/${repo}:${tag}") + } + return matches[1], matches[2], matches[3], nil +} + +// ClusterProfile is a k8s cluster test for some workload profile +type ClusterProfile struct { + cfg *ClusterConfig + c *K8sClient + Ctx context.Context + Cancel context.CancelFunc +} + +// NewClusterProfile creates new cluster profile +func NewClusterProfile(cfg *ClusterConfig) (*ClusterProfile, error) { + if err := cfg.Validate(); err != nil { + return nil, err + } + if err := cfg.Defaults(); err != nil { + return nil, err + } + log.Info().Interface("Config", cfg).Msg("Cluster configuration") + dur, err := time.ParseDuration(cfg.HelmValues["test.timeout"]) + if err != nil { + return nil, fmt.Errorf("failed to parse test timeout duration") + } + ctx, cancelFunc := context.WithTimeout(context.Background(), dur) + cp := &ClusterProfile{ + cfg: cfg, + c: NewK8sClient(), + Ctx: ctx, + Cancel: cancelFunc, + } + if cp.cfg.UpdateImage { + return cp, cp.buildAndPushImage() + } + return cp, nil +} + +func (m *ClusterProfile) buildAndPushImage() error { + registry, repo, tag, err := parseECRImageURI(m.cfg.HelmValues["image"]) + if err != nil { + return err + } + cmd := fmt.Sprintf("%s %s %s %s %s %s %s", + m.cfg.BuildScriptPath, + m.cfg.DockerfilePath, + m.cfg.BuildCtxPath, + tag, + registry, + repo, + m.cfg.DockerCmdExecPath, + ) + log.Info().Str("Cmd", cmd).Msg("Building docker") + return ExecCmd(cmd) +} + +func (m *ClusterProfile) deployHelm(testName string) error { + //nolint + defer os.Remove(m.cfg.tmpHelmFilePath) + var cmd strings.Builder + cmd.WriteString(fmt.Sprintf("helm install %s %s", testName, m.cfg.ChartPath)) + for k, v := range m.cfg.HelmValues { + cmd.WriteString(fmt.Sprintf(" --set %s=%s", k, v)) + } + cmd.WriteString(fmt.Sprintf(" -n %s", m.cfg.Namespace)) + cmd.WriteString(fmt.Sprintf(" --timeout %s", m.cfg.HelmDeployTimeoutSec)) + log.Info().Str("Cmd", cmd.String()).Msg("Deploying jobs") + return ExecCmd(cmd.String()) +} + +// Run starts a new test +func (m *ClusterProfile) Run() error { + testName := uuid.NewString()[0:8] + tn := []rune(testName) + // replace first letter, since helm does not allow it to start with numbers + tn[0] = 'a' + if err := m.deployHelm(string(tn)); err != nil { + return err + } + jobNum, err := strconv.Atoi(m.cfg.HelmValues["jobs"]) + if err != nil { + return err + } + return m.c.TrackJobs(m.Ctx, m.cfg.Namespace, m.cfg.HelmValues["sync"], jobNum, m.cfg.KeepJobs) +} diff --git a/wasp/cmd.go b/wasp/cmd.go new file mode 100644 index 000000000..ddedbf238 --- /dev/null +++ b/wasp/cmd.go @@ -0,0 +1,49 @@ +package wasp + +import ( + "bufio" + "io" + "os/exec" + "strings" + + "github.com/rs/zerolog/log" +) + +// ExecCmd executes os command, logging both streams +func ExecCmd(command string) error { + return ExecCmdWithStreamFunc(command, func(m string) { + log.Info().Str("Text", m).Msg("Command output") + }) +} + +// readStdPipe continuously read a pipe from the command +func readStdPipe(pipe io.ReadCloser, streamFunc func(string)) { + scanner := bufio.NewScanner(pipe) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + m := scanner.Text() + if streamFunc != nil { + streamFunc(m) + } + } +} + +// ExecCmdWithStreamFunc executes command with stream function +func ExecCmdWithStreamFunc(command string, outputFunction func(string)) error { + c := strings.Split(command, " ") + cmd := exec.Command(c[0], c[1:]...) + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { + return err + } + go readStdPipe(stderr, outputFunction) + go readStdPipe(stdout, outputFunction) + return cmd.Wait() +} diff --git a/wasp/compose/conf/defaults.ini b/wasp/compose/conf/defaults.ini new file mode 100644 index 000000000..b2034acc9 --- /dev/null +++ b/wasp/compose/conf/defaults.ini @@ -0,0 +1,1238 @@ +##################### Grafana Configuration Defaults ##################### +# +# Do not modify this file in grafana installs +# + +# possible values : production, development +app_mode = production + +# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty +instance_name = ${HOSTNAME} + +# force migration will run migrations that might cause dataloss +force_migration = false + +#################################### Paths ############################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +data = data + +# Temporary files in `data` directory older than given duration will be removed +temp_data_lifetime = 24h + +# Directory where grafana can store logs +logs = data/log + +# Directory where grafana will automatically scan and look for plugins +plugins = data/plugins + +# folder that contains provisioning config files that grafana will apply on startup and while running. +provisioning = conf/provisioning + +#################################### Server ############################## +[server] +# Protocol (http, https, h2, socket) +protocol = http + +# The ip address to bind to, empty will bind to all interfaces +http_addr = + +# The http port to use +http_port = 3000 + +# The public facing domain name used to access grafana from a browser +domain = localhost + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +enforce_domain = false + +# The full public facing url +root_url = %(protocol)s://%(domain)s:%(http_port)s/ + +# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. +serve_from_sub_path = false + +# Log web requests +router_logging = false + +# the path relative working path +static_root_path = public + +# enable gzip +enable_gzip = false + +# https certs & key file +cert_file = +cert_key = + +# Unix socket path +socket = /tmp/grafana.sock + +# CDN Url +cdn_url = + +# Sets the maximum time in minutes before timing out read of an incoming request and closing idle connections. +# `0` means there is no timeout for reading the request. +read_timeout = 0 + +#################################### Database ############################ +[database] +# You can configure the database connection by specifying type, host, name, user and password +# as separate properties or as on string using the url property. + +# Either "mysql", "postgres" or "sqlite3", it's your choice +type = sqlite3 +host = 127.0.0.1:3306 +name = grafana +user = root +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +password = +# Use either URL or the previous fields to configure the database +# Example: mysql://user:secret@host:port/database +url = + +# Max idle conn setting default is 2 +max_idle_conn = 2 + +# Max conn setting default is 0 (mean not set) +max_open_conn = + +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +conn_max_lifetime = 14400 + +# Set to true to log the sql calls and execution times. +log_queries = + +# For "postgres", use either "disable", "require" or "verify-full" +# For "mysql", use either "true", "false", or "skip-verify". +ssl_mode = disable + +# Database drivers may support different transaction isolation levels. +# Currently, only "mysql" driver supports isolation levels. +# If the value is empty - driver's default isolation level is applied. +# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE". +isolation_level = + +ca_cert_path = +client_key_path = +client_cert_path = +server_cert_name = + +# For "sqlite3" only, path relative to data_path setting +path = grafana.db + +# For "sqlite3" only. cache mode setting used for connecting to the database +cache_mode = private + +# For "mysql" only if lockingMigration feature toggle is set. How many seconds to wait before failing to lock the database for the migrations, default is 0. +locking_attempt_timeout_sec = 0 + +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +type = database + +# cache connectionstring options +# database: will use Grafana primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. +# memcache: 127.0.0.1:11211 +connstr = + +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +logging = false + +# How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds. +# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. +timeout = 30 + +# How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds. +dialTimeout = 10 + +# How many seconds the data proxy waits before sending a keepalive request. +keep_alive_seconds = 30 + +# How many seconds the data proxy waits for a successful TLS Handshake before timing out. +tls_handshake_timeout_seconds = 10 + +# How many seconds the data proxy will wait for a server's first response headers after +# fully writing the request headers if the request has an "Expect: 100-continue" +# header. A value of 0 will result in the body being sent immediately, without +# waiting for the server to approve. +expect_continue_timeout_seconds = 1 + +# Optionally limits the total number of connections per host, including connections in the dialing, +# active, and idle states. On limit violation, dials will block. +# A value of zero (0) means no limit. +max_conns_per_host = 0 + +# The maximum number of idle connections that Grafana will keep alive. +max_idle_connections = 100 + +# How many seconds the data proxy keeps an idle connection open before timing out. +idle_conn_timeout_seconds = 90 + +# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request. +send_user_header = false + +# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests. +response_limit = 0 + +# Limits the number of rows that Grafana will process from SQL data sources. +row_limit = 1000000 + +#################################### Analytics ########################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +reporting_enabled = true + +# The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs +reporting_distributor = grafana-labs + +# Set to false to disable all checks to https://grafana.com +# for new versions of grafana. The check is used +# in some UI views to notify that a grafana update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://raw.githubusercontent.com/grafana/grafana/main/latest.json to get the latest version. +check_for_updates = true + +# Set to false to disable all checks to https://grafana.com +# for new versions of plugins. The check is used +# in some UI views to notify that a plugin update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com to get the latest versions. +check_for_plugin_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +google_analytics_ua_id = + +# Google Tag Manager ID, only enabled if you specify an id here +google_tag_manager_id = + +# Rudderstack write key, enabled only if rudderstack_data_plane_url is also set +rudderstack_write_key = + +# Rudderstack data plane url, enabled only if rudderstack_write_key is also set +rudderstack_data_plane_url = + +# Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set +rudderstack_sdk_url = + +# Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config +rudderstack_config_url = + +# Application Insights connection string. Specify an URL string to enable this feature. +application_insights_connection_string = + +# Optional. Specifies an Application Insights endpoint URL where the endpoint string is wrapped in backticks ``. +application_insights_endpoint_url = + +# Controls if the UI contains any links to user feedback forms +feedback_links_enabled = true + +#################################### Security ############################ +[security] +# disable creation of admin user on first start of grafana +disable_initial_admin_creation = false + +# default admin user, created on startup +admin_user = admin + +# default admin password, can be changed before first start of grafana, or in profile settings +admin_password = admin + +# used for signing +secret_key = SW2YcwTIb9zpOOhoPsMm + +# current key provider used for envelope encryption, default to static value specified by secret_key +encryption_provider = secretKey.v1 + +# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1 +available_encryption_providers = + +# disable gravatar profile images +disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +data_source_proxy_whitelist = + +# disable protection against brute force login attempts +disable_brute_force_login_protection = false + +# set to true if you host Grafana behind HTTPS. default is false. +cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" +cookie_samesite = lax + +# set to true if you want to allow browsers to render Grafana in a ,