diff --git a/.github/actions/oc-keycloak-login/action.yml b/.github/actions/oc-keycloak-login/action.yml index 0310e46e..4ffb7d31 100644 --- a/.github/actions/oc-keycloak-login/action.yml +++ b/.github/actions/oc-keycloak-login/action.yml @@ -15,3 +15,8 @@ runs: working-directory: provision/aws env: CLUSTER_NAME: ${{ inputs.clusterName }} + + - id: enable-self-describe + shell: bash + working-directory: provision/aws + run: echo "SUT_DESCRIBE=$(pwd)/rosa_describe.sh ${{ inputs.clusterName }}" >> "$GITHUB_ENV" diff --git a/.github/workflows/docs-pages.yml b/.github/workflows/docs-pages.yml index fd6ab5ba..54beb580 100644 --- a/.github/workflows/docs-pages.yml +++ b/.github/workflows/docs-pages.yml @@ -1,12 +1,19 @@ name: Publishing Documentation Site on: + workflow_dispatch: + workflow_run: + workflows: + - "Aggregate Benchmark results" + types: + - completed push: branches: - main paths: - '.github/workflows/docs-pages.yml' - 'antora/**' + - 'dashbuilder/**' - 'doc/**' concurrency: @@ -19,7 +26,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Node.js + - name: Set up Node.js for Antora uses: actions/setup-node@v3 with: node-version: '16.x' @@ -28,6 +35,23 @@ jobs: - name: Build docs working-directory: antora run: ./build.sh + - name: Add benchmark data to website + run: | + mkdir -p antora/_site/keycloak-benchmark/results + gh release download aggregated-benchmarks -D antora/_site/keycloak-benchmark/results --repo keycloak/keycloak-benchmark + env: + GH_TOKEN: ${{ github.token }} + - name: Set up Node.js for DashBuilder + uses: actions/setup-node@v3 + with: + node-version: '16.x' + cache: 'yarn' + cache-dependency-path: 'dashbuilder/yarn.lock' + - name: Prepare dashbuilder + working-directory: dashbuilder + run: | + ./build.sh + mv dist ../antora/_site/keycloak-benchmark/dashbuilder - name: Upload artifact uses: actions/upload-pages-artifact@v1 with: diff --git a/.github/workflows/keycloak-scalability-benchmark.yml b/.github/workflows/keycloak-scalability-benchmark.yml index 89a1815b..bc611618 100644 --- a/.github/workflows/keycloak-scalability-benchmark.yml +++ b/.github/workflows/keycloak-scalability-benchmark.yml @@ -82,7 +82,7 @@ jobs: - name: Build with Maven run: | - ./mvnw -B clean install -DskipTests + ./mvnw -B clean package -DskipTests -pl benchmark tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz mv keycloak-benchmark-* keycloak-benchmark @@ -94,14 +94,14 @@ jobs: disableStickySessions: true - name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" clients - if: ${{ !inputs.skipCreateDataset }} && inputs.scenarioName == 'authentication.ClientSecret' + if: "!inputs.skipCreateDataset && inputs.scenarioName == 'authentication.ClientSecret'" uses: ./.github/actions/keycloak-create-dataset with: project: ${{ env.PROJECT }} clients: ${{ inputs.numberOfEntitiesInRealm }} - name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" users - if: ${{ !inputs.skipCreateDataset }} && inputs.scenarioName == 'authentication.AuthorizationCode' + if: "!inputs.skipCreateDataset && inputs.scenarioName == 'authentication.AuthorizationCode'" uses: ./.github/actions/keycloak-create-dataset with: project: ${{ env.PROJECT }} @@ -126,6 +126,7 @@ jobs: - name: Run "authentication.AuthorizationCode" scalability scenario if: inputs.scenarioName == 'authentication.AuthorizationCode' + id: kcb run: | bin/kcb.sh --scenario=keycloak.scenario."${{ inputs.scenarioName }}" \ --server-url=${{ env.KEYCLOAK_URL }} \ @@ -143,8 +144,50 @@ jobs: path: keycloak-benchmark/results retention-days: 5 + - name: Archive Summary + uses: actions/upload-artifact@v3 + with: + name: summary + path: ${{ steps.kcb.outputs.kcb_result }} + retention-days: 5 + - name: Delete Keycloak deployment if: ${{ !inputs.skipDeleteProject }} uses: ./.github/actions/keycloak-delete-deployment with: project: ${{ env.PROJECT }} + + archive: + name: Commit results to Git repository + runs-on: ubuntu-latest + permissions: + contents: write + needs: + - run + steps: + + - name: Checkout repository for results + uses: actions/checkout@v3 + with: + ref: 'result-data' + + - uses: actions/download-artifact@v3 + with: + name: summary + + - name: Commit result-summary + shell: bash + env: + GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + DATE_FOLDER=scalability/$(date +"%Y/%m/%d") + mkdir -p ${DATE_FOLDER} + mv *.json ${DATE_FOLDER} + git add . + git commit -m "generated" + git push + + diff --git a/.github/workflows/rosa-run-benchmark.yml b/.github/workflows/rosa-run-benchmark.yml index 2cd4710a..99717cd3 100644 --- a/.github/workflows/rosa-run-benchmark.yml +++ b/.github/workflows/rosa-run-benchmark.yml @@ -64,7 +64,7 @@ jobs: - name: Build with Maven run: | - ./mvnw -B clean install -DskipTests + ./mvnw -B clean package -DskipTests -pl benchmark tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz mv keycloak-benchmark-* keycloak-benchmark @@ -87,6 +87,7 @@ jobs: project: ${{ env.PROJECT }} - name: Run Authorization Code scenario + id: kcb run: bin/kcb.sh --scenario=keycloak.scenario.authentication.AuthorizationCode --server-url=${{ env.KEYCLOAK_URL }} --realm-name=realm-0 working-directory: keycloak-benchmark @@ -97,8 +98,49 @@ jobs: path: keycloak-benchmark/results retention-days: 5 + - name: Archive Summary + uses: actions/upload-artifact@v3 + with: + name: summary + path: ${{ steps.kcb.outputs.kcb_result }} + retention-days: 5 + - name: Delete Keycloak deployment if: ${{ !inputs.skipDeleteProject }} uses: ./.github/actions/keycloak-delete-deployment with: project: ${{ env.PROJECT }} + + archive: + name: Commit results to Git repository + runs-on: ubuntu-latest + permissions: + contents: write + needs: + - run + steps: + + - name: Checkout repository for results + uses: actions/checkout@v3 + with: + ref: 'result-data' + + - uses: actions/download-artifact@v3 + with: + name: summary + + - name: Commit result-summary + shell: bash + env: + GITHUB_OAUTH: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + DATE_FOLDER=benchmark/$(date +"%Y/%m/%d") + mkdir -p ${DATE_FOLDER} + mv *.json ${DATE_FOLDER} + git add . + git commit -m "generated" + git push + diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml index f98d01ea..a3271c57 100644 --- a/.github/workflows/snapshot.yml +++ b/.github/workflows/snapshot.yml @@ -21,6 +21,8 @@ jobs: build: name: Publish binaries runs-on: ubuntu-latest + # publishing can only happen on the main repository, not in forks + if: github.repository == 'keycloak/keycloak-benchmark' steps: - uses: actions/checkout@v3 diff --git a/benchmark/src/main/content/bin/kcb.sh b/benchmark/src/main/content/bin/kcb.sh index 2d579e29..168e3c5f 100755 --- a/benchmark/src/main/content/bin/kcb.sh +++ b/benchmark/src/main/content/bin/kcb.sh @@ -1,5 +1,4 @@ #!/bin/bash - case "$(uname)" in CYGWIN*) CFILE=$(cygpath "$0") @@ -112,11 +111,50 @@ run_benchmark_with_workload() { local OUTPUT_DIR="${4:-"$DIRNAME/../results/"}" echo "INFO: Running benchmark with $1=$2, result output will be available in: $OUTPUT_DIR" mkdir -p "$OUTPUT_DIR" + DATE_START_UNIX=$(date +%s) + DATE_START_ISO=$(date --iso-8601=seconds) + DATE_START_ISO_COMPRESSED=$(date '+%Y%m%d-%H%M%S') if [ "$MODE" = "incremental" ]; then java $JAVA_OPTS "${SERVER_OPTS[@]}" "${CONFIG_ARGS[@]}" "-D$1=$2" "-Dmeasurement=${3:-30}" -cp $CLASSPATH_OPTS io.gatling.app.Gatling -bf $DIRNAME -rf "$OUTPUT_DIR" -s $SCENARIO > "$OUTPUT_DIR/gatling.log" 2>&1 else java $JAVA_OPTS "${SERVER_OPTS[@]}" "${CONFIG_ARGS[@]}" "-D$1=$2" "-Dmeasurement=${3:-30}" -cp $CLASSPATH_OPTS io.gatling.app.Gatling -bf $DIRNAME -rf "$OUTPUT_DIR" -s $SCENARIO 2>&1 | tee "$OUTPUT_DIR/gatling.log" fi + EXIT_RESULT=$? + # don't include URLs or password information into the configuration recorded + CONFIG_ARGS_CLEAN=() + for index in "${!CONFIG_ARGS[@]}" ; do [[ "${CONFIG_ARGS[$index]}" =~ .*(url|pass).* ]] || CONFIG_ARGS_CLEAN+=(${CONFIG_ARGS[$index]}) ; done + OUTPUT_FOLDER=$OUTPUT_DIR/$(ls $OUTPUT_DIR -Art | grep -- -20 | tail -1) + DATE_END_UNIX=$(date +%s) + DATE_END_ISO=$(date --iso-8601=seconds) + jq '{ "grafana_output": { "stats": . } }' $OUTPUT_FOLDER/js/stats.json > $OUTPUT_FOLDER/result_grafana_stats.json + UUID=$(uuidgen) + jq '.' > $OUTPUT_FOLDER/result_grafana_inputs.json < ${OUTPUT_FOLDER}/result_sut.json + fi + jq -s add ${OUTPUT_FOLDER}/result_*.json > ${OUTPUT_FOLDER}/result-${DATE_START_ISO_COMPRESSED}-${UUID}.json + return ${EXIT_RESULT} } if [ "$MODE" = "incremental" ]; then @@ -133,6 +171,7 @@ if [ "$MODE" = "incremental" ]; then RESULT_ROOT_DIR="$DIRNAME/../results/$MODE-$(date '+%Y%m%d%H%M%S')" mkdir -p $RESULT_ROOT_DIR + RESULT_ROOT_DIR=$(realpath ${RESULT_ROOT_DIR}) #Incremental run is expected to do a warm up run to setup the system for the subsequent Incremental runs, you can ignore this run's result. echo "INFO: Running warm-up phase." @@ -162,6 +201,11 @@ if [ "$MODE" = "incremental" ]; then echo "INFO: Last Successful workload for scenario $SCENARIO is $WORKLOAD_UNIT=$LAST_SUCCESSFUL_WORKLOAD." if [ $INCREMENT -eq 1 ]; then + ln -s $RESULT_ROOT_DIR/$WORKLOAD_UNIT-$LAST_SUCCESSFUL_WORKLOAD $RESULT_ROOT_DIR/last-successful + if [[ "${GITHUB_OUTPUT}" != "" ]]; then + OUTPUT_FOLDER=$RESULT_ROOT_DIR/last-successful/$(ls $RESULT_ROOT_DIR/last-successful -Art | grep -- -20 | tail -1) + echo "kcb_result=$OUTPUT_FOLDER/result-*.json" >> "${GITHUB_OUTPUT}" + fi echo "INFO: Reached the limit for scenario $SCENARIO with $WORKLOAD_UNIT=$LAST_SUCCESSFUL_WORKLOAD." exit fi @@ -181,5 +225,10 @@ if [ "$MODE" = "incremental" ]; then else echo "INFO: Running benchmark in single-run mode." run_benchmark_with_workload $WORKLOAD_UNIT $CURRENT_WORKLOAD $MEASUREMENT + if [[ "${GITHUB_OUTPUT}" != "" ]]; then + OUTPUT_FOLDER=$DIRNAME/../results/$(ls $DIRNAME/../results -Art | grep -- -20 | tail -1) + OUTPUT_FOLDER=$(realpath ${OUTPUT_FOLDER}) + echo "kcb_result=$OUTPUT_FOLDER/result-*.json" >> "${GITHUB_OUTPUT}" + fi exit fi diff --git a/dashbuilder/.gitignore b/dashbuilder/.gitignore new file mode 100644 index 00000000..de4d1f00 --- /dev/null +++ b/dashbuilder/.gitignore @@ -0,0 +1,2 @@ +dist +node_modules diff --git a/dashbuilder/build.sh b/dashbuilder/build.sh new file mode 100755 index 00000000..904d317f --- /dev/null +++ b/dashbuilder/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash -e +yarn install +yarn run build diff --git a/dashbuilder/package.json b/dashbuilder/package.json new file mode 100644 index 00000000..e173225a --- /dev/null +++ b/dashbuilder/package.json @@ -0,0 +1,17 @@ +{ + "name": "dashbuilder-webapp", + "version": "0.0.0", + "description": "Dashbuilder WebApp", + "scripts": { + "bootstrap": "npm install", + "clean": "rm -rf dist/", + "copy:dashbuilder": "cp -r node_modules/@kie-tools/dashbuilder-client/dist/* dist", + "copy:sources": "cp -r static/* dist", + "build": "npm run clean && mkdir dist/ && npm run copy:dashbuilder && npm run copy:sources", + "server": "npm run build && cd dist && http-server -p 8000" + }, + "devDependencies": { + "@kie-tools/dashbuilder-client": "^0.29.0", + "http-server": "^14.1.1" + } +} diff --git a/dashbuilder/static/dashboard.yaml b/dashbuilder/static/dashboard.yaml new file mode 100644 index 00000000..d5f503e1 --- /dev/null +++ b/dashbuilder/static/dashboard.yaml @@ -0,0 +1,99 @@ +# YAML description for a dashboard, see https://www.dashbuilder.org/docs/ +# Use the interactive dashboard builder to create variants: https://start.kubesmarts.org/ +# As a JSON schema, use https://www.dashbuilder.org/schemas/0.1/dashbuilder.json +datasets: + - uuid: scalability + # URL where to pick up the results + url: https://keycloak.github.io/keycloak-benchmark/results/scaletest-results.json + # JSONata style expression to pick values in a tabular format, see https://jsonata.org/ + expression: '$sort(*.[uuid,grafana_input.start.iso,grafana_input.input.value,grafana_output.stats.stats.numberOfRequests.total,grafana_output.stats.stats.percentiles1.total,grafana_output.stats.stats.percentiles2.total,grafana_output.stats.stats.percentiles3.total,grafana_output.stats.stats.percentiles4.total], function($l, $r) { $l[1] > $r[1]})' + columns: + - id: ID + type: LABEL + - id: Start + type: LABEL + - id: Users + type: NUMBER + - id: Requests + type: NUMBER + - id: Response Time 50th pct + type: NUMBER + - id: Response Time 75th pct + type: NUMBER + - id: Response Time 95th pct + type: NUMBER + - id: Response Time 99th pct + type: NUMBER +pages: + - rows: + - columns: + - span: '12' + components: + - html: Scalability results + properties: + font-size: xx-large + margin-bottom: 30px + - columns: + - span: '6' + components: + - settings: + type: LINECHART + general: + title: "Users per second" + visible: true + axis: + x: + labels_show: false + lookup: + uuid: scalability + group: + - columnGroup: + source: ID + functions: + - source: Start + - source: Users + - span: '6' + components: + - settings: + type: LINECHART + chart: + margin: + left: 60 + general: + title: "Response time in milliseconds" + visible: true + axis: + x: + labels_show: false + lookup: + uuid: scalability + group: + - columnGroup: + source: ID + functions: + - source: Start + - source: Response Time 50th pct + - source: Response Time 75th pct + - source: Response Time 95th pct + - source: Response Time 99th pct + - columns: + - span: '16' + components: + - settings: + columns: + - id: Response Time 50th pct + pattern: '#' + - id: Response Time 50th pct + pattern: '#' + - id: Response Time 75th pct + pattern: '#' + - id: Response Time 95th pct + pattern: '#' + - id: Response Time 99th pct + pattern: '#' + - id: Users + pattern: '#' + - id: Requests + pattern: '#' + lookup: + uuid: scalability diff --git a/dashbuilder/static/setup.js b/dashbuilder/static/setup.js new file mode 100644 index 00000000..2e7c07a3 --- /dev/null +++ b/dashbuilder/static/setup.js @@ -0,0 +1,14 @@ +dashbuilder = { + // possible modes are EDITOR and CLIENT - if dashboards is set then CLIENT mode is assumed + mode: "CLIENT", + // The list of client dashboards - if CLIENT mode is used and no list is provided, than the dashboard "dashboard.yml" will be attempted to load. If no dashboard is found, then client opens for upload. + // The dashboard can also be an URL + dashboards: [ "dashboard.yaml" ], + + // base path to look for dashboards. Default is / + path: "", + + // if true, then model from external urls will be allowed + allowExternal: true, + +}; diff --git a/dashbuilder/yarn.lock b/dashbuilder/yarn.lock new file mode 100644 index 00000000..1a0c439e --- /dev/null +++ b/dashbuilder/yarn.lock @@ -0,0 +1,271 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@kie-tools/dashbuilder-client@^0.29.0": + version "0.29.0" + resolved "https://registry.yarnpkg.com/@kie-tools/dashbuilder-client/-/dashbuilder-client-0.29.0.tgz#101da374e0c3f58182e980aaeef8e69be8944858" + integrity sha512-xHQZQQIqrsLRyudPA4ekajSBbONG/z4TSKcfG2CISS83698QnB2oLrzC9fiFVM/1G0MFFRbf37vBEWPvqk7w0A== + +ansi-styles@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + +async@^2.6.4: + version "2.6.4" + resolved "https://registry.yarnpkg.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221" + integrity sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA== + dependencies: + lodash "^4.17.14" + +basic-auth@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.1.tgz#b998279bf47ce38344b4f3cf916d4679bbf51e3a" + integrity sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg== + dependencies: + safe-buffer "5.1.2" + +call-bind@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== + dependencies: + function-bind "^1.1.1" + get-intrinsic "^1.0.2" + +chalk@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + +corser@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/corser/-/corser-2.0.1.tgz#8eda252ecaab5840dcd975ceb90d9370c819ff87" + integrity sha512-utCYNzRSQIZNPIcGZdQc92UVJYAhtGAteCFg0yRaFm8f0P+CPtyGyHXJcGXnffjCybUCEx3FQ2G7U3/o9eIkVQ== + +debug@^3.2.7: + version "3.2.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== + dependencies: + ms "^2.1.1" + +eventemitter3@^4.0.0: + version "4.0.7" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" + integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== + +follow-redirects@^1.0.0: + version "1.15.2" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" + integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== + +function-bind@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" + integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== + +get-intrinsic@^1.0.2: + version "1.2.1" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.1.tgz#d295644fed4505fc9cde952c37ee12b477a83d82" + integrity sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw== + dependencies: + function-bind "^1.1.1" + has "^1.0.3" + has-proto "^1.0.1" + has-symbols "^1.0.3" + +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== + +has-proto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/has-proto/-/has-proto-1.0.1.tgz#1885c1305538958aff469fef37937c22795408e0" + integrity sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg== + +has-symbols@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" + integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== + +has@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" + integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== + dependencies: + function-bind "^1.1.1" + +he@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" + integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== + +html-encoding-sniffer@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz#2cb1a8cf0db52414776e5b2a7a04d5dd98158de9" + integrity sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA== + dependencies: + whatwg-encoding "^2.0.0" + +http-proxy@^1.18.1: + version "1.18.1" + resolved "https://registry.yarnpkg.com/http-proxy/-/http-proxy-1.18.1.tgz#401541f0534884bbf95260334e72f88ee3976549" + integrity sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ== + dependencies: + eventemitter3 "^4.0.0" + follow-redirects "^1.0.0" + requires-port "^1.0.0" + +http-server@^14.1.1: + version "14.1.1" + resolved "https://registry.yarnpkg.com/http-server/-/http-server-14.1.1.tgz#d60fbb37d7c2fdff0f0fbff0d0ee6670bd285e2e" + integrity sha512-+cbxadF40UXd9T01zUHgA+rlo2Bg1Srer4+B4NwIHdaGxAGGv59nYRnGGDJ9LBk7alpS0US+J+bLLdQOOkJq4A== + dependencies: + basic-auth "^2.0.1" + chalk "^4.1.2" + corser "^2.0.1" + he "^1.2.0" + html-encoding-sniffer "^3.0.0" + http-proxy "^1.18.1" + mime "^1.6.0" + minimist "^1.2.6" + opener "^1.5.1" + portfinder "^1.0.28" + secure-compare "3.0.1" + union "~0.5.0" + url-join "^4.0.1" + +iconv-lite@0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" + integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== + dependencies: + safer-buffer ">= 2.1.2 < 3.0.0" + +lodash@^4.17.14: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== + +mime@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" + integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== + +minimist@^1.2.6: + version "1.2.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" + integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== + +mkdirp@^0.5.6: + version "0.5.6" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" + integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== + dependencies: + minimist "^1.2.6" + +ms@^2.1.1: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +object-inspect@^1.9.0: + version "1.12.3" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.3.tgz#ba62dffd67ee256c8c086dfae69e016cd1f198b9" + integrity sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g== + +opener@^1.5.1: + version "1.5.2" + resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598" + integrity sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A== + +portfinder@^1.0.28: + version "1.0.32" + resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.32.tgz#2fe1b9e58389712429dc2bea5beb2146146c7f81" + integrity sha512-on2ZJVVDXRADWE6jnQaX0ioEylzgBpQk8r55NE4wjXW1ZxO+BgDlY6DXwj20i0V8eB4SenDQ00WEaxfiIQPcxg== + dependencies: + async "^2.6.4" + debug "^3.2.7" + mkdirp "^0.5.6" + +qs@^6.4.0: + version "6.11.2" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.2.tgz#64bea51f12c1f5da1bc01496f48ffcff7c69d7d9" + integrity sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA== + dependencies: + side-channel "^1.0.4" + +requires-port@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" + integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ== + +safe-buffer@5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + +"safer-buffer@>= 2.1.2 < 3.0.0": + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== + +secure-compare@3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/secure-compare/-/secure-compare-3.0.1.tgz#f1a0329b308b221fae37b9974f3d578d0ca999e3" + integrity sha512-AckIIV90rPDcBcglUwXPF3kg0P0qmPsPXAj6BBEENQE1p5yA1xfmDJzfi1Tappj37Pv2mVbKpL3Z1T+Nn7k1Qw== + +side-channel@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" + integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== + dependencies: + call-bind "^1.0.0" + get-intrinsic "^1.0.2" + object-inspect "^1.9.0" + +supports-color@^7.1.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== + dependencies: + has-flag "^4.0.0" + +union@~0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/union/-/union-0.5.0.tgz#b2c11be84f60538537b846edb9ba266ba0090075" + integrity sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA== + dependencies: + qs "^6.4.0" + +url-join@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.1.tgz#b642e21a2646808ffa178c4c5fda39844e12cde7" + integrity sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA== + +whatwg-encoding@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz#e7635f597fd87020858626805a2729fa7698ac53" + integrity sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg== + dependencies: + iconv-lite "0.6.3" diff --git a/doc/benchmark/modules/ROOT/nav.adoc b/doc/benchmark/modules/ROOT/nav.adoc index 6347cc85..94db7072 100644 --- a/doc/benchmark/modules/ROOT/nav.adoc +++ b/doc/benchmark/modules/ROOT/nav.adoc @@ -10,6 +10,7 @@ ** xref:report/differential-report.adoc[] ** xref:report/trend-report.adoc[] ** xref:report/diagram-types.adoc[] +** xref:report/result-summary.adoc[] * xref:scenario-overview.adoc[] ** xref:scenario/authorization-code.adoc[] ** xref:scenario/list-sessions.adoc[] diff --git a/doc/benchmark/modules/ROOT/pages/report/result-summary.adoc b/doc/benchmark/modules/ROOT/pages/report/result-summary.adoc new file mode 100644 index 00000000..56566d00 --- /dev/null +++ b/doc/benchmark/modules/ROOT/pages/report/result-summary.adoc @@ -0,0 +1,56 @@ += Results summary +:description: There shell script creates a summary JSON which can be used to compare different runs. + +{description} + +== JSON created for each run + +For every run, `kcb.sh` creates a file `results.json` which contains the input parameters +to Gatling and also an output of the results. + +Each result summarizes a run, and can be processed further in a pipeline. +Each run has a descriptive name and a UUID which identifies the run. +The file name contains a timestamp and the UUID, so it is simple to put multiple result files in a single folder for further processing. + +A user of the `kcb.sh` script might want to capture information about the system under test (SUT) as part of the result file to record the information for later use. +To do this, the user has to provide a script which collects the information and prints them on the console. +For a ROSA environment, there is link:{github-files}/provision/aws/rosa_describe.sh[rosa_describe.sh] which would collect the information about the cluster. +When the environment variable `SUT_DESCRIBE` is set, `kcb.sh` calls the executable and CLI arguments given in the environment variable `SUT_DESCRIBE` to retrieve the JSON description for the system under test. +It then adds it to the file with the key `system_under_test`. +See link:{github-files}/.github/actions/oc-keycloak-login/action.yml[oc-keycloak-login/action.yml] how the environment variable is set for a GitHub workflow. + +When running this from a GitHub Workflow, the file name is recorded in the GitHub Action output `kcb_result` so it can be picked up in later steps in the GitHub Workflow. + +.Example excerpt from a results JSON file. +[source,json] +---- +{ + "uuid": ..., + "name": ..., + "grafana_input": { + "start": ..., + "end": ..., + "input": ... + }, + "grafana_output": ..., + "system_under_test": ... +} +---- + +== Summarizing report for the scalability test + +For the scalability test, there is link:/keycloak-benchmark/dashbuilder[a report visualizing the latest runs here]. + +The process that leads to this diagram is as follows: + +. The link:{github-files}/.github/workflows/keycloak-scalability-benchmark.yml[Keycloak - Scalability benchmark GitHub workflow] commits a result file to the `link:{github-files}/../result-data/[result-data]` branch of the repository on each successful run. + +. For https://www.dashbuilder.org/[Dashbuilder], the data needs to be available as a single JSON file. +This is generated using the link:{github-files}/../result-data/.github/workflows/aggregate-results.yaml[Aggregate Benchmark results GitHub workflow] in the `result-data` branch of the repository. + +. The data is then published to GitHub pages in the link:{github-files}/.github/workflows/docs-pages.yml[Publishing Documentation Site GitHub workflow], as the file created for the releases can't be fetched by Dashbuilder, as the initial response is an HTTP redirect. + +. The dashboard is built using https://www.dashbuilder.org[Dashbuilder] using this `link:{github-files}/dashbuilder/static/dashboard.yaml[dashboard.yaml]` file. +There is an https://start.kubesmarts.org/[interactive dashboard builder] to interactively change the diagram. +See the https://www.dashbuilder.org/docs/[Dashbuilder docs] on how to update the YAML file. + diff --git a/provision/aws/rosa_describe.sh b/provision/aws/rosa_describe.sh new file mode 100755 index 00000000..30d2cd56 --- /dev/null +++ b/provision/aws/rosa_describe.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail +if [[ "$#" -ne "1" ]]; then + echo "This describes one ROSA cluster in JSON, but only information that is not security related." + echo "Provide exactly one parameter with the name of the ROSA cluster" + exit 1 +fi +rosa describe cluster -c $1 -o json | jq '. | { rosa: { creation_timestamp: .creation_timestamp, flavour: .flavour, multi_az: .multi_az, nodes: .nodes, openshift_version: .openshift_version, region: .region, version: .version }} '