-
Notifications
You must be signed in to change notification settings - Fork 330
137 lines (125 loc) · 7.16 KB
/
cbench.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
name: CBENCH
on:
workflow_dispatch:
push:
branches:
- main
paths:
- "quickwit/**"
- "!quickwit/quickwit-ui/**"
# For security reasons (to make sure the list of allowed users is
# trusted), make sure we run the workflow definition from the base
# commit of the pull request.
pull_request_target:
# This is required for github.rest.issues.createComment.
permissions:
issues: write
pull-requests: write
env:
RUSTFLAGS: --cfg tokio_unstable
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
tests:
name: Benchmark
# The self-hosted runner must have the system deps installed for QW and
# the benchmark, because we don't have root access.
runs-on: self-hosted
timeout-minutes: 60
steps:
- name: Set authorized users
id: authorized-users
# List of users allowed to trigger this workflow.
# Because it executes code on a self-hosted runner, it must be restricted to trusted users.
run: |
echo 'users=["ddelemeny", "fmassot", "fulmicoton", "guilload", "PSeitz", "rdettai", "trinity-1686a"]' >> $GITHUB_OUTPUT
- uses: actions/checkout@v4
if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) && github.event_name == 'pull_request_target'
name: Checkout quickwit (pull request commit)
with:
repository: quickwit-oss/quickwit
ref: ${{ github.event.pull_request.head.sha }}
path: ./quickwit
- uses: actions/checkout@v4
if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) && github.event_name != 'pull_request_target'
name: Checkout quickwit
with:
repository: quickwit-oss/quickwit
ref: ${{ github.sha }}
path: ./quickwit
- name: Checkout benchmarking code
uses: actions/checkout@v4
if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor)
with:
repository: quickwit-oss/benchmarks
ref: main
path: ./benchmarks
- name: Install Rust
run: rustup update stable
- name: Install protoc
uses: taiki-e/install-action@v2
with:
tool: protoc
# We don't use rust-cache as it requires root access on the self-hosted runner, which we don't have.
- name: cargo build
if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor)
run: cargo build --release --bin quickwit
working-directory: ./quickwit/quickwit
- name: Compile qbench
if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor)
run: cargo build --release
working-directory: ./benchmarks/qbench
- name: Run Benchmark on SSD
if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor)
id: bench-run-ssd
run: python3 ./run.py --search-only --storage pd-ssd --engine quickwit --track generated-logs --tags "${{ github.event_name }}_${{ github.ref_name }}" --manage-engine --source github_workflow --binary-path ../quickwit/quickwit/target/release/quickwit --instance "{autodetect_gcp}" --export-to-endpoint=https://qw-benchmarks.104.155.161.122.nip.io --engine-data-dir "{qwdata_local}" --github-workflow-user "${{ github.actor }}" --github-workflow-run-id "${{ github.run_id }}" --comparison-reference-tag="push_main" --github-pr "${{ github.event_name == 'pull_request_target' && github.event.number || 0 }}" --comparison-reference-commit "${{ github.event_name == 'pull_request_target' && github.sha || github.event.before }}" --write-exported-run-url-to-file $GITHUB_OUTPUT
working-directory: ./benchmarks
- name: Run Benchmark on cloud storage
if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor)
id: bench-run-cloud-storage
run: python3 ./run.py --search-only --storage gcs --engine quickwit --track generated-logs --tags "${{ github.event_name }}_${{ github.ref_name }}" --manage-engine --source github_workflow --binary-path ../quickwit/quickwit/target/release/quickwit --instance "{autodetect_gcp}" --export-to-endpoint=https://qw-benchmarks.104.155.161.122.nip.io --engine-data-dir "{qwdata_gcs}" --engine-config-file engines/quickwit/configs/cbench_quickwit_gcs.yaml --github-workflow-user "${{ github.actor }}" --github-workflow-run-id "${{ github.run_id }}" --comparison-reference-tag="push_main" --github-pr "${{ github.event_name == 'pull_request_target' && github.event.number || 0 }}" --comparison-reference-commit "${{ github.event_name == 'pull_request_target' && github.sha || github.event.before }}" --write-exported-run-url-to-file $GITHUB_OUTPUT
working-directory: ./benchmarks
- name: Show results links
if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor)
run: |
echo "::notice title=Benchmark Results on SSD::${{ steps.bench-run-ssd.outputs.url }}"
echo "::notice title=Comparison of results on SSD::${{ steps.bench-run-ssd.outputs.comparison_text }}"
echo "::notice title=Benchmark Results on Cloud Storage::${{ steps.bench-run-cloud-storage.outputs.url }}"
echo "::notice title=Comparison of results on Cloud Storage::${{ steps.bench-run-cloud-storage.outputs.comparison_text }}"
- name: In case of auth error
if: ${{ ! contains(fromJSON(steps.authorized-users.outputs.users), github.actor) }}
run: |
echo "::error title=User not allowed to run the benchmark::User must be in list ${{ steps.authorized-users.outputs.users }}"
- name: Add a PR comment with comparison results
uses: actions/github-script@v7
if: contains(fromJSON(steps.authorized-users.outputs.users), github.actor) && github.event_name == 'pull_request_target'
# Inspired from: https://github.com/actions/github-script/blob/60a0d83039c74a4aee543508d2ffcb1c3799cdea/.github/workflows/pull-request-test.yml
with:
script: |
// Get the existing comments.
const {data: comments} = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.payload.number,
})
// Find any comment already made by the bot to update it.
const botComment = comments.find(comment => comment.user.id === 41898282)
const commentBody = "### On SSD:\n${{ steps.bench-run-ssd.outputs.comparison_text }}\n### On GCS:\n${{ steps.bench-run-cloud-storage.outputs.comparison_text }}\n"
if (botComment) {
// Update existing comment.
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: botComment.id,
body: commentBody
})
} else {
// New comment.
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.payload.number,
body: commentBody
})
}