diff --git a/app/models/beta_pages.rb b/app/models/beta_pages.rb index db0db01564..952c7cc0bd 100644 --- a/app/models/beta_pages.rb +++ b/app/models/beta_pages.rb @@ -1,7 +1,8 @@ class BetaPages def self.all [ - 'test-analytics/test-ownership' + 'path/page-name', + 'path/last-page-name-in-list-without-trailing-comma' ] end end diff --git a/app/views/homepage/_popular_guides.html.erb b/app/views/homepage/_popular_guides.html.erb index 3382ac42a9..3426fc904b 100644 --- a/app/views/homepage/_popular_guides.html.erb +++ b/app/views/homepage/_popular_guides.html.erb @@ -32,7 +32,7 @@ }, { id: "glossary", - title: "Glossary", + title: "Pipelines glossary", summary: "A list of common terms that describe key concepts", link: "/docs/pipelines/glossary", } diff --git a/data/content/test_splitting_env.yaml b/data/content/test_splitting_env.yaml index 422ae9aa71..368c3acecf 100644 --- a/data/content/test_splitting_env.yaml +++ b/data/content/test_splitting_env.yaml @@ -1,7 +1,7 @@ predefined: - name: BUILDKITE_BUILD_ID desc: - - The UUID of the pipeline build. Test Splitter uses this UUID along with `BUILDKITE_STEP_ID` to uniquely identify the test plan. + - The UUID of the pipeline build. The Test Engine Client uses this UUID along with `BUILDKITE_STEP_ID` to uniquely identify the test plan. - name: BUILDKITE_JOB_ID desc: - The UUID of the job in the pipeline's build. @@ -18,40 +18,90 @@ predefined: - Ensure you configure `parallelism` in your pipeline definition. Learn more about parallel build steps in [Concurrency and parallelism](https://buildkite.com/docs/pipelines/controlling-concurrency#concurrency-and-parallelism). - name: BUILDKITE_STEP_ID desc: - - The UUID of the step group in the pipeline build. Test Splitter uses this UUID along with `BUILDKITE_BUILD_ID` to uniquely identify the test plan. + - The UUID of the step group in the pipeline build. The Test Engine Client uses this UUID along with `BUILDKITE_BUILD_ID` to uniquely identify the test plan. mandatory: - - name: BUILDKITE_SPLITTER_API_ACCESS_TOKEN + - name: BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN desc: - - Buildkite API access token with `read_suites`, `read_test_plan`, and `write_test_plan` scopes. You can create an [API access token](https://buildkite.com/user/api-access-tokens) from **Personal Settings** > **API Access Tokens** in the Buildkite interface. - - name: BUILDKITE_SPLITTER_SUITE_SLUG + - Buildkite API access token with `read_suites`, `read_test_plan`, and `write_test_plan` scopes. You can create an [API access token](https://buildkite.com/user/api-access-tokens) from **Personal Settings** > **API Access Tokens** in the Buildkite interface. To avoid this token being tied to any one employee or person, ideally, this token should be created for a bot user account within your Buildkite organization. + - name: BUILDKITE_TEST_ENGINE_RESULT_PATH desc: - - The slug of your Buildkite Test Engine test suite. You can find the suite slug in the url for your test suite. + - The path to store the test result. The Test Engine Client uses this environment variable to tell the runner where to store the test result. The Test Engine Client reads the test result after each test run for retries and verification. + - For RSpec, the result is generated using the `--format json` and `--out` CLI options, while for Jest, it is generated using the `--json` and `--outputFile` options. We have included these options in the default test command for RSpec and Jest. If you need to customize your test command, make sure to append the CLI options to save the result to a file. + - Please refer to the `BUILDKITE_TEST_ENGINE_TEST_CMD` environment variable for more details. + note: + - The Test Engine Client will not delete the file after running the test, however it will be deleted by Buildkite Agent as part of the build lifecycle. + - name: BUILDKITE_TEST_ENGINE_RUNNER + desc: + - The test runner to use for running tests. Currently `rspec` and `jest` are supported. + - name: BUILDKITE_TEST_ENGINE_SUITE_SLUG + desc: + - The slug of your Test Engine test suite. You can find the suite slug in the url for your test suite. - "For example, the slug for the url: `https://buildkite.com/organizations/my-organization/analytics/suites/my-suite` is `my-suite`." optional: - - name: BUILDKITE_SPLITTER_DEBUG_ENABLED - default: false - desc: - - A flag to enable more verbose logging. - - name: BUILDKITE_SPLITTER_RETRY_COUNT - default: 0 - desc: - - The number of retries permitted. Test Splitter runs the test command defined in `BUILDKITE_SPLITTER_TEST_CMD`, and retries only the failing tests for a maximum of `BUILDKITE_SPLITTER_RETRY_COUNT` times. For RSpec, the Test Splitter runs `BUILDKITE_SPLITTER_TEST_CMD` with `--only-failures` as the retry command. - - name: BUILDKITE_SPLITTER_SPLIT_BY_EXAMPLE - default: false - desc: - - A flag to enable split by example. When this option is `true`, the Test Splitter will split the execution of slow test files over multiple partitions. - - name: BUILDKITE_SPLITTER_TEST_CMD - default: bundle exec rspec {{testExamples}} - desc: - - The test command to run your tests. The Test Splitter will replace and populate the `{{testExamples}}` placeholder with the test plan. - - name: BUILDKITE_SPLITTER_TEST_FILE_EXCLUDE_PATTERN - desc: - - The glob pattern to exclude certain test files or directories. The exclusion will be applied after discovering the test files using a pattern configured with `BUILDKITE_SPLITTER_TEST_FILE_PATTERN`. - - _This option accepts the pattern syntax supported by the [zzglob](https://github.com/DrJosh9000/zzglob?tab=readme-ov-file#pattern-syntax) library._ - - name: BUILDKITE_SPLITTER_TEST_FILE_PATTERN - default: spec/**/*_spec.rb - desc: - - The glob pattern to discover test files. You can exclude certain test files or directories from the discovered test files using a pattern that can be configured with `BUILDKITE_SPLITTER_TEST_FILE_EXCLUDE_PATTERN`. - - _This option accepts the pattern syntax supported by the [zzglob](https://github.com/DrJosh9000/zzglob?tab=readme-ov-file#pattern-syntax) library._ + rspec: + - name: BUILDKITE_TEST_ENGINE_DEBUG_ENABLED + default: false + desc: + - A flag to enable more verbose logging. + - name: BUILDKITE_TEST_ENGINE_RETRY_CMD + desc: + - The command to retry the failed tests. The Test Engine Client will replace the `{{testExamples}}` placeholder with the failed tests. If not set, the client will use the same command defined in `BUILDKITE_TEST_ENGINE_TEST_CMD`. + - name: BUILDKITE_TEST_ENGINE_RETRY_COUNT + default: 0 + desc: + - The number of retries. The Test Engine Client runs the test command defined in `BUILDKITE_TEST_ENGINE_TEST_CMD` and retries only the failed tests up to `BUILDKITE_TEST_ENGINE_RETRY_COUNT` times, using the retry command defined in `BUILDKITE_TEST_ENGINE_RETRY_CMD`. + - name: BUILDKITE_TEST_ENGINE_SPLIT_BY_EXAMPLE + default: false + desc: + - A flag to enable split by example. When this option is `true`, the Test Engine Client will split the execution of slow test files over multiple partitions. + - name: BUILDKITE_TEST_ENGINE_TEST_CMD + default: bundle exec rspec --format progress --format json --out {{resultPath}} {{testExamples}} + desc: + - The test command to run your tests. The Test Engine Client will replace the `{{testExamples}}` placeholder with the test plan. + note: + - It is necessary to include `--format json --out {{resultPath}}` in the test command, because the Test Engine Client needs to read the result after each test run. + - Please refer to the `BUILDKITE_TEST_ENGINE_RESULT_PATH` environment variable for more details. + - name: BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN + desc: + - The glob pattern to exclude certain test files or directories. The exclusion will be applied after discovering the test files using a pattern configured with `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN`. + - _This option accepts the pattern syntax supported by the [zzglob](https://github.com/DrJosh9000/zzglob?tab=readme-ov-file#pattern-syntax) library._ + - name: BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN + default: spec/**/*_spec.rb + desc: + - The glob pattern to discover test files. You can exclude certain test files or directories from the discovered test files using a pattern that can be configured with `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN`. + - _This option accepts the pattern syntax supported by the [zzglob](https://github.com/DrJosh9000/zzglob?tab=readme-ov-file#pattern-syntax) library._ + jest: + - name: BUILDKITE_TEST_ENGINE_DEBUG_ENABLED + default: false + desc: + - A flag to enable more verbose logging. + - name: BUILDKITE_TEST_ENGINE_RETRY_CMD + default: yarn test --testNamePattern '{{testNamePattern}}' --json --testLocationInResults --outputFile {{resultPath}} + desc: + - The command to retry the failed tests. The Test Engine Client will replace the `{{testNamePattern}}` placeholder with the failed tests. + note: + - It is necessary to include `--json --testLocationInResults --outputFile {{resultPath}}` in the command, because the Test Engine Client needs to read the result after each test run. + - Please refer to the `BUILDKITE_TEST_ENGINE_RESULT_PATH` environment variable for more details. + - name: BUILDKITE_TEST_ENGINE_RETRY_COUNT + default: 0 + desc: + - The number of retries. The Test Engine Client runs the test command defined in `BUILDKITE_TEST_ENGINE_TEST_CMD` and retries only the failed tests up to `BUILDKITE_TEST_ENGINE_RETRY_COUNT` times, using the retry command defined in `BUILDKITE_TEST_ENGINE_RETRY_CMD`. + - name: BUILDKITE_TEST_ENGINE_TEST_CMD + default: yarn test {{testExamples}} --json --testLocationInResults --outputFile {{resultPath}} + desc: + - The test command to run your tests. The Test Engine Client will replace and populate the `{{testExamples}}` placeholder with the test plan. + note: + - It is necessary to include `--json --testLocationInResults --outputFile {{resultPath}}` in the command, because the Test Engine Client needs to read the result after each test run. + - Please refer to the `BUILDKITE_TEST_ENGINE_RESULT_PATH` environment variable for more details. + - name: BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN + default: "node_modules" + desc: + - The glob pattern to exclude certain test files or directories. The exclusion will be applied after discovering the test files using a pattern configured with `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN`. + - _This option accepts the pattern syntax supported by the [zzglob](https://github.com/DrJosh9000/zzglob?tab=readme-ov-file#pattern-syntax) library._ + - name: BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN + default: "**/{__tests__/**/*,*.spec,*.test}.{ts,js,tsx,jsx}" + desc: + - The glob pattern to discover test files. You can exclude certain test files or directories from the discovered test files using a pattern that can be configured with `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN`. + - _This option accepts the pattern syntax supported by the [zzglob](https://github.com/DrJosh9000/zzglob?tab=readme-ov-file#pattern-syntax) library._ diff --git a/data/nav.yml b/data/nav.yml index 8468908c8a..aaaf21fda4 100644 --- a/data/nav.yml +++ b/data/nav.yml @@ -422,23 +422,28 @@ children: - name: "Configuring test suites" path: "test-engine/test-suites" - - name: "Configuring test splitting" - path: "test-engine/test-splitting" - name: "Permissions" path: "test-engine/permissions" - name: "CI environment variables" path: "test-engine/ci-environments" - - name: "Test executions" - path: "test-engine/test-executions" - name: "Public test suites" path: "test-engine/public-test-suites" - name: "Test ownership" - pill: "beta" path: "test-engine/test-ownership" - name: "Flaky test assignment" path: "test-engine/flaky-test-assignment" - - name: "Languages" + - name: "Usage and billing" + path: "test-engine/usage-and-billing" + - name: "Test splitting" start_expanded: true + children: + - name: "Overview" + path: "test-engine/test-splitting" + - name: "Configuring" + path: "test-engine/test-splitting/configuring" + - name: "Test Engine Client installation" + path: "test-engine/test-splitting/client-installation" + - name: "Languages" children: - name: "Ruby" path: "test-engine/ruby-collectors" @@ -463,7 +468,6 @@ - name: "Other languages" path: "test-engine/other-collectors" - name: "References" - start_expanded: true children: - name: "Importing JUnit XML" path: "test-engine/importing-junit-xml" diff --git a/data/tiles.yml b/data/tiles.yml index 027ac27d87..20c15e4f05 100644 --- a/data/tiles.yml +++ b/data/tiles.yml @@ -6,6 +6,9 @@ test_engine_features: - title: "Find and fix flaky tests" url: "/docs/test-engine/test-suites#detecting-flaky-tests" desc: "Quickly identify which tests are the most disruptive for your team, and get a head-start on fixing them." + - title: "Reduce build times with test splitting" + url: "/docs/test-engine/test-splitting" + desc: "Split tests evenly across agents to reduce overall pipeline build times, especially for highly complex test suites." test_engine_guides: - title: "Languages" links: diff --git a/images/docs/test_engine/test_splitting/setup-page-summary.png b/images/docs/test_engine/test_splitting/setup-page-summary.png new file mode 100644 index 0000000000..7f91222139 Binary files /dev/null and b/images/docs/test_engine/test_splitting/setup-page-summary.png differ diff --git a/images/docs/test_engine/test_executions/test_executions.png b/images/docs/test_engine/usage_and_billing/test_executions.png similarity index 100% rename from images/docs/test_engine/test_executions/test_executions.png rename to images/docs/test_engine/usage_and_billing/test_executions.png diff --git a/pages/test_engine.md b/pages/test_engine.md index 11701c5a6b..b237a46177 100644 --- a/pages/test_engine.md +++ b/pages/test_engine.md @@ -19,7 +19,7 @@ _Buildkite Test Engine_ was previously called _Buildkite Test Analytics_. ## Get started -Run through the 'Getting started' section of these Test Engine docs, beginning with [Configuring test suites](/docs/test-engine/test-suites) for an overview of Test Engine's concepts and functionality, followed by the appropriate test collector for project's language: +To get started with Test Engine, and to begin setting up your _test suites_, configure the relevant supported _test collectors_ for your development project. @@ -45,6 +45,8 @@ Run through the 'Getting started' section of these Test Engine docs, beginning w You can also upload test results by importing [JSON](/docs/test-engine/importing-json) or [JUnit XML](/docs/test-engine/importing-junit-xml). +Once your test collectors have been set up, you can begin configuring your test suites by running through the relevant 'Getting started' sections, beginning with [Configuring test suites](/docs/test-engine/test-suites) for an overview of Test Engine's concepts and functionality. +
<%= tiles "test_engine_features" %> diff --git a/pages/test_engine/test_executions.md b/pages/test_engine/test_executions.md deleted file mode 100644 index 9823eac0b1..0000000000 --- a/pages/test_engine/test_executions.md +++ /dev/null @@ -1,14 +0,0 @@ -# Test executions - -Each [Buildkite plan](https://buildkite.com/pricing) has test execution inclusions, which vary depending on the plan type and the number of users in your organization. - -You can find the test execution details for a run at the bottom of the run page, and your organization's [total usage](#usage-page) in Settings. -<%= image "test_executions.png", alt: "Test executions run page" %> - -## Usage page - -The [Usage page](https://buildkite.com/organizations/~/usage) is available on every Buildkite plan and shows a breakdown of job minutes and test executions for your organization. - -The [test executions usage page](https://buildkite.com/organizations/~/usage/test_executions) graphs the total executions over the organization's billing periods. It includes a breakdown of usage by suite and a CSV download of usage over the period. - -Your organization's usage is also accessible in the [GraphQL API](/docs/apis/graphql/cookbooks/organizations#query-the-usage-api). diff --git a/pages/test_engine/test_ownership.md b/pages/test_engine/test_ownership.md index f8a676790c..08070dc9e5 100644 --- a/pages/test_engine/test_ownership.md +++ b/pages/test_engine/test_ownership.md @@ -1,6 +1,6 @@ # Test ownership -Test ownership is critical in adopting a healthy testing culture at your organization. Defining one or more teams as test owners allows these teams to become accountable for maintaining a fast and reliable test suite, ensuring confidence when you deploy your code. +Test ownership is critical in adopting a healthy testing culture at your organization. Defining one or more teams as test owners allows these teams to become accountable for maintaining tests within your test suite, ensuring it is fast and reliable, and providing confidence when you deploy your code. Customers on the [Pro and Enterprise plans](https://buildkite.com/pricing) can assign test ownership to [teams](/docs/test-engine/permissions#manage-teams-and-permissions). diff --git a/pages/test_engine/test_splitting.md b/pages/test_engine/test_splitting.md index f80ce28693..7a978243fe 100644 --- a/pages/test_engine/test_splitting.md +++ b/pages/test_engine/test_splitting.md @@ -1,108 +1,16 @@ -# Configuring test splitting +--- +toc: false +--- -Test splitting is the process of partitioning a test suite to run in parallel across multiple Buildkite agents. Buildkite maintains its open source Test Splitter ([test-splitter](https://github.com/buildkite/test-splitter)) tool. This tool uses your Buildkite Test Engine test suite data to intelligently partition tests throughout your test suite into multiple sets, such that each set of tests runs in parallel across your agents. This process is known as _orchestration_ and results in a _test plan_, where a test plan defines which tests are run on which agents. Currently, the Test Splitter tool only supports RSpec. +# Test splitting -## Dependencies +Test splitting is a feature that: -The Test Splitter relies on execution timing data captured by the Buildkite test collectors from previous builds to partition your tests evenly across your agents. Therefore, you will need to configure the [Ruby test collector](./ruby-collectors) for your test suite. +- Allows you to substantially reduce the duration of your overall build times, especially for pipelines with highly complex and computationally intensive test suites. +- Intelligently partitions your test suites to run in parallel across multiple agents, with the intent to even out test execution times across your agents, such that each agent will complete its partitioned test executions at approximately similar times. -## Installation +The following image from Test Engine's test splitting setup page illustrates how this feature works. In this example, _without_ test splitting, the test suite build time would take as long as it takes for the slowest combination of tests to run on a single partition (Buildkite job), which is 10 minutes. Since the sum of all test executions across all agents is 16 minutes, _with_ test splitting implemented, all four partitions would take approximately 4 minutes to run, such that the overall test suite build time would be approximately 4 minutes, or a 6-minute reduction. -The [latest version of test-splitter](https://github.com/buildkite/test-splitter/releases) can be downloaded from GitHub for installation to your agent/s. Binaries are available for both Mac and Linux with 64-bit ARM and AMD architectures. Download the executable and make it available in your testing environment. +<%= image "setup-page-summary.png", alt: "The test splitting setup page in Test Engine" %> -## Using the test splitter - -Once you have downloaded the test-splitter binary and it's executable in your Buildkite pipeline, you'll need to configure some additional environment variables for the test splitter to function. You can then update your pipeline step to call test-splitter instead of calling RSpec to run your tests. - -### Configure environment variables - -The Test Splitter tool uses a number of [predefined](#predefined-environment-variables), [mandatory](#mandatory-environment-variables), and [optional](#optional-environment-variables) environment variables. - - - -#### Predefined environment variables - -By default, the following predefined environment variables are available to your testing environment and do not need any further configuration. If, however, you use Docker or some other type of containerization tool to run your tests, and you wish to use these predefined environment variables in these tests, you may need to expose these environment variables to your containers. - - - - <% TEST_SPLITTING_ENV['predefined'].each do |var| %> - - - - - <% end %> - -
- <%= var['name'] %> # - - <% var['desc'].each do |d| %> - <%= render_markdown(text: d) %> - <% end %> -
- - - -#### Mandatory environment variables - -The following mandatory environment variables must be set. - - - - <% TEST_SPLITTING_ENV['mandatory'].each do |var| %> - - - - - <% end %> - -
- <%= var['name'] %> # - - <% var['desc'].each do |d| %> - <%= render_markdown(text: d) %> - <% end %> -
- - - -#### Optional environment variables - -The following optional environment variables can also be used to configure the Test Splitter's behavior. - - - - <% TEST_SPLITTING_ENV['optional'].each do |var| %> - - - - - <% end %> - -
- <%= var['name'] %> # -

- Default: - <%= var['default'] %> -

- 		- <% var['desc'].each do |d| %> - <%= render_markdown(text: d) %> - <% end %> -
- - -### Update the pipeline step - -With the environment variables configured, you can now update your pipeline step to use test-splitter instead of running RSpec. The following example pipeline step demonstrates how to partition your test suite across 10 nodes. - -``` -steps: - - name: "RSpec" - command: ./test-splitter - parallelism: 10 - env: - BUILDKITE_SPLITTER_SUITE_SLUG: my-suite - BUILDKITE_SPLITTER_API_ACCESS_TOKEN: your-secret-token -``` -{: codeblock-file="pipeline.yml"} +Learn more about how to configure test splitting for your test suites in [Configuring test splitting](/docs/test-engine/test-splitting/configuring). diff --git a/pages/test_engine/test_splitting/client_installation.md b/pages/test_engine/test_splitting/client_installation.md new file mode 100644 index 0000000000..f929ce64d7 --- /dev/null +++ b/pages/test_engine/test_splitting/client_installation.md @@ -0,0 +1,69 @@ +# Installing the Test Engine Client + +This page provides instructions on how to install the Test Engine Client via installers provided by Buildkite. + +If you need to install this tool on a system without an installer listed below, you'll need to perform a manual installation using one of the binaries from [Test Engine Client's releases page](https://github.com/buildkite/test-engine-client/releases/latest). Once you have the binary, make it executable in your pipeline. + +## Debian + +1. Ensure you have curl and gpg installed first: + + ```shell + apt update && apt install curl gpg -y + ``` + +1. Install the registry signing key: + + ```shell + curl -fsSL "https://packages.buildkite.com/buildkite/test-engine-client-deb/gpgkey" | gpg --dearmor -o /etc/apt/keyrings/buildkite_test-engine-client-deb-archive-keyring.gpg + ``` + +1. Configure the registry: + + ```shell + echo -e "deb [signed-by=/etc/apt/keyrings/buildkite_test-engine-client-deb-archive-keyring.gpg] https://packages.buildkite.com/buildkite/test-engine-client-deb/any/ any main\ndeb-src [signed-by=/etc/apt/keyrings/buildkite_test-engine-client-deb-archive-keyring.gpg] https://packages.buildkite.com/buildkite/test-engine-client-deb/any/ any main" > /etc/apt/sources.list.d/buildkite-buildkite-test-engine-client-deb.list + ``` + +1. Install the package: + + ```shell + apt update && apt install bktec + ``` + +## Red Hat + +1. Configure the registry: + + ```shell + echo -e "[test-engine-client-rpm]\nname=Test Engine Client - rpm\nbaseurl=https://packages.buildkite.com/buildkite/test-engine-client-rpm/rpm_any/rpm_any/\$basearch\nenabled=1\nrepo_gpgcheck=1\ngpgcheck=0\ngpgkey=https://packages.buildkite.com/buildkite/test-engine-client-rpm/gpgkey\npriority=1" > /etc/yum.repos.d/test-engine-client-rpm.repo + ``` + +2. Install the package: + + ```shell + dnf install -y bktec + ``` + +## macOS + +The Test Engine Client can be installed using [Homebrew](https://brew.sh) with [Buildkite tap formulae](https://github.com/buildkite/homebrew-buildkite). To install, run: + +```shell +brew tap buildkite/buildkite && brew install buildkite/buildkite/bktec +``` + +## Docker + +You can run the Test Engine Client inside a Docker container using the official image in [Docker Hub](https://hub.docker.com/r/buildkite/test-engine-client/tags). + +To run the client using Docker: + +```shell +docker run buildkite/test-engine-client +``` + +Or, to add the Test Engine Client binary to your Docker image, include the following in your Dockerfile: + +```dockerfile +COPY --from=buildkite/test-engine-client /usr/local/bin/bktec /usr/local/bin/bktec +``` diff --git a/pages/test_engine/test_splitting/configuring.md b/pages/test_engine/test_splitting/configuring.md new file mode 100644 index 0000000000..3c5fdcd23d --- /dev/null +++ b/pages/test_engine/test_splitting/configuring.md @@ -0,0 +1,172 @@ +# Configuring test splitting + +Buildkite maintains its open source Test Engine Client ([bktec](https://github.com/buildkite/test-engine-client)) tool. This tool uses your Test Engine test suite data to intelligently partition tests throughout your test suite into multiple sets, such that each set of tests runs in parallel across your agents. This results in a _test plan_, where a test plan defines which tests are run on which agents. Currently, the Test Engine Client tool only supports RSpec and Jest. + +## Dependencies + +The Test Engine Client relies on execution timing data captured by the test collectors from previous builds to partition your tests evenly across your agents. Therefore, you will need to configure the [Ruby test collector](/docs/test-engine/ruby-collectors) for your test suite if you are running RSpec, and [JavaScript test collector](/docs/test-engine/javascript-collectors) if you are running Jest. + +## Installation + +The Test Engine Client is supported on both Linux and macOS with 64-bit ARM and AMD architectures. You can install the client using the following installers: + +- [Debian](/docs/test-engine/test-splitting/client-installation#debian) +- [Red Hat](/docs/test-engine/test-splitting/client-installation#red-hat) +- [macOS](/docs/test-engine/test-splitting/client-installation#macos) +- [Docker](/docs/test-engine/test-splitting/client-installation#docker) + +If you need to install this tool on a system without an installer listed above, you'll need to perform a manual installation using one of the binaries from [Test Engine Client's releases page](https://github.com/buildkite/test-engine-client/releases/latest). Once you have the binary, make it executable in your pipeline. + +## Using the Test Engine Client + +Once you have downloaded the Test Engine Client (bktec) binary and it is executable in your pipeline, you'll need to configure some additional environment variables for the Test Engine Client to function. You can then update your pipeline step to call `bktec` instead of calling RSpec to run your tests. Learn more about how to do this in [Update the pipeline step](#using-the-test-engine-client-update-the-pipeline-step). + +### Configure environment variables + +The Test Engine Client tool uses a number of [predefined](#predefined-environment-variables) and [mandatory](#mandatory-environment-variables) environment variables, as well as several optional ones for either [RSpec](#optional-rspec-environment-variables) or [Jest](#optional-jest-environment-variables). + + + +#### Predefined environment variables + +By default, the following predefined environment variables are available to your testing environment and do not need any further configuration. If, however, you use Docker or some other type of containerization tool to run your tests, and you wish to use these predefined environment variables in these tests, you may need to expose these environment variables to your containers. + + + + <% TEST_SPLITTING_ENV['predefined'].each do |var| %> + + + + + <% end %> + +
+ <%= var['name'] %> # + + <% var['desc'].each do |d| %> + <%= render_markdown(text: d) %> + <% end %> +
+ + + +#### Mandatory environment variables + +The following mandatory environment variables must be set. + + + + <% TEST_SPLITTING_ENV['mandatory'].each do |var| %> + + + + + <% end %> + +
+ <%= var['name'] %> # + + <% var['desc'].each do |d| %> + <%= render_markdown(text: d) %> + <% end %> + + <% if var['note'].present? %> +
+ <% var['note'].each do |d| %> + <%= render_markdown(text: d) %> + <% end %> +
+ <% end %> +
+ + + +#### Optional RSpec environment variables + +The following optional RSpec environment variables can also be used to configure the Test Engine Client's behavior. + + + + <% TEST_SPLITTING_ENV['optional']['rspec'].each do |var| %> + + + + + <% end %> + +
+ <%= var['name'] %> # +

+ Default:
+ <%= var['default'] || "-" %> +

+ 		+ <% var['desc'].each do |d| %> + <%= render_markdown(text: d) %> + <% end %> + + <% if var['note'].present? %> +
+ <% var['note'].each do |d| %> + <%= render_markdown(text: d) %> + <% end %> +
+ <% end %> +
+ + + +#### Optional Jest environment variables + +The following optional Jest environment variables can also be used to configure the Test Engine Client's behavior. + + + + <% TEST_SPLITTING_ENV['optional']['jest'].each do |var| %> + + + + + <% end %> + +
+ <%= var['name'] %> # +

+ Default:
+ <%= var['default'] || "-" %> +

+ 		+ <% var['desc'].each do |d| %> + <%= render_markdown(text: d) %> + <% end %> + + <% if var['note'].present? %> +
+ <% var['note'].each do |d| %> + <%= render_markdown(text: d) %> + <% end %> +
+ <% end %> +
+ + +### Update the pipeline step + +With the environment variables configured, you can now update your pipeline step to run the Test Engine Client instead of running RSpec, or Jest directly. The following example pipeline step demonstrates how to partition your RSpec test suite across 10 nodes. + +``` +steps: + - name: "RSpec" + command: bktec + parallelism: 10 + env: + BUILDKITE_TEST_ENGINE_API_ACCESS_TOKEN: your-secret-token + BUILDKITE_TEST_ENGINE_RESULT_PATH: tmp/rspec-result.json + BUILDKITE_TEST_ENGINE_SUITE_SLUG: my-suite + BUILDKITE_TEST_ENGINE_RUNNER: rspec +``` +{: codeblock-file="pipeline.yml"} + +## API rate limits + +There is a limit on the number of API requests that the Test Engine Client can make to the server. This limit is 10,000 requests per minute per Buildkite organization. When this limit is reached, the Test Engine Client will pause and wait until the next minute is reached before retrying the request. This rate limit is independent of the [REST API rate limits](/docs/apis/rest-api/limits), and only applies to the Test Engine Client's interactions with the Test Splitting API. diff --git a/pages/test_engine/test_suites.md b/pages/test_engine/test_suites.md index 5ffdee728a..aef1e61a2d 100644 --- a/pages/test_engine/test_suites.md +++ b/pages/test_engine/test_suites.md @@ -1,18 +1,24 @@ # Configuring test suites -In Test Engine, a test _suite_ is a collection of tests. A suite has a _run_, which is the execution of tests in a suite. A suite's run is analogous to a pipeline's build. +In Test Engine, a _test suite_ (or _suite_) is a collection of tests. A suite has a _run_, which is the execution of tests in a suite. A pipeline's build may create one or more of these runs. Many organizations set up one suite per test framework, for example one suite for RSpec, and another suite for Jest. Others use a common standard, such as JUnit XML, to combine tests from multiple frameworks to set up custom backend and frontend suites. Each suite inside Test Engine has a unique API token that you can use to route test information to the correct suite. Pipelines and test suites do not need to have a one-to-one relationship. +To start configuring your test suite, you first need to have configured the appropriate _test collectors_ for your development project. Learn more about how to do this from the [Get started](/docs/test-engine#get-started) section of these docs. + To delete a suite, or regenerate its API token, go to suite settings. ## Parallelized builds -Test Engine works even when your test runs are split across different agents by de-duplicating against the Test Engine API token and unique build identifier. +In CI/CD, a build's tests can be made to run in parallel using features of your own CI/CD pipeline or workflow tool. Parallelized pipeline/workflow builds typically run and complete faster than builds which are not parallelized. + +In Buildkite Pipelines, you can run tests in parallel when they are configured as [parallel jobs](https://buildkite.com/docs/tutorials/parallel-builds#parallel-jobs). -The information that serves as a unique build identifier differs between CI environments. For details, see `run_env[key]` environment variables on our [CI environments page](/docs/test-engine/ci-environments). +> 📘 +> When tests are run in parallel across multiple agents, they can be grouped into the same run by defining the same `run_env[key]` environment variable. Learn more about this environment variable and others in [CI environments](/docs/test-engine/ci-environments). +> You can further speed up the duration of parallelized builds across multiple agents by implementing [test splitting](/docs/test-engine/test-splitting). ## Compare across branches diff --git a/pages/test_engine/usage_and_billing.md b/pages/test_engine/usage_and_billing.md new file mode 100644 index 0000000000..c814a12229 --- /dev/null +++ b/pages/test_engine/usage_and_billing.md @@ -0,0 +1,43 @@ +# Usage and billing + +Test Engine is designed to optimize your test suites through the management of your tests. + +## Managed tests + +Each and every test that can be uniquely identified by its combination of test suite, scope, and name, is a _managed test_, which in turn is used for billing purposes in Test Engine. + +For example, each of the following three tests are unique managed tests: + +- Test Suite 1 - here.is.scope.one - Login Test name +- Test Suite 1 - here.is.another.scope - Login Test name +- Test Suite 2 - here.is.scope.one - Login Test name + +Test Engine conducts the following on each managed test: + +- Tracks its history +- Calculates its [flakiness](/docs/test-engine/test-suites#detecting-flaky-tests) +- Maintains its state (for example, [Enterprise plan](https://buildkite.com/pricing) customers can quarantine tests by disabling them under certain conditions) +- Attributes [ownership by team](/docs/test-engine/test-ownership) + +For billing purposes, Buildkite measures usage by calculating the number of managed tests that have executed (run) at least once each day, and then bills based on the 90th percentile of this usage for the month. This billing method ensures that occasional spikes in usage, such as those caused by refactoring, don't result in excessive charges. + +> 📘 +> Be aware that if a specific managed test has run multiple times on a specific day, then this only counts once towards the usage measurement for that day. + +## Test executions + +Some legacy Buildkite plans measure usage based on the _total number of times_ a test was executed (run). + +You can find the test execution details for a run at the bottom of the run page, and your organization's [total usage](#usage-page) in Settings. + +<%= image "test_executions.png", alt: "Test executions run page" %> + +## Usage page + +The [Usage page](https://buildkite.com/organizations/~/usage?product=test_engine) is available on every Buildkite plan, and shows a breakdown of all billable usage for your organization including managed tests and test executions. + +The [managed tests usage page](https://buildkite.com/organizations/~/usage/test_engine_managed_tests) graphs the maximum number of unique tests per day over the organization's billing periods. This page includes a breakdown of usage by suite and a CSV download of usage over the period. + +The [test executions usage page](https://buildkite.com/organizations/~/usage/test_executions) graphs the total executions over the organization's billing periods. This page includes a breakdown of usage by suite and a CSV download of usage over the period. + +Your organization's usage is also accessible in the [GraphQL API](/docs/apis/graphql/cookbooks/organizations#query-the-usage-api). diff --git a/vale/styles/Buildkite/h1-h6_sentence_case.yml b/vale/styles/Buildkite/h1-h6_sentence_case.yml index 79c1175c99..8a7931b2fa 100644 --- a/vale/styles/Buildkite/h1-h6_sentence_case.yml +++ b/vale/styles/Buildkite/h1-h6_sentence_case.yml @@ -110,6 +110,7 @@ exceptions: - Installation - JavaScript - Jenkins + - Jest - JSON - JUnit - KMS @@ -148,6 +149,7 @@ exceptions: - SSH - SSO - Test Engine + - Test Engine Client - TESTOWNER - Ubuntu - URL diff --git a/vale/styles/vocab.txt b/vale/styles/vocab.txt index 58544c5abf..2bf1f625b1 100644 --- a/vale/styles/vocab.txt +++ b/vale/styles/vocab.txt @@ -29,6 +29,7 @@ Basecamp Basscss Bazel Bitium +bktec boolean Boomper Buildbox @@ -90,6 +91,7 @@ githooks GitLab globbing gotestsum +gpg Gradle Graviton gz