BenchBase (Java with Maven) #2479
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Template: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven | |
| # | |
| # Useful links | |
| # - GitHub Actions: https://docs.github.com/en/actions/learn-github-actions/introduction-to-github-actions | |
| # - Service containers: https://docs.github.com/en/actions/guides/creating-postgresql-service-containers | |
| # | |
| # The CI jobs are set up as follows: | |
| # - One job to build and upload artifacts. | |
| # - One job per DBMS test suite. | |
| # - One job to build/test the docker images. | |
| # - One job to publish the docker image. | |
| name: BenchBase (Java with Maven) | |
| on: | |
| push: | |
| branches: [ main ] | |
| # Generate new docker images on release tags. | |
| tags: | |
| - 'v*' | |
| pull_request: | |
| branches: [ main ] | |
| # Run these workflows on a schedule so that docker images are regularly updated for security patches. | |
| schedule: | |
| - cron: "1 0 * * *" | |
| # Give us a button to allow running the workflow on demand for testing. | |
| workflow_dispatch: | |
| inputs: | |
| tags: | |
| description: 'Manual Workflow Run' | |
| required: false | |
| type: string | |
| env: | |
| POM_VERSION: 2023-SNAPSHOT | |
| JAVA_VERSION: 21 | |
| ERRORS_THRESHOLD: 0.01 | |
| PYTHON_VERSION: "3.10" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| compile-and-test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{env.JAVA_VERSION}} | |
| cache: 'maven' | |
| distribution: 'temurin' | |
| - name: Check formatting | |
| run: mvn -B fmt:check --file pom.xml | |
| - name: Compile with Maven | |
| run: mvn -B compile test-compile --file pom.xml | |
| - name: Test with Maven | |
| run: mvn -B test --file pom.xml | |
| package-and-upload: | |
| needs: compile-and-test | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| profile: [ 'cockroachdb', 'mariadb', 'mysql', 'oracle', 'phoenix', 'postgres', 'spanner', 'sqlite', 'sqlserver' ] | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{env.JAVA_VERSION}} | |
| cache: 'maven' | |
| distribution: 'temurin' | |
| - name: Package and verify with Maven | |
| run: | | |
| mvn -B package verify -P ${{matrix.profile}} --file pom.xml -DskipTests -D descriptors=src/main/assembly/tgz.xml | |
| - name: Upload TGZ artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchbase-${{matrix.profile}} | |
| path: target/benchbase-${{matrix.profile}}.tgz | |
| # Needed for running a customized service containers using docker/*/up.sh scripts. | |
| # See postgres example below. | |
| # https://github.com/actions/runner/issues/2139 | |
| - name: Package docker-compose configs | |
| if: ${{ matrix.profile == 'postgres' }} | |
| run: | | |
| tar czvpf docker-compose-${{matrix.profile}}.tar.gz docker/${{matrix.profile}}-latest | |
| - name: Upload docker-compose configs | |
| if: ${{ matrix.profile == 'postgres' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-compose-${{matrix.profile}} | |
| path: docker-compose-${{matrix.profile}}.tar.gz | |
| ## ---------------------------------------------------------------------------------- | |
| ## SQLITE | |
| ## ---------------------------------------------------------------------------------- | |
| sqlite: | |
| needs: package-and-upload | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # BROKEN: tpch | |
| benchmark: [ 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ] | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchbase-sqlite | |
| - name: Extract artifact | |
| run: | | |
| tar xvzf benchbase-sqlite.tgz --strip-components=1 | |
| - name: Delete artifact | |
| run: | | |
| rm -rf benchbase-sqlite.tgz | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{env.JAVA_VERSION}} | |
| distribution: 'temurin' | |
| - name: Run benchmark | |
| run: | | |
| # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation | |
| # In this case, we load the tpcc data. | |
| if [[ ${{matrix.benchmark}} == templated ]] || [[ ${{matrix.benchmark}} == tpcc ]]; then | |
| # Disable synchronous mode for sqlite tpcc data loading to save some time. | |
| java -jar benchbase.jar -b tpcc -c config/sqlite/sample_tpcc_nosync_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| # Run the templated benchmark. | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json | |
| elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then | |
| echo "The ${{matrix.benchmark}} benchmark is not supported for sqlite." | |
| exit 0 | |
| else | |
| if [ ${{matrix.benchmark}} == chbenchmark ]; then | |
| # Disable synchronous mode for sqlite tpcc data loading to save some time. | |
| java -jar benchbase.jar -b tpcc -c config/sqlite/sample_tpcc_nosync_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| fi | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json | |
| fi | |
| # FIXME: Reduce the error rate so we don't need these overrides. | |
| results_benchmark=${{matrix.benchmark}} | |
| if [ ${{matrix.benchmark}} == auctionmark ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| elif [ ${{matrix.benchmark}} == resourcestresser ]; then | |
| ERRORS_THRESHOLD=0.05 | |
| elif [ ${{matrix.benchmark}} == smallbank ]; then | |
| ERRORS_THRESHOLD=0.03 | |
| elif [ ${{matrix.benchmark}} == tatp ]; then | |
| ERRORS_THRESHOLD=0.05 | |
| elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| results_benchmark=tpcc | |
| fi | |
| ./scripts/check_latest_benchmark_results.sh $results_benchmark | |
| ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD | |
| ## ---------------------------------------------------------------------------------- | |
| ## MARIADB | |
| ## ---------------------------------------------------------------------------------- | |
| mariadb: | |
| needs: package-and-upload | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # FIXME: Add tpch back in (#333). | |
| benchmark: [ 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'twitter', 'voter', 'wikipedia', 'ycsb' ] | |
| services: | |
| mariadb: # https://hub.docker.com/_/mariadb | |
| image: mariadb:latest | |
| env: | |
| MARIADB_ROOT_PASSWORD: rootyMcRooty | |
| MARIADB_DATABASE: benchbase | |
| MARIADB_USER: admin | |
| MARIADB_PASSWORD: password | |
| # Use status instead of ping due to exit code issues: https://jira.mariadb.org/browse/MDEV-31550 | |
| options: >- | |
| --health-cmd "mariadb-admin status -uroot -prootyMcRooty" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 3306:3306 | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchbase-mariadb | |
| - name: Extract artifact | |
| run: | | |
| tar xvzf benchbase-mariadb.tgz --strip-components=1 | |
| - name: Delete artifact | |
| run: | | |
| rm -rf benchbase-mariadb.tgz | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{env.JAVA_VERSION}} | |
| distribution: 'temurin' | |
| - name: Run benchmark | |
| env: | |
| MARIADB_PORT: ${{ job.services.mariadb.ports[3306] }} | |
| run: | | |
| mysql -h127.0.0.1 -P$MARIADB_PORT -uadmin -ppassword -e "DROP DATABASE IF EXISTS benchbase; CREATE DATABASE benchbase" | |
| # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation | |
| # In this case, we load the tpcc data. | |
| if [[ ${{matrix.benchmark}} == templated ]]; then | |
| java -jar benchbase.jar -b tpcc -c config/mariadb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mariadb/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json | |
| elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then | |
| # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh | |
| java -jar benchbase.jar -b tpcc -c config/mariadb/sample_tpcc_config.xml --create=true --load=true | |
| (sleep 10 && ./scripts/interrupt-docker-db-service.sh mariadb) & | |
| java -jar benchbase.jar -b tpcc -c config/mariadb/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json | |
| else | |
| if [ ${{matrix.benchmark}} == chbenchmark ]; then | |
| java -jar benchbase.jar -b tpcc -c config/mariadb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| fi | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mariadb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json | |
| fi | |
| # FIXME: Reduce the error rate so we don't need these overrides. | |
| results_benchmark=${{matrix.benchmark}} | |
| if [ ${{matrix.benchmark}} == auctionmark ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| elif [ ${{matrix.benchmark}} == tatp ]; then | |
| ERRORS_THRESHOLD=0.05 | |
| elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| results_benchmark=tpcc | |
| fi | |
| ./scripts/check_latest_benchmark_results.sh $results_benchmark | |
| ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD | |
| ## ---------------------------------------------------------------------------------- | |
| ## MYSQL | |
| ## ---------------------------------------------------------------------------------- | |
| mysql: | |
| needs: package-and-upload | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| benchmark: [ 'anonymization', 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ] | |
| services: | |
| mysql: # https://hub.docker.com/_/mysql | |
| image: mysql:latest | |
| env: | |
| MYSQL_ROOT_PASSWORD: rootyMcRooty | |
| MYSQL_DATABASE: benchbase | |
| MYSQL_USER: admin | |
| MYSQL_PASSWORD: password | |
| options: >- | |
| --health-cmd "mysqladmin ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 3306:3306 | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchbase-mysql | |
| - name: Extract artifact | |
| run: | | |
| tar xvzf benchbase-mysql.tgz --strip-components=1 | |
| - name: Delete artifact | |
| run: | | |
| rm -rf benchbase-mysql.tgz | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{env.JAVA_VERSION}} | |
| distribution: 'temurin' | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{env.PYTHON_VERSION}} | |
| - name: Install Python dependencies | |
| working-directory: ./scripts/anonymization | |
| run: | | |
| if [[ ${{matrix.benchmark}} == anonymization ]]; then | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| else | |
| echo "Dependency installation not necessary for benchmark" | |
| fi | |
| - name: Run benchmark | |
| env: | |
| MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} | |
| run: | | |
| mysql -h127.0.0.1 -P$MYSQL_PORT -uadmin -ppassword -e "DROP DATABASE IF EXISTS benchbase; CREATE DATABASE benchbase" | |
| # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation | |
| # In this case, we load the tpcc data. | |
| if [[ ${{matrix.benchmark}} == templated ]]; then | |
| java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mysql/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json | |
| # For anonymization, we load tpcc and anonymize a single table. The workload itself is not executed | |
| # FIXME: 'exit 0' is called because there is no benchmark executed and analyzed. Must be removed once the Anonymization script is | |
| # fully implemented. See Pull Request 455. | |
| elif [[ ${{matrix.benchmark}} == anonymization ]]; then | |
| java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| java -jar benchbase.jar -b tpcc -c config/mysql/sample_${{matrix.benchmark}}_config.xml --anonymize=true | |
| exit 0 | |
| elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then | |
| # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh | |
| java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --create=true --load=true | |
| (sleep 10 && ./scripts/interrupt-docker-db-service.sh mysql) & | |
| java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json | |
| else | |
| if [ ${{matrix.benchmark}} == chbenchmark ]; then | |
| java -jar benchbase.jar -b tpcc -c config/mysql/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| fi | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/mysql/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json | |
| fi | |
| # FIXME: Reduce the error rate so we don't need these overrides. | |
| results_benchmark=${{matrix.benchmark}} | |
| if [ ${{matrix.benchmark}} == auctionmark ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| elif [ ${{matrix.benchmark}} == tatp ]; then | |
| ERRORS_THRESHOLD=0.05 | |
| elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| results_benchmark=tpcc | |
| fi | |
| ./scripts/check_latest_benchmark_results.sh $results_benchmark | |
| ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD | |
| ## ---------------------------------------------------------------------------------- | |
| ## ORACLE | |
| ## ---------------------------------------------------------------------------------- | |
| oracle: | |
| needs: package-and-upload | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| benchmark: [ 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ] | |
| services: | |
| oracle: | |
| image: gvenzl/oracle-xe:21.3.0-slim-faststart | |
| ports: | |
| - "1521:1521" | |
| - "5500:5500" | |
| env: | |
| ORACLE_PASSWORD: password | |
| ORACLE_CHARACTERSET: AL32UTF8 | |
| APP_USER: benchbase | |
| APP_USER_PASSWORD: password | |
| options: >- | |
| --name oracle | |
| --health-cmd "echo exit | sqlplus benchbase/password@xepdb1 | grep Connected" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| --health-start-period 5s | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchbase-oracle | |
| - name: Extract artifact | |
| run: | | |
| tar xvzf benchbase-oracle.tgz --strip-components=1 | |
| - name: Delete artifact | |
| run: | | |
| rm -rf benchbase-oracle.tgz | |
| - name: Set up user reset script | |
| run: | | |
| docker cp config/oracle/scripts/reset.sql oracle:/opt/oracle/reset.sql | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{env.JAVA_VERSION}} | |
| distribution: 'temurin' | |
| - name: Run benchmark | |
| run: | | |
| docker exec oracle sqlplus "sys/password@xepdb1 as sysdba" @reset.sql | |
| # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation | |
| # In this case, we load the tpcc data. | |
| if [[ ${{matrix.benchmark}} == templated ]]; then | |
| java -jar benchbase.jar -b tpcc -c config/oracle/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/oracle/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json | |
| elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then | |
| # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh | |
| java -jar benchbase.jar -b tpcc -c config/oracle/sample_tpcc_config.xml --create=true --load=true | |
| (sleep 10 && ./scripts/interrupt-docker-db-service.sh oracle) & | |
| java -jar benchbase.jar -b tpcc -c config/oracle/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json | |
| else | |
| if [ ${{matrix.benchmark}} == chbenchmark ]; then | |
| java -jar benchbase.jar -b tpcc -c config/oracle/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| fi | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/oracle/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json | |
| fi | |
| # FIXME: Reduce the error rate so we don't need these overrides. | |
| results_benchmark=${{matrix.benchmark}} | |
| if [ ${{matrix.benchmark}} == auctionmark ]; then | |
| ERRORS_THRESHOLD=0.04 | |
| elif [ ${{matrix.benchmark}} == resourcestresser ]; then | |
| ERRORS_THRESHOLD=0.04 | |
| elif [ ${{matrix.benchmark}} == tatp ]; then | |
| ERRORS_THRESHOLD=0.05 | |
| elif [ ${{matrix.benchmark}} == tpcc ]; then | |
| ERRORS_THRESHOLD=0.03 | |
| elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| results_benchmark=tpcc | |
| elif [ ${{matrix.benchmark}} == wikipedia ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| fi | |
| ./scripts/check_latest_benchmark_results.sh $results_benchmark | |
| ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD | |
| ## ---------------------------------------------------------------------------------- | |
| ## POSTGRESQL | |
| ## ---------------------------------------------------------------------------------- | |
| postgresql: | |
| needs: package-and-upload | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| benchmark: [ 'auctionmark', 'chbenchmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ] | |
| steps: | |
| # Note: we download just the docker-compose scripts/configs rather than the | |
| # whole source code repo for better testing. | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: docker-compose-postgres | |
| - name: Extract docker-compose artifacts | |
| run: | | |
| tar xvzf docker-compose-postgres.tar.gz | |
| # Use docker-compose to start the postgres service so we can modify the | |
| # command line args to include extensions. | |
| # https://github.com/actions/runner/issues/2139 | |
| - name: Start custom postgres service | |
| run: | | |
| ./docker/postgres-latest/up.sh --quiet-pull postgres | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchbase-postgres | |
| - name: Extract artifact | |
| run: | | |
| tar xvzf benchbase-postgres.tgz --strip-components=1 | |
| - name: Delete artifact | |
| run: | | |
| rm -rf benchbase-postgres.tgz | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{env.JAVA_VERSION}} | |
| distribution: 'temurin' | |
| - name: Run benchmark | |
| run: | | |
| PGPASSWORD=password dropdb -h localhost -U admin benchbase --if-exists | |
| PGPASSWORD=password createdb -h localhost -U admin benchbase | |
| # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation | |
| # In this case, we load the tpcc data. | |
| if [[ ${{matrix.benchmark}} == templated ]]; then | |
| java -jar benchbase.jar -b tpcc -c config/postgres/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/postgres/sample_${{matrix.benchmark}}_config.xml -im 1000 -mt advanced --create=false --load=false --execute=true --json-histograms results/histograms.json | |
| elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then | |
| # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh | |
| java -jar benchbase.jar -b tpcc -c config/postgres/sample_tpcc_config.xml --create=true --load=true | |
| (sleep 10 && ./scripts/interrupt-docker-db-service.sh postgres) & | |
| java -jar benchbase.jar -b tpcc -c config/postgres/sample_tpcc_config.xml -im 1000 -mt advanced --execute=true --json-histograms results/histograms.json | |
| else | |
| if [ ${{matrix.benchmark}} == chbenchmark ]; then | |
| java -jar benchbase.jar -b tpcc -c config/postgres/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| fi | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/postgres/sample_${{matrix.benchmark}}_config.xml -im 1000 -mt advanced --create=true --load=true --execute=true --json-histograms results/histograms.json | |
| fi | |
| # FIXME: Reduce the error rate so we don't need these overrides. | |
| results_benchmark=${{matrix.benchmark}} | |
| if [ ${{matrix.benchmark}} == auctionmark ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| elif [ ${{matrix.benchmark}} == tatp ]; then | |
| ERRORS_THRESHOLD=0.05 | |
| elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| results_benchmark=tpcc | |
| fi | |
| ./scripts/check_latest_benchmark_results.sh $results_benchmark | |
| ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD | |
| # Running the monitor should create at least three files in the 'monitor' directory. | |
| if ![ $(find "./results/monitor" -maxdepth 1 -mindepth 1 | wc -l) -gt 2]; then | |
| echo "ERROR: Advanced monitoring unsuccessful, file directory and/or appropriate files not created." >&2 | |
| exit 1 | |
| fi | |
| - name: Stop custom postgres service | |
| run: | | |
| ./docker/postgres-latest/down.sh | |
| ## ---------------------------------------------------------------------------------- | |
| ## COCKROACHDB | |
| ## ---------------------------------------------------------------------------------- | |
| cockroachdb: | |
| needs: package-and-upload | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # TODO: Add tpcc-with-reconnects benchmark support | |
| # TODO: Add chbenchmark benchmark support | |
| benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ] | |
| services: | |
| cockroach: # https://hub.docker.com/repository/docker/timveil/cockroachdb-single-node | |
| image: timveil/cockroachdb-single-node:latest | |
| env: | |
| DATABASE_NAME: benchbase | |
| # TODO: Expand for additional config adjustments (See Also: #405, #519, #525) | |
| MEMORY_SIZE: .75 | |
| ports: | |
| - 26257:26257 | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchbase-cockroachdb | |
| - name: Extract artifact | |
| run: | | |
| tar xvzf benchbase-cockroachdb.tgz --strip-components=1 | |
| - name: Delete artifact | |
| run: | | |
| rm -rf benchbase-cockroachdb.tgz | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{env.JAVA_VERSION}} | |
| distribution: 'temurin' | |
| - name: Run benchmark | |
| run: | | |
| # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation | |
| # In this case, we load the tpcc data. | |
| if [[ ${{matrix.benchmark}} == templated ]]; then | |
| java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json | |
| elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then | |
| # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh | |
| java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --create=true --load=true | |
| (sleep 10 && ./scripts/interrupt-docker-db-service.sh cockroachdb) & | |
| java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --execute=true --json-histograms results/histograms.json | |
| else | |
| if [ ${{matrix.benchmark}} == chbenchmark ]; then | |
| java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| fi | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json | |
| fi | |
| # FIXME: Reduce the error rate so we don't need these overrides. | |
| results_benchmark=${{matrix.benchmark}} | |
| if [ ${{matrix.benchmark}} == auctionmark ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| elif [ ${{matrix.benchmark}} == tatp ]; then | |
| ERRORS_THRESHOLD=0.05 | |
| elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| results_benchmark=tpcc | |
| fi | |
| ./scripts/check_latest_benchmark_results.sh $results_benchmark | |
| ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD | |
| ## ---------------------------------------------------------------------------------- | |
| ## MSSQL | |
| ## ---------------------------------------------------------------------------------- | |
| sqlserver: | |
| needs: package-and-upload | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # TODO: add auctionmark and seats benchmark | |
| benchmark: [ 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpcc-with-reconnects', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ] | |
| services: | |
| sqlserver: | |
| image: mcr.microsoft.com/mssql/server:latest | |
| env: | |
| ACCEPT_EULA: Y | |
| SA_PASSWORD: SApassword1 | |
| options: >- | |
| --health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P SApassword1 -b -Q 'SELECT 1;'" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 1433:1433 | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: benchbase-sqlserver | |
| - name: Extract artifact | |
| run: | | |
| tar xvzf benchbase-sqlserver.tgz --strip-components=1 | |
| - name: Delete artifact | |
| run: | | |
| rm -rf benchbase-sqlserver.tgz | |
| - name: Set up JDK | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{env.JAVA_VERSION}} | |
| distribution: 'temurin' | |
| - name: Cleanup database | |
| uses: docker://mcr.microsoft.com/mssql-tools:latest | |
| with: | |
| entrypoint: /opt/mssql-tools/bin/sqlcmd | |
| args: -U sa -P SApassword1 -S sqlserver -b -Q "DROP DATABASE IF EXISTS benchbase;" | |
| - name: Setup database | |
| uses: docker://mcr.microsoft.com/mssql-tools:latest | |
| with: | |
| entrypoint: /opt/mssql-tools/bin/sqlcmd | |
| args: -U sa -P SApassword1 -S sqlserver -b -Q "CREATE DATABASE benchbase;" | |
| - name: Setup login | |
| uses: docker://mcr.microsoft.com/mssql-tools:latest | |
| with: | |
| entrypoint: /opt/mssql-tools/bin/sqlcmd | |
| args: -U sa -P SApassword1 -S sqlserver -Q "CREATE LOGIN benchuser01 WITH PASSWORD='P@ssw0rd';" | |
| - name: Setup access | |
| uses: docker://mcr.microsoft.com/mssql-tools:latest | |
| with: | |
| entrypoint: /opt/mssql-tools/bin/sqlcmd | |
| args: -U sa -P SApassword1 -S sqlserver -b -Q "USE benchbase; CREATE USER benchuser01 FROM LOGIN benchuser01; EXEC sp_addrolemember 'db_owner', 'benchuser01';" | |
| - name: Run benchmark | |
| # Note: user/pass should match those used in sample configs. | |
| run: | | |
| # For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation | |
| # In this case, we load the tpcc data. | |
| if [[ ${{matrix.benchmark}} == templated ]]; then | |
| java -jar benchbase.jar -b tpcc -c config/sqlserver/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlserver/sample_${{matrix.benchmark}}_config.xml -im 1000 -mt advanced --create=false --load=false --execute=true --json-histograms results/histograms.json | |
| elif [[ ${{matrix.benchmark}} == tpcc-with-reconnects ]]; then | |
| # See Also: WITH_SERVICE_INTERRUPTIONS=true docker/build-run-benchmark-with-docker.sh | |
| java -jar benchbase.jar -b tpcc -c config/sqlserver/sample_tpcc_config.xml --create=true --load=true | |
| (sleep 10 && ./scripts/interrupt-docker-db-service.sh sqlserver) & | |
| java -jar benchbase.jar -b tpcc -c config/sqlserver/sample_tpcc_config.xml -im 1000 -mt advanced --execute=true --json-histograms results/histograms.json | |
| else | |
| if [ ${{matrix.benchmark}} == chbenchmark ]; then | |
| java -jar benchbase.jar -b tpcc -c config/sqlserver/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json | |
| fi | |
| java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlserver/sample_${{matrix.benchmark}}_config.xml -im 1000 -mt advanced --create=true --load=true --execute=true --json-histograms results/histograms.json | |
| fi | |
| # FIXME: Reduce the error rate so we don't need these overrides. | |
| results_benchmark=${{matrix.benchmark}} | |
| if [ ${{matrix.benchmark}} == tatp ]; then | |
| ERRORS_THRESHOLD=0.05 | |
| elif [ ${{matrix.benchmark}} == tpcc-with-reconnects ]; then | |
| ERRORS_THRESHOLD=0.02 | |
| results_benchmark=tpcc | |
| fi | |
| ./scripts/check_latest_benchmark_results.sh $results_benchmark | |
| ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD | |
| # Running the monitor should create at least three files in the 'monitor' directory. | |
| if ![ $(find "./results/monitor" -maxdepth 1 -mindepth 1 | wc -l) -gt 2]; then | |
| echo "ERROR: Advanced monitoring unsuccessful, file directory and/or appropriate files not created." >&2 | |
| exit 1 | |
| fi | |
| ## ---------------------------------------------------------------------------------- | |
| ## Docker Build Test Publish | |
| ## ---------------------------------------------------------------------------------- | |
| docker-build-test-publish: | |
| runs-on: ubuntu-latest | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| BENCHBASE_PROFILES: 'cockroachdb mariadb mysql oracle phoenix postgres spanner sqlite sqlserver' | |
| CONTAINER_REGISTRY_NAME: ${{ secrets.ACR_LOGINURL }} | |
| services: | |
| postgres: # https://hub.docker.com/_/postgres | |
| image: postgres:latest | |
| env: | |
| POSTGRES_DB: benchbase | |
| POSTGRES_USER: admin | |
| POSTGRES_PASSWORD: password | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| steps: | |
| - name: Setup postgres test DB | |
| run: | | |
| PGPASSWORD=password dropdb -h localhost -U admin benchbase --if-exists | |
| PGPASSWORD=password createdb -h localhost -U admin benchbase | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| # https://github.com/actions/cache/blob/master/examples.md#java---maven | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository | |
| key: setup-java-${{ runner.os }}-docker-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| setup-java-${{ runner.os }}-docker-maven- | |
| - name: Pull base image caches for PR builds | |
| if: ${{ github.ref != 'refs/heads/main' }} | |
| run: | | |
| docker pull benchbase.azurecr.io/benchbase-dev:latest || true | |
| docker pull benchbase.azurecr.io/benchbase:latest || true | |
| - name: Set NO_CACHE env var for main branch builds | |
| if: ${{ github.ref == 'refs/heads/main' }} | |
| run: | | |
| echo "NO_CACHE=true" >> $GITHUB_ENV | |
| - name: Build benchbase-dev image | |
| run: | | |
| ./docker/benchbase/build-dev-image.sh | |
| # Note: this script maps the local .m2 cache into the container. | |
| - name: Build the benchbase docker image with all profiles using the dev image | |
| env: | |
| SKIP_TESTS: 'false' | |
| DO_FORMAT_CHECKS: 'true' | |
| run: | | |
| ./docker/benchbase/build-full-image.sh | |
| - name: Run a basic test from the docker image against postgres test DB | |
| env: | |
| benchmark: noop | |
| run: | | |
| for image in benchbase benchbase-postgres; do | |
| # Adjust the sample config to talk to the container service instead of localhost. | |
| cat "config/postgres/sample_${benchmark}_config.xml" | sed -e 's/localhost:5432/postgres:5432/g' > /tmp/config.xml | |
| # Lookup the service container's docker network so we can place the benchbase container in it too. | |
| docker_network="$(docker ps --filter expose=5432 --format '{{.Networks}}')" | |
| # Map the adjusted config into the container and use it to run the test. | |
| rm -rf results | |
| mkdir -p results | |
| docker run --rm --name benchbase-postgres --network "$docker_network" \ | |
| --env BENCHBASE_PROFILE=postgres -v /tmp/config.xml:/tmp/config.xml -v "$PWD/results:/benchbase/results" \ | |
| "$image" -b "$benchmark" -c /tmp/config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json | |
| # Test that the results files were produced. | |
| ls results/${benchmark}_*.csv | |
| ./scripts/check_latest_benchmark_results.sh ${benchmark} | |
| ./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD | |
| done | |
| # Publish the docker image if the build/test was successful. | |
| # Only do this with approved PRs and if the login secrets are available. | |
| # Typically we expect to publish to benchbase.azurecr.io, | |
| # but setting ACR_LOGINURL to something else allows us to do testing on forks. | |
| - name: Log in to Docker Hub | |
| if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && env.CONTAINER_REGISTRY_NAME != '' }} | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ secrets.ACR_LOGINURL }} | |
| username: ${{ secrets.ACR_USERNAME }} | |
| password: ${{ secrets.ACR_PASSWORD }} | |
| - name: Push Docker image | |
| if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && env.CONTAINER_REGISTRY_NAME != '' }} | |
| run: | | |
| docker push -a ${{ secrets.ACR_LOGINURL}}/benchbase-dev | |
| docker push -a ${{ secrets.ACR_LOGINURL}}/benchbase | |
| for profile in $BENCHBASE_PROFILES; do | |
| docker push -a ${{ secrets.ACR_LOGINURL }}/benchbase-$profile | |
| done |