diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml index 4de76d1bd5..be2b035c18 100644 --- a/.github/workflows/ion-java-performance-regression-detector.yml +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -11,140 +11,110 @@ on: permissions: contents: read +env: + data_size: '50000' + test_data_id: 'generated-test-data' + jobs: - detect-regression: - name: Detect Regression + generate-test-data: + name: Generate Test Data runs-on: ubuntu-latest - steps: - - name: Set up JDK 11 - uses: actions/setup-java@0ab4596768b603586c0de567f2430c30f5b0d2b0 # v3.13.0 - with: - distribution: 'corretto' - java-version: 11 - - - name: Checkout ion-java from the new commit. - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 - with: - ref: ${{ github.event.pull_request.head.sha }} - path: ion-java-new - - - name: Build ion-java from the new commit - run: cd ion-java-new && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal - - - name: Checkout ion-java-benchmark-cli - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 - with: - repository: amazon-ion/ion-java-benchmark-cli - ref: master - path: ion-java-benchmark-cli - - - name: Build ion-java-benchmark-cli - run: cd ion-java-benchmark-cli && mvn clean install - - name: Checkout ion-data-generator - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 with: repository: amazon-ion/ion-data-generator ref: main - path: ion-data-generator - name: Build ion-data-generator - run: cd ion-data-generator && mvn clean install + run: mvn clean install - - name: Check the version of ion-java. - run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version - - - name: Generate test Ion Data + - name: Generate test data + env: + jar_file: target/ion-data-generator-1.0-SNAPSHOT.jar + schema_dir: tst/com/amazon/ion/workflow run: | mkdir -p testData - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedStruct.isl testData/testStructs.10n - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/nestedList.isl testData/testLists.10n - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/sexp.isl testData/testSexps.10n - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema01.isl testData/realWorldData01.10n - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema02.isl testData/realWorldData02.10n - java -jar ion-data-generator/target/ion-data-generator-1.0-SNAPSHOT.jar generate -S 50000 --input-ion-schema ion-data-generator/tst/com/amazon/ion/workflow/realWorldDataSchema03.isl testData/realWorldData03.10n + for test in nestedStruct nestedList sexp realWorldDataSchema01 realWorldDataSchema02 realWorldDataSchema03 + do + java -jar $jar_file generate -S ${{env.data_size}} --input-ion-schema $schema_dir/${test}.isl testData/${test}.10n + done - name: Upload test Ion Data to artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@v3 with: - name: test Ion Data + name: ${{env.test_data_id}} path: testData - - name: Benchmark ion-java from the new commit - run: | - mkdir -p benchmarkResults - cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults + detect-regression: + name: Detect Regression + runs-on: ubuntu-latest + needs: generate-test-data + strategy: + matrix: + test-data: ['nestedStruct', 'nestedList', 'sexp', 'realWorldDataSchema01', 'realWorldDataSchema02', 'realWorldDataSchema03'] + usage-patterns: [' read --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api streaming ',' read --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api dom ', ' write --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api streaming --io-type buffer --ion-length-preallocation 1 ', ' write --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api dom --io-type buffer --ion-length-preallocation 1 '] + fail-fast: false - - name: Upload benchmark results to artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + steps: + - name: Set up JDK 11 + uses: actions/setup-java@v3 with: - name: Benchmark result - path: benchmarkResults - - - name: Clean maven dependencies repository - run : rm -r /home/runner/.m2 + distribution: 'corretto' + java-version: 11 - - name: Checkout the current commit - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v3.6.0 + - name: Checkout ion-java-benchmark-cli + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 with: - repository: amazon-ion/ion-java + repository: amazon-ion/ion-java-benchmark-cli ref: master - path: ion-java - - - name: Build ion-java from the previous commit - run: cd ion-java && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal - - - name: Build ion-java-benchmark-cli - run: cd ion-java-benchmark-cli && mvn clean install + path: ion-java-benchmark-cli - - name: Check the version of ion-java - run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version + - name: Checkout ion-java from the previous commit + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + ref: ${{ github.base_ref }} + path: baseline - - name: Create directories for test data and benchmark results - run: | - mkdir -p benchmarkResults - mkdir -p testData + - name: Checkout ion-java from the new commit. + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + ref: ${{ github.head_ref }} + path: new - name: Download test Ion Data from artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@v3 with: - name: test Ion Data + name: generated-test-data path: testData - - name: Download benchmark results of ion-java from the new commit from artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - with: - name: Benchmark result - path: benchmarkResults - - - name: Benchmark ion-java from the previous commit and add the generated benchmark results to the existing directories - run: cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults + # Get the benchmark results from the base commit + - name: Build ion-java from the base commit + working-directory: baseline + run: | + git submodule init && git submodule update && ./gradlew clean publishToMavenLocal - - name: Upload new benchmark results directory to artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: Benchmark result - path: benchmarkResults + - name: Benchmark ion-java from the base commit + working-directory: ion-java-benchmark-cli + run: | + mvn clean install + mkdir /home/runner/work/ion-java/ion-java/benchmarkresults + java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} -r ion -o /home/runner/work/ion-java/ion-java/benchmarkresults/before.ion /home/runner/work/ion-java/ion-java/testData/${{matrix.test-data}}.10n - - name: Detect performance regression - id: regression_result + - name: Build ion-java from the new commit + working-directory: new run: | - result=true - cd benchmarkResults && for FILE in *; do message=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion | tee /dev/stderr) && if [ "$message" != "no regression detected" ]; then result=false; fi; done - echo "::set-output name=regression-result::$result" - if [ "$result" = "true" ]; then echo "No regression detected!" >> $GITHUB_STEP_SUMMARY; fi + git submodule init && git submodule update && ./gradlew clean publishToMavenLocal - - name: Upload comparison reports to the benchmark results directory - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: Benchmark result - path: benchmarkResults + - name: Benchmark ion-java from the new commit + working-directory: ion-java-benchmark-cli + run: | + mvn clean install + java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} -r ion -o /home/runner/work/ion-java/ion-java/benchmarkresults/after.ion /home/runner/work/ion-java/ion-java/testData/${{matrix.test-data}}.10n - - name: Fail the workflow if regression happened - env: - regression_detect: ${{steps.regression_result.outputs.regression-result}} - if: ${{ env.regression_detect == 'false' }} + #Detect regression + - name: Detect regression + working-directory: ion-java-benchmark-cli run: | - cd benchmarkResults && echo "| Benchmark command | GC Allocation Rate | Heap Usage | Speed |" >> $GITHUB_STEP_SUMMARY && echo "| ----------- | ----------- |----------- | ----------- |" >> $GITHUB_STEP_SUMMARY && for FILE in *; do regressionDetection=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion) && if [ "$regressionDetection" != "no regression detected" ]; then command=$(echo $FILE | sed "s/_/ /g") && read gc heapUsage speed <<< $( echo ${regressionDetection} | awk -F", " '{print $1" "$2" "$3}' ) && echo "|$command|$gc|$heapUsage|$speed|" >> $GITHUB_STEP_SUMMARY; fi; done - exit 1 \ No newline at end of file + a=$(java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous /home/runner/work/ion-java/ion-java/benchmarkresults/before.ion --benchmark-result-new /home/runner/work/ion-java/ion-java/benchmarkresults/after.ion) + if [ ! -z "$a" ]; then echo "${a}" >> $GITHUB_STEP_SUMMARY; exit 1; fi