From f14b24a845f7ff36519bdee5f64181f7ee389091 Mon Sep 17 00:00:00 2001 From: Elango Cheran Date: Thu, 14 Sep 2023 14:01:16 -0700 Subject: [PATCH] ICU-22324 Allow single Maven cache creation and multiple read-only usage --- .github/workflows/cache_retain.yml | 16 +++- .github/workflows/icu_ci.yml | 125 +++++++++++++++-------------- .github/workflows/icu_merge_ci.yml | 111 ++++++++++++++++++++++--- 3 files changed, 181 insertions(+), 71 deletions(-) diff --git a/.github/workflows/cache_retain.yml b/.github/workflows/cache_retain.yml index c468b93ffd2f..1931ca309c2c 100644 --- a/.github/workflows/cache_retain.yml +++ b/.github/workflows/cache_retain.yml @@ -26,6 +26,9 @@ on: # this cron schedule is set to run every 6 days to ensure retention - cron: '0 12 */6 * *' +env: + SHARED_MVN_ARGS: '--show-version --no-transfer-progress' + permissions: contents: read @@ -44,8 +47,15 @@ jobs: with: distribution: 'temurin' java-version: '11' - cache: maven + - name: Restore read-only cache of local Maven repository + uses: actions/cache/restore@v3 + id: cache + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Run Maven unit & integration tests run: | - cd icu4j/maven-build; - mvn --batch-mode verify + cd icu4j; + mvn ${SHARED_MVN_ARGS} verify diff --git a/.github/workflows/icu_ci.yml b/.github/workflows/icu_ci.yml index 6db39dd67d44..c20800a0f83a 100644 --- a/.github/workflows/icu_ci.yml +++ b/.github/workflows/icu_ci.yml @@ -17,7 +17,7 @@ on: # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow env: - MAVEN_ARGS: '--show-version --no-transfer-progress' + SHARED_MVN_ARGS: '--show-version --no-transfer-progress' permissions: contents: read @@ -39,12 +39,11 @@ jobs: # Regex note: (?! ... ) is a negative lookahead. Succeed if the pattern is not present. set +o pipefail && make doc 2>&1 | tee doxygen.log && ( ! grep -P 'warning:(?! .* file .?Doxyfile)' doxygen.log ) - # ICU4J build and unit test using Maven - icu4j-mvn-build-and-test: - strategy: - fail-fast: false - matrix: - java-version: [ '8', '11', '17' ] + # Initialize the Maven artifact cache + # + # This job is created according to the cache strategy of reuse from a single job: + # https://github.com/actions/cache/blob/main/caching-strategies.md#make-cache-read-only--reuse-cache-from-centralized-job + icu4j-mvn-init-cache: runs-on: ubuntu-latest steps: - name: Checkout and setup @@ -53,60 +52,31 @@ jobs: lfs: true - name: Checkout lfs objects run: git lfs pull + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: ${{ matrix.java-version }} - - name: ICU4J + java-version: '8' + # Download all of the artifacts needed for the code and build plugins, but + # exclude any needed by profiles depending on system artifacts + - name: Download all artifacts run: | cd icu4j; - mvn install - - name: List failures (if any) - run: | - cd icu4j && cat `find . -name surefire-reports -type d -exec grep -l -r --include="*.txt" FAILED {} \;`; - if: ${{ failure() }} - - # ICU4J build and unit tests using Maven + mvn ${SHARED_MVN_ARGS} dependency:go-offline -P '!old_jdk_taglet' + + # ICU4J build and unit test using Maven # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven - - # Run `test` to execute unit tests and ensure it is possible for tests to run even without packaging submodule - # dependencies as jar files - icu4j-test-maven: - name: Run unit tests with Maven for Java version - runs-on: ubuntu-latest - # Make this unit test target job depend on a later phase target job to prevent race condition when - # trying to persist the Maven cache to the Github cache, knowing that artifacts needed for - # the later phase `verify` are a superset of the artifacts needed for the earlier phase `test`. - needs: icu4j-verify-maven + icu4j-mvn-build-and-test: + needs: icu4j-mvn-init-cache strategy: fail-fast: false matrix: java-version: [ '8', '11', '17' ] - steps: - - name: Checkout and setup - uses: actions/checkout@v3 - with: - lfs: true - - name: Checkout lfs objects - run: git lfs pull - - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.java-version }} - cache: maven - - name: Run Maven test - run: | - cd icu4j; - mvn --batch-mode test - - # Run `verify` to ensure that `package` (creating .jar files) and `integration-test` (special setup for localespi tests) work - icu4j-verify-maven: - name: Run integration tests with Maven for Java version runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - java-version: [ '8', '11', '17' ] steps: - name: Checkout and setup uses: actions/checkout@v3 @@ -114,22 +84,31 @@ jobs: lfs: true - name: Checkout lfs objects run: git lfs pull + - name: Restore read-only cache of local Maven repository + uses: actions/cache/restore@v3 + id: cache + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ matrix.java-version }} - cache: maven - # The Maven `verify` phase causes the following to happen first, and in order: - # build/compile (`compile`), unit tests (`test`), Jar building (`package`), - # integration tests, if any (`integration-test`) - - name: Run Maven verify + - name: ICU4J run: | cd icu4j; - mvn --batch-mode verify + mvn ${SHARED_MVN_ARGS} verify + - name: List failures (if any) + run: | + cd icu4j && cat `find . -name surefire-reports -type d -exec grep -l -r --include="*.txt" FAILED {} \;`; + if: ${{ failure() }} # ICU4J build and unit test under lstm lstm-icu4j-build-and-test: if: false # TODO(ICU-22505) + needs: icu4j-mvn-init-cache runs-on: ubuntu-latest steps: - name: Checkout and setup @@ -138,6 +117,15 @@ jobs: lfs: true - name: Checkout lfs objects run: git lfs pull + - name: Restore read-only cache of local Maven repository + uses: actions/cache/restore@v3 + id: cache + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + lookup-only: true - uses: actions/setup-java@v3 with: distribution: 'temurin' @@ -152,7 +140,7 @@ jobs: - name: ICU4J run: | cd icu4j; - mvn install + mvn ${SHARED_MVN_ARGS} verify - name: List failures (if any) run: | cd icu4j && cat `find . -name surefire-reports -type d -exec grep -l -r --include="*.txt" FAILED {} \;`; @@ -161,6 +149,7 @@ jobs: # ICU4J build and unit test under adaboost adaboost-icu4j-build-and-test: if: false # Temporary disable, until we disable the .jar creation from C and distribute the individual files + needs: icu4j-mvn-init-cache runs-on: ubuntu-latest steps: - name: Checkout and setup @@ -169,6 +158,15 @@ jobs: lfs: true - name: Checkout lfs objects run: git lfs pull + - name: Restore read-only cache of local Maven repository + uses: actions/cache/restore@v3 + id: cache + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + lookup-only: true - uses: actions/setup-java@v3 with: distribution: 'temurin' @@ -183,7 +181,7 @@ jobs: - name: ICU4J run: | cd icu4j; - mvn -Dcom.ibm.icu.impl.breakiter.useMLPhraseBreaking=true install + mvn ${SHARED_MVN_ARGS} -Dcom.ibm.icu.impl.breakiter.useMLPhraseBreaking=true verify - name: List failures (if any) run: | cd icu4j && cat `find . -name surefire-reports -type d -exec grep -l -r --include="*.txt" FAILED {} \;`; @@ -499,10 +497,19 @@ jobs: # Verify icu4c release tools buildability. icu4c-release-tools: + needs: icu4j-mvn-init-cache runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: mvn -f tools/release/java/pom.xml package dependency:analyze + - name: Restore read-only cache of local Maven repository + uses: actions/cache/restore@v3 + id: cache + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - run: mvn ${SHARED_MVN_ARGS} -f tools/release/java/pom.xml package dependency:analyze # Run unit tests with UCONFIG_NO_XXX variations. uconfig-unit-tests: diff --git a/.github/workflows/icu_merge_ci.yml b/.github/workflows/icu_merge_ci.yml index 1d05edccd656..3347b246168e 100644 --- a/.github/workflows/icu_merge_ci.yml +++ b/.github/workflows/icu_merge_ci.yml @@ -15,15 +15,46 @@ on: # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow env: - MAVEN_ARGS: '--show-version --no-transfer-progress' + SHARED_MVN_ARGS: '--show-version --no-transfer-progress' permissions: contents: read jobs: + # Initialize the Maven artifact cache + # Uses Java 8 because Java version not deemed significant for downloading + # artifacts + # + # This job is created according to the cache strategy of reuse from a single job: + # https://github.com/actions/cache/blob/main/caching-strategies.md#make-cache-read-only--reuse-cache-from-centralized-job + icu4j-mvn-init-cache: + runs-on: ubuntu-latest + steps: + - name: Checkout and setup + uses: actions/checkout@v3 + with: + lfs: true + - name: Checkout lfs objects + run: git lfs pull + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '8' + # Download all of the artifacts needed for the code and build plugins + - name: Download all artifacts + run: | + cd icu4j; + mvn ${SHARED_MVN_ARGS} dependency:go-offline -P '!old_jdk_taglet' + # Test ICU4J with little-endian ICU4C data only icu4j-little-endian-data-test: + needs: icu4j-mvn-init-cache runs-on: ubuntu-latest steps: - name: Checkout and setup @@ -32,6 +63,14 @@ jobs: lfs: true - name: Checkout lfs objects run: git lfs pull + - name: Lookup read-only cache of local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + lookup-only: true - uses: actions/setup-java@v3 with: distribution: 'temurin' @@ -44,8 +83,8 @@ jobs: mkdir /tmp/icu4j_data_test; cp -P data/out/tmp/icudt*l* /tmp/icu4j_data_test/ cd ../../icu4j; - mvn clean; - mvn -Dicu4c.data.path=/tmp/icu4j_data_test install + mvn ${SHARED_MVN_ARGS} clean; + mvn ${SHARED_MVN_ARGS} -Dicu4c.data.path=/tmp/icu4j_data_test install # Compile libraries used by all ICU4C performance tests. icu4c-store-perf-libs: @@ -270,6 +309,7 @@ jobs: icu4j-unicodesetperf: # Run performance tests only on the main branch of the ICU repository. if: github.repository == 'unicode-org/icu' && github.ref == 'refs/heads/main' + needs: icu4j-mvn-init-cache strategy: fail-fast: false matrix: @@ -287,6 +327,14 @@ jobs: lfs: true - name: Checkout lfs objects run: git lfs pull + - name: Lookup read-only cache of local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + lookup-only: true - uses: actions/setup-java@v3 with: distribution: 'temurin' @@ -300,7 +348,7 @@ jobs: git restore --staged tools/multi/proj/icu4cscan/old-xmls.tar.bz2; git restore tools/multi/proj/icu4cscan/old-xmls.tar.bz2; cd icu4j; - mvn install -DskipITs -DskipTests; + mvn ${SHARED_MVN_ARGS} verify -DskipITs -DskipTests; git status cd perf-tests; mkdir -p perf/results/j_unicodesetperf/${{ matrix.perf }}; @@ -325,6 +373,7 @@ jobs: icu4j-ucharacterperf: # Run performance tests only on the main branch of the ICU repository. if: github.repository == 'unicode-org/icu' && github.ref == 'refs/heads/main' + needs: icu4j-mvn-init-cache strategy: fail-fast: false @@ -340,6 +389,14 @@ jobs: lfs: true - name: Checkout lfs objects run: git lfs pull + - name: Lookup read-only cache of local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + lookup-only: true - uses: actions/setup-java@v3 with: distribution: 'temurin' @@ -353,7 +410,7 @@ jobs: git restore --staged tools/multi/proj/icu4cscan/old-xmls.tar.bz2; git restore tools/multi/proj/icu4cscan/old-xmls.tar.bz2; cd icu4j; - mvn install -DskipITs -DskipTests; + mvn ${SHARED_MVN_ARGS} verify -DskipITs -DskipTests; cd perf-tests; mkdir -p perf/results/j_ucharacterperf; java -cp ./target/*:../tools/misc/target/*:../main/core/target/* com.ibm.icu.dev.test.perf.UCharacterPerf -a -t 2 -p 4 0 ffff | tee perf/results/j_ucharacterperf/output.txt @@ -377,6 +434,7 @@ jobs: icu4j-decimalformatperf: # Run performance tests only on the main branch of the ICU repository. if: github.repository == 'unicode-org/icu' && github.ref == 'refs/heads/main' + needs: icu4j-mvn-init-cache strategy: fail-fast: false matrix: @@ -395,6 +453,14 @@ jobs: lfs: true - name: Checkout lfs objects run: git lfs pull + - name: Lookup read-only cache of local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + lookup-only: true - uses: actions/setup-java@v3 with: distribution: 'temurin' @@ -408,7 +474,7 @@ jobs: git restore --staged tools/multi/proj/icu4cscan/old-xmls.tar.bz2; git restore tools/multi/proj/icu4cscan/old-xmls.tar.bz2; cd icu4j; - mvn install -DskipITs -DskipTests; + mvn ${SHARED_MVN_ARGS} verify -DskipITs -DskipTests; cd perf-tests; mkdir -p perf/results/j_decimalformatperf/${{ matrix.locale }}/${{ matrix.perf }}; # Delay execution by random number of seconds. Spreading execution of multiple @@ -436,6 +502,7 @@ jobs: icu4j-normperf: # Run performance tests only on the main branch of the ICU repository. if: github.repository == 'unicode-org/icu' && github.ref == 'refs/heads/main' + needs: icu4j-mvn-init-cache strategy: fail-fast: false matrix: @@ -455,6 +522,14 @@ jobs: lfs: true - name: Checkout lfs objects run: git lfs pull + - name: Lookup read-only cache of local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + lookup-only: true - uses: actions/setup-java@v3 with: distribution: 'temurin' @@ -470,7 +545,7 @@ jobs: git restore --staged tools/multi/proj/icu4cscan/old-xmls.tar.bz2; git restore tools/multi/proj/icu4cscan/old-xmls.tar.bz2; cd icu4j; - mvn install -DskipITs -DskipTests; + mvn ${SHARED_MVN_ARGS} verify -DskipITs -DskipTests; cd perf-tests; mkdir -p perf/results/j_normperf/${{ matrix.source_text }}/${{ matrix.perf }}; # Delay execution by random number of seconds. Spreading execution of multiple @@ -499,6 +574,7 @@ jobs: icu4j-converterperf: # Run performance tests only on the main branch of the ICU repository. if: github.repository == 'unicode-org/icu' && github.ref == 'refs/heads/main' + needs: icu4j-mvn-init-cache strategy: fail-fast: false matrix: @@ -585,6 +661,14 @@ jobs: lfs: true - name: Checkout lfs objects run: git lfs pull + - name: Lookup read-only cache of local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + lookup-only: true - uses: actions/setup-java@v3 with: distribution: 'temurin' @@ -600,7 +684,7 @@ jobs: git restore --staged tools/multi/proj/icu4cscan/old-xmls.tar.bz2; git restore tools/multi/proj/icu4cscan/old-xmls.tar.bz2; cd icu4j; - mvn install -DskipITs -DskipTests; + mvn ${SHARED_MVN_ARGS} verify -DskipITs -DskipTests; cd perf-tests; mkdir -p perf/results/j_converterperf/${{ matrix.source_text }}/${{ matrix.test_enc }}/${{ matrix.perf }}; # Delay execution by random number of seconds. Spreading execution of multiple @@ -628,6 +712,7 @@ jobs: icu4j-dateformatperf: # Run performance tests only on the main branch of the ICU repository. if: github.repository == 'unicode-org/icu' && github.ref == 'refs/heads/main' + needs: icu4j-mvn-init-cache strategy: fail-fast: false matrix: @@ -659,6 +744,14 @@ jobs: lfs: true - name: Checkout lfs objects run: git lfs pull + - name: Lookup read-only cache of local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + lookup-only: true - uses: actions/setup-java@v3 with: distribution: 'temurin' @@ -677,7 +770,7 @@ jobs: git restore --staged tools/multi/proj/icu4cscan/old-xmls.tar.bz2 git restore tools/multi/proj/icu4cscan/old-xmls.tar.bz2 cd icu4j; - mvn install -DskipITs -DskipTests; + mvn ${SHARED_MVN_ARGS} verify -DskipITs -DskipTests; cd perf-tests; mkdir -p perf/results/j_dateformatperf/${{ matrix.locale }}/${{ matrix.perf }}/${{ env.DDIR }}; # Delay execution by random number of seconds. Spreading execution of multiple