Skip to content

Commit

Permalink
chore: optimize CI caching (#2183)
Browse files Browse the repository at this point in the history
  • Loading branch information
SgtPooki authored Dec 2, 2023
1 parent 951c7bd commit b8a71d9
Show file tree
Hide file tree
Showing 10 changed files with 114 additions and 163 deletions.
17 changes: 5 additions & 12 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,10 @@ jobs:
- uses: actions/[email protected]

- name: Setup node
uses: actions/setup-node@v3.6.0
uses: actions/setup-node@v4
with:
node-version: 18.14.0

- name: Cache bigger downloads
uses: actions/[email protected]
id: cache
with:
path: ${{ github.workspace }}/.cache
key: ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
restore-keys: |
${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
${{ runner.os }}-
cache: 'npm'

# Separate cache for build dir, we reuse it in release publish workflow
- name: Cache build output
Expand All @@ -37,7 +28,7 @@ jobs:
- name: Install dependencies
# We only need to install deps if build isn't cached.
if: steps.build-cache.outputs.cache-hit != 'true'
run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
run: npm ci --prefer-offline --no-audit --progress=false

- name: Confirm build works
if: steps.build-cache.outputs.cache-hit != 'true'
Expand All @@ -47,6 +38,8 @@ jobs:
# - this is not for releases, but for quick testing during the dev
# - action artifacts can be downloaded for 90 days, then are removed by github
- name: Attach produced build to Github Action
# only re-upload if we re-built
if: steps.build-cache.outputs.cache-hit != 'true'
uses: actions/upload-artifact@v3
with:
name: ipfs-webui_${{ github.sha }}-${{ runner.os }}-build
Expand Down
18 changes: 12 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,20 @@ jobs:
- uses: actions/[email protected]

- name: Setup node
uses: actions/setup-node@v3.6.0
uses: actions/setup-node@v4
with:
node-version: 18.14.0
cache: 'npm'

- name: Download build artifact
uses: actions/download-artifact@v3
- name: Cache build output
uses: actions/[email protected]
id: build-cache
with:
name: ipfs-webui_${{ github.sha }}-${{ runner.os }}-build
path: build
# Build hash should be based on runner.os, src files, and dependencies used.
key: ${{ runner.os }}-build-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
restore-keys: |
${{ runner.os }}-build-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
- name: Cache bigger downloads
uses: actions/[email protected]
Expand Down Expand Up @@ -232,12 +237,13 @@ jobs:
echo ${{ needs.publishPreview.outputs.cid }} > .cid
- name: Setup node
uses: actions/setup-node@v3.6.0
uses: actions/setup-node@v4
with:
node-version: 18.14.0
cache: 'npm'

- name: Install dependencies
run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
run: npm ci --prefer-offline --no-audit --progress=false

- name: Download CAR artifact
uses: actions/download-artifact@v3
Expand Down
15 changes: 3 additions & 12 deletions .github/workflows/eslint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,13 @@ jobs:
- uses: actions/[email protected]

- name: Setup node
uses: actions/setup-node@v3.6.0
uses: actions/setup-node@v4
with:
node-version: 18.14.0

- name: Cache bigger downloads
uses: actions/[email protected]
id: cache
with:
path: ${{ github.workspace }}/.cache
key: ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
restore-keys: |
${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
${{ runner.os }}-
cache: 'npm'

- name: Install dependencies
run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
run: npm ci --prefer-offline --no-audit --progress=false

- name: ESLint
run: npm run eslint
53 changes: 22 additions & 31 deletions .github/workflows/node-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@ jobs:
if: github.ref == 'refs/heads/main'
uses: RubbaBoy/BYOB@24f464284c1fd32028524b59607d417a2e36fee7
with:
ICON: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
NAME: "node-${{ inputs.gh-node-version }}"
LABEL: "${{ inputs.gh-node-version }}"
STATUS: "Building..."
COLOR: grey
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
icon: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
name: "node-${{ inputs.gh-node-version }}"
label: "${{ inputs.gh-node-version }}"
status: "Building..."
color: grey
github_token: ${{ secrets.GITHUB_TOKEN }}

test-node:
needs: [reset-badge]
Expand All @@ -40,9 +40,10 @@ jobs:
- uses: actions/[email protected]

- name: Use Node.js ${{ inputs.gh-node-version }}
uses: actions/setup-node@v3.6.0
uses: actions/setup-node@v4
with:
node-version: ${{ inputs.gh-node-version }}
cache: 'npm'

- name: Remove .npmrc to allow non-strict engines
run: |
Expand All @@ -53,22 +54,12 @@ jobs:
run: |
echo "version=$(node --version)" >> $GITHUB_OUTPUT
- name: Cache bigger downloads
uses: actions/[email protected]
id: cache
with:
path: ${{ github.workspace }}/.cache
key: ${{ runner.os }}-${{ inputs.gh-node-version }}-${{ hashFiles('package.json', 'package-lock.json') }}
restore-keys: |
${{ runner.os }}-${{ inputs.gh-node-version }}-${{ hashFiles('package.json', 'package-lock.json') }}
${{ runner.os }}-${{ inputs.gh-node-version }}
- name: Install node-pre-gyp
run: |
npm install node-pre-gyp --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
npm install node-pre-gyp --prefer-offline --no-audit --progress=false
- name: Install dependencies
run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
run: npm install --progress=false

- name: Confirm build works with Node.js ${{ inputs.gh-node-version }} on ${{ matrix.os }}
run: npm run build
Expand All @@ -82,20 +73,20 @@ jobs:
if: ${{ needs.test-node.result == 'success' && github.ref == 'refs/heads/main' }}
uses: RubbaBoy/BYOB@24f464284c1fd32028524b59607d417a2e36fee7
with:
ICON: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
NAME: "node-${{ inputs.gh-node-version }}"
LABEL: "${{ needs.test-node.outputs.node-version }}"
STATUS: ${{needs.test-node.result}}
COLOR: green
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
icon: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
name: "node-${{ inputs.gh-node-version }}"
label: "${{ needs.test-node.outputs.node-version }}"
status: ${{needs.test-node.result}}
color: green
github_token: ${{ secrets.GITHUB_TOKEN }}

- name: Create failure badge
if: ${{ needs.test-node.result != 'success' && github.ref == 'refs/heads/main' }}
uses: RubbaBoy/BYOB@24f464284c1fd32028524b59607d417a2e36fee7
with:
ICON: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
NAME: "node-${{ inputs.gh-node-version }}"
LABEL: "${{ needs.test-node.outputs.node-version }}"
STATUS: ${{needs.test-node.result}}
COLOR: red
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
icon: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
name: "node-${{ inputs.gh-node-version }}"
label: "${{ needs.test-node.outputs.node-version }}"
status: ${{needs.test-node.result}}
color: red
github_token: ${{ secrets.GITHUB_TOKEN }}
20 changes: 7 additions & 13 deletions .github/workflows/node-versions.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Test build on different node versions on all platforms
name: Test nodejs support for build on major platforms

on:
schedule:
Expand All @@ -8,26 +8,20 @@ on:

jobs:

test-node-12:
uses: ./.github/workflows/node-test.yml
secrets: inherit
with:
gh-node-version: '12.x'

test-node-14:
test-node-18:
uses: ./.github/workflows/node-test.yml
secrets: inherit
with:
gh-node-version: '14.x'
gh-node-version: '18.x'

test-node-16:
test-node-20:
uses: ./.github/workflows/node-test.yml
secrets: inherit
with:
gh-node-version: '16.x'
gh-node-version: '20.x'

test-node-18:
test-node-21:
uses: ./.github/workflows/node-test.yml
secrets: inherit
with:
gh-node-version: '18.x'
gh-node-version: '21.x'
104 changes: 52 additions & 52 deletions .github/workflows/test-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,32 @@ jobs:
- uses: actions/[email protected]

- name: Setup node
uses: actions/setup-node@v3.6.0
uses: actions/setup-node@v4
with:
node-version: 18.14.0
cache: 'npm'

- name: Cache bigger downloads
- name: Cache test-build output
uses: actions/[email protected]
id: cache
id: test-build-cache
with:
path: ${{ github.workspace }}/.cache
key: ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
path: build
key: test-build-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
restore-keys: |
${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
${{ runner.os }}-
- name: Install dependencies
run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
test-build-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
- name: Cache test-build output
- name: Cache node_modules
uses: actions/[email protected]
id: test-build-cache
id: test-npm-cache
with:
path: build
key: test-build-${{ runner.os }}-${{ github.sha }}
path: node_modules
key: test-npm-${{ runner.os }}-${{ github.sha }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', 'patches/**', '*.js', '*.ts') }}
restore-keys: |
test-build-${{ runner.os }}-${{ github.sha }}
test-npm-${{ runner.os }}-${{ github.sha }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', 'patches/**', '*.js', '*.ts') }}
- name: Install dependencies
if: steps.test-npm-cache.outputs.cache-hit != 'true'
run: npm ci --prefer-offline --no-audit --progress=false

# This is required to ensure that our code is instrumented with coverage details
- name: Run test build
Expand All @@ -53,39 +54,43 @@ jobs:
- uses: actions/[email protected]

- name: Setup node
uses: actions/setup-node@v3.6.0
uses: actions/setup-node@v4
with:
node-version: 18.14.0
cache: 'npm'

- name: Cache bigger downloads
- name: Cache test-build output
uses: actions/[email protected]
id: cache
id: test-build-cache
with:
path: ${{ github.workspace }}/.cache
key: ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
path: build
key: test-build-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
restore-keys: |
${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
${{ runner.os }}-
test-build-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
- name: Install dependencies
run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm

- name: Install playwright browsers
run: npx playwright install --with-deps

- name: Cache test-build output
# We don't want to re-install dependencies for every sharded run
- name: Cache node_modules
uses: actions/[email protected]
id: test-build-cache
id: test-npm-cache
with:
path: build
key: test-build-${{ runner.os }}-${{ github.sha }}
path: node_modules
key: test-npm-${{ runner.os }}-${{ github.sha }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', 'patches/**', '*.js', '*.ts') }}
restore-keys: |
test-build-${{ runner.os }}-${{ github.sha }}
test-npm-${{ runner.os }}-${{ github.sha }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', 'patches/**', '*.js', '*.ts') }}
- name: Install dependencies
if: steps.test-npm-cache.outputs.cache-hit != 'true'
run: npm ci --prefer-offline --no-audit --progress=false

# This is required to ensure that our code is instrumented with coverage details
- name: Run test build
if: steps.test-build-cache.outputs.cache-hit != 'true'
run: npm run test:build
# Cache playwright binaries
- uses: actions/cache@v3
id: playwright-cache
with:
path: |
~/.cache/ms-playwright
key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }}
- run: npm ci
- run: npx playwright install --with-deps
if: steps.playwright-cache.outputs.cache-hit != 'true'

- name: Cache nyc_output dir
uses: actions/[email protected]
Expand All @@ -104,6 +109,14 @@ jobs:
if: github.ref == 'refs/heads/main' # run non-duplicated tests on non-PRs
run: npm run test:e2e -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}

- name: Store Artifacts from Failed Tests
if: failure()
uses: actions/upload-artifact@v3
with:
name: test-results
path: test-results/
retention-days: 7

e2e-coverage: # since we run coverage in shards and some files may not contain coverage, we cache the .nyc_output directory
name: 'e2e-coverage'
runs-on: ubuntu-latest
Expand All @@ -112,23 +125,10 @@ jobs:
- uses: actions/[email protected]

- name: Setup node
uses: actions/setup-node@v3.6.0
uses: actions/setup-node@v4
with:
node-version: 18.14.0

- name: Cache bigger downloads
uses: actions/[email protected]
id: cache
with:
path: ${{ github.workspace }}/.cache
key: ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
restore-keys: |
${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
${{ runner.os }}-
- name: Install dependencies
run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
cache: 'npm'

- name: Cache nyc_output dir
uses: actions/[email protected]
Expand Down
Loading

0 comments on commit b8a71d9

Please sign in to comment.