ci: cache all build tasks (#3119)

https://coveord.atlassian.net/browse/KIT-2710

.github/actions/build/action.yml

@@ -0,0 +1,16 @@
+name: 'Install, build & cache'
+description: 'Installs dependencies, builds all packages, and saves the cache of Nx'
+runs:
+  using: composite
+  steps:
+    - uses: ./.github/actions/setup
+      with:
+        load-cache: 'false'
+    - uses: actions/cache@v3
+      id: nx-cache
+      with:
+        path: node_modules/.cache/nx
+        key: nx-${{ github.sha }}
+    - if: steps.nx-cache.outputs.cache-hit != 'true'
+      run: npm run build
+      shell: bash

.github/actions/e2e-atomic-angular/action.yml

@@ -16,9 +16,8 @@ runs:
       name: Run Cypress
       with:
         browser: chrome
-        command-prefix: 'npx cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
-        working-directory: ./packages/samples/angular
-        start: npm run dev
+        command-prefix: 'npx -w @coveo/atomic-angular-samples cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
+        start: npx nx run atomic-angular-samples:dev
         wait-on: 'http://localhost:4200'
         install: false
         record: false

.github/actions/e2e-atomic-hosted-page/action.yml

@@ -12,9 +12,8 @@ runs:
       name: Run Cypress
       with:
         browser: chrome
-        command-prefix: 'npx cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
-        working-directory: ./packages/atomic-hosted-page
-        start: npm run dev
+        command-prefix: 'npx -w @coveo/atomic-hosted-page cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
+        start: npx nx run atomic-hosted-page:dev
         wait-on: 'http://localhost:3335'
         install: false
         record: false

.github/actions/e2e-atomic-insight-panel/action.yml

@@ -12,10 +12,9 @@ runs:
       name: Run Cypress
       with:
         browser: chrome
-        command-prefix: 'npx cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
-        working-directory: ./packages/atomic
+        command-prefix: 'npx -w @coveo/atomic cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
         config-file: cypress-insight-panel.config.ts
-        start: npm run prod
+        start: npx nx run atomic:prod
         wait-on: 'http://localhost:3333'
         install: false
         record: false

.github/actions/e2e-atomic-next/action.yml

@@ -12,9 +12,8 @@ runs:
       name: Run Cypress
       with:
         browser: chrome
-        command-prefix: 'npx cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
-        working-directory: ./packages/samples/atomic-next
-        start: npm run dev
+        command-prefix: 'npx -w @coveo/atomic-next-samples cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
+        start: npx nx run atomic-next-samples:dev
         wait-on: 'http://localhost:3000'
         install: false
         record: false

.github/actions/e2e-atomic-react/action.yml

@@ -12,9 +12,8 @@ runs:
       name: Run Cypress
       with:
         browser: chrome
-        command-prefix: 'npx cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
-        working-directory: ./packages/samples/atomic-react
-        start: npm run dev
+        command-prefix: 'npx -w @coveo/atomic-react-samples cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
+        start: npx nx run atomic-react-samples:dev
         wait-on: 'http://localhost:3000'
         install: false
         record: false

.github/actions/e2e-atomic-screenshots/action.yml

@@ -12,10 +12,9 @@ runs:
       name: Run Cypress
       with:
         browser: chrome
-        command-prefix: 'npx cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
-        working-directory: ./packages/atomic
+        command-prefix: 'npx -w @coveo/atomic cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
         config-file: cypress-screenshots.config.ts
-        start: npm run prod
+        start: npx nx run atomic:prod
         wait-on: 'http://localhost:3333'
         install: false
         record: false

.github/actions/e2e-atomic/action.yml

@@ -15,10 +15,9 @@ runs:
       name: Run Cypress
       with:
         browser: chrome
-        command-prefix: 'npx cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
-        working-directory: ./packages/atomic
+        command-prefix: 'npx -w @coveo/atomic cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
         spec: ${{ inputs.spec }}
-        start: npm run prod
+        start: npx nx run atomic:prod
         wait-on: 'http://localhost:3333'
         install: false
         record: false

.github/actions/e2e-headless-ssr/action.yml

@@ -12,9 +12,8 @@ runs:
       name: Run Cypress
       with:
         browser: chrome
-        command-prefix: 'npm run e2e'
-        working-directory: ./packages/samples/headless-ssr
-        start: npm run prod
+        command-prefix: 'npm run -w @coveo/headless-ssr-samples e2e'
+        start: npx nx run headless-ssr-samples:prod
         wait-on: 'http://localhost:3000'
         install: false
         record: false

.github/actions/e2e-iife/action.yml

@@ -12,9 +12,8 @@ runs:
       name: Run Cypress
       with:
         browser: chrome
-        command-prefix: 'npx cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
-        working-directory: ./packages/samples/iife
-        start: npm run prod
+        command-prefix: 'npx -w @coveo/iife-samples cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
+        start: npx nx run iife-samples:prod
         wait-on: 'http://localhost:3000'
         install: false
         record: false

.github/actions/e2e-quantic-setup/action.yml

@@ -10,6 +10,8 @@ inputs:
 runs:
   using: composite
   steps:
+    - run: npx nx run quantic:build
+      shell: bash
     - run: |
         echo "${{ env.SFDX_AUTH_JWT_KEY }}" > server.key
         npx --no-install sfdx force:auth:jwt:grant \

.github/actions/e2e-stencil/action.yml

@@ -12,9 +12,8 @@ runs:
       name: Run Cypress
       with:
         browser: chrome
-        command-prefix: 'npx cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
-        working-directory: ./packages/samples/stencil
-        start: npm run dev
+        command-prefix: 'npx -w @coveo/atomic-stencil-samples cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
+        start: npx nx run atomic-stencil-samples:dev
         wait-on: 'http://localhost:3666'
         install: false
         record: false

.github/actions/e2e-vuejs/action.yml

@@ -12,9 +12,8 @@ runs:
       name: Run Cypress
       with:
         browser: chrome
-        command-prefix: 'npx cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
-        working-directory: ./packages/samples/vuejs
-        start: npm run dev
+        command-prefix: 'npx -w @coveo/atomic-vuejs-samples cypress-repeat run -n 3 --until-passes --rerun-failed-only --'
+        start: npx nx run atomic-vuejs-samples:dev
         wait-on: 'http://localhost:8080'
         install: false
         record: false

.github/actions/setup/action.yml

@@ -1,51 +1,43 @@
 name: 'Setup & build'
 description: 'Setup node, install dependencies & build'
+inputs:
+  load-cache:
+    description: "Whether to restore the Nx cache."
+    required: false
+    default: 'true'
 runs:
   using: composite
   steps:
     - uses: actions/setup-node@v3
       with:
         node-version-file: '.nvmrc'
+    - name: Ensure Nx cache can be shared between different CI environments # Source: https://nx.dev/recipes/troubleshooting/unknown-local-cache
+      run: echo "NX_REJECT_UNKNOWN_LOCAL_CACHE=0" >> $GITHUB_ENV
+      shell: bash
     - name: Install npm
       run: npm i -g npm@9
       shell: bash
     - uses: actions/cache@v3
-      id: build-cache
+      id: npm-cache
       with:
         path: |
-          packages/bueno/dist
-          packages/headless/dist
-          packages/auth/dist
-          packages/atomic/dist
-          packages/atomic/docs
-          packages/atomic/loader
-          packages/atomic/www
-          packages/atomic/src/components/search/atomic-search-interface/lang/
-          packages/atomic/src/external-builds/
-          packages/atomic/src/generated/
-          packages/quantic/docs/out
-          packages/atomic-react/dist
-          packages/atomic-hosted-page/dist
-          packages/atomic-angular/projects/atomic-angular/dist
-          packages/samples/headless-react
-          packages/samples/iife/www/cdn
-          packages/samples/headless-ssr/.next
-          packages/quantic/force-app/main/default/staticresources/coveoheadless
-          packages/quantic/force-app/main/default/staticresources/coveobueno
-          utils/atomic-storybook/storybook-static
-          !node_modules
-          !**/node_modules
-        key: ${{ github.sha }}
-    - uses: actions/cache@v3
-      with:
-        path: ~/.npm
-        key: npm-${{ hashFiles('**/package-lock.json') }}
+          **/node_modules/
+          !node_modules/.cache/nx/
+        key: npm-${{ hashFiles('package-lock.json') }}-${{ hashFiles('utils/atomic-storybook/package-lock.json') }}
     - uses: actions/cache@v3
       with:
         path: ~/.cache/Cypress
-        key: cy-${{ hashFiles('**/package-lock.json') }}
-    - run: npm ci
+        key: cy-${{ hashFiles('package-lock.json') }}-${{ hashFiles('utils/atomic-storybook/package-lock.json') }}
+    - if: inputs.load-cache == 'true'
+      uses: actions/cache/restore@v3
+      id: nx-cache
+      with:
+        path: node_modules/.cache/nx
+        key: nx-${{ github.sha }}
+        fail-on-cache-miss: true
+    - if: steps.npm-cache.outputs.cache-hit != 'true'
+      run: npm ci
       shell: bash
-    - if: steps.build-cache.outputs.cache-hit != 'true'
+    - if: steps.nx-cache.outputs.cache-hit != 'true'
       run: npm run build
       shell: bash

.github/workflows/create-quantic-package.yml

@@ -41,7 +41,7 @@ jobs:
               run_id: context.runId
             })
       - name: Setup
-        uses: ./.github/actions/setup
+        uses: ./.github/actions/build
       - name: Create package version
         working-directory: ./packages/quantic
         run: |

.github/workflows/masterbot.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup
+      - uses: ./.github/actions/build
   lint-check:
     name: 'Check with linter'
     needs: build

.github/workflows/prbot.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup
+      - uses: ./.github/actions/build
       - uses: ./.github/actions/commit-generated-files
   lint-check:
     name: 'Check with linter'

internal-docs/scripts-and-tasks.md

@@ -31,6 +31,79 @@ Some scripts may be nested under other scripts using the colon (`:`) symbol. Thi
 - Doesn't expect another script from the same project to be run before this script.
 - If the build fails, exits with an error.
 
+### Caching builds
+
+When running tasks using [Nx](https://nx.dev/), packages and dependencies are automatically built if needed. To avoid the need to run tasks without Nx, builds are [cached](https://nx.dev/concepts/how-caching-works#how-caching-works) and should only be re-built when their source files are changed.
+
+By default, when adding new packages, builds aren't cached. In order to make a project's builds cache-able, you should follow these steps:
+
+1. Create a [`project.json` file](https://nx.dev/recipes/tips-n-tricks/integrated-in-package-based#project.json) at the root of the project.
+
+   - How:
+
+     - You can copy this scaffold (replace all instances of "turtle" with the name of your project):
+
+       ```json
+       {
+         "name": "turtle",
+         "$schema": "../../node_modules/nx/schemas/project-schema.json",
+         "namedInputs": {},
+         "targets": {
+           "cached:build": {
+             "executor": "nx:run-commands",
+             "options": {
+               "commands": [
+                 "rimraf ./dist",
+                 "mkdir ./dist",
+                 "echo \"hello\" > ./dist/hello.txt",
+                 "cp ./dist/hello.txt ./dist/world.txt"
+               ],
+               "parallel": false,
+               "cwd": "packages/turtle"
+             }
+           },
+           "build": {
+             "dependsOn": ["cached:build"],
+             "executor": "nx:noop"
+           }
+         }
+       }
+       ```
+
+2. In your new `project.json` file, adapt the "cached:build" [task](https://nx.dev/core-features/run-tasks#define-tasks) (tasks are sometimes referred to as "targets") so that it builds your project.
+
+   - Ensure it runs the correct command.
+     - How:
+       - A good place to start is by looking at the [`nx:run-commands` executor's documentation](https://nx.dev/packages/nx/executors/run-commands).
+       - Make sure to prefix commands that come from dependencies with "npx ".
+   - Do not rename it.
+     - Why:
+       - The name of the task must be "cached:build" since Nx only caches tasks whose name is part of the [cacheable operations](https://nx.dev/nx-cloud/reference/config#cacheable-operations). We defined "cached:build" as a cacheable operation in [`nx.json`](/nx.json).
+   - Define the correct [inputs](https://nx.dev/concepts/more-concepts/customizing-inputs#customizing-inputs-and-named-inputs).
+     - When:
+       - If some files which aren't part of the default inputs (defined in [`nx.json`](/nx.json)) and affect the final build output are outside the project's directory.
+       - If some auto-generated files from a different task or a different project are part of the current directory.
+       - If you **really** want to be selective about which changes in your project's directory will invalidate their cache.
+     - How:
+       - If possible, only customize the `buildInputs` [named inputs](https://nx.dev/concepts/more-concepts/customizing-inputs#customizing-inputs-and-named-inputs).
+         - The default inputs for build tasks defined in [`nx.json`](/nx.json) uses the `buildInputs` in addition to some other files.
+       - Input glob patterns should match every file that can affect the final build output.
+     - Why:
+       - When a build task is triggered, Nx will verify if any of its inputs was changed. If none of its input was changed, it will skip re-building the package and may load its outputs from the cache.
+   - Define the correct [outputs](https://nx.dev/concepts/how-caching-works#what-is-cached).
+     - When:
+       - If the default outputs defined in [`nx.json`](/nx.json) either don't match every file generated by the task or match some files that aren't generated by the task.
+     - How:
+       - The output glob patterns should match all the files generated by the build task, and no more than that.
+       - You must also copy all these glob patterns to the `negativeBuildOutputs` [named inputs](https://nx.dev/concepts/more-concepts/customizing-inputs#customizing-inputs-and-named-inputs), but prefix them with an exclamation symbol (`!`).
+         - The default inputs defined in [`nx.json`](/nx.json) won't match files which are also matched in `negativeBuildOutputs`.
+     - Why:
+       - After Nx runs your task, it will save every file matched by the output glob patterns. The next time you run the same task, Nx may restore the files it saved.
+
+3. Replace the `build` script in your `package.json`
+   - How:
+     - Simply replace it with `nx build`.
+
 ## `dev` script
 
 - Generates all the files needed for a dependent project to run.