diff --git a/.changeset/pink-seas-stare.md b/.changeset/pink-seas-stare.md new file mode 100644 index 000000000..081f50d81 --- /dev/null +++ b/.changeset/pink-seas-stare.md @@ -0,0 +1,5 @@ +--- +'skuba': minor +--- + +template, format: Mount `.npmrc` files in `/tmp/` rather than `/tmp/`, to avoid accidental inclusion in commits or published artifacts, as per the original intent of this handling. diff --git a/docs/deep-dives/pnpm.md b/docs/deep-dives/pnpm.md index 3570209af..343f0c237 100644 --- a/docs/deep-dives/pnpm.md +++ b/docs/deep-dives/pnpm.md @@ -287,7 +287,7 @@ This migration guide assumes that your project was scaffolded with a **skuba** t ```diff seek-oss/private-npm#v1.2.0: env: NPM_READ_TOKEN - + output-path: tmp/ + + output-path: /tmp/ ``` ```diff @@ -301,7 +301,7 @@ This migration guide assumes that your project was scaffolded with a **skuba** t + - pnpm-lock.yaml dockerfile: Dockerfile.dev-deps - secrets: id=npm,src=.npmrc - + secrets: id=npm,src=tmp/.npmrc + + secrets: id=npm,src=/tmp/.npmrc ``` 15. Run `pnpm install --offline` and replace `yarn` with `pnpm` in `.buildkite/pipeline.yml` diff --git a/src/cli/__snapshots__/format.int.test.ts.snap b/src/cli/__snapshots__/format.int.test.ts.snap index 64e284a14..ac1686580 100644 --- a/src/cli/__snapshots__/format.int.test.ts.snap +++ b/src/cli/__snapshots__/format.int.test.ts.snap @@ -27,6 +27,8 @@ Patch skipped: Remove version field from docker-compose files - no docker-compos Patch skipped: Update docker image references to use public.ecr.aws and remove --platform flag - no Dockerfile or docker-compose files found +Patch skipped: Move .npmrc mounts from tmp/.npmrc to /tmp/.npmrc - no Buildkite files found + skuba update complete. Refreshed .gitignore. refresh-config-files @@ -109,6 +111,8 @@ Patch skipped: Remove version field from docker-compose files - no docker-compos Patch skipped: Update docker image references to use public.ecr.aws and remove --platform flag - no Dockerfile or docker-compose files found +Patch skipped: Move .npmrc mounts from tmp/.npmrc to /tmp/.npmrc - no Buildkite files found + skuba update complete. Refreshed .gitignore. refresh-config-files @@ -186,6 +190,8 @@ Patch skipped: Remove version field from docker-compose files - no docker-compos Patch skipped: Update docker image references to use public.ecr.aws and remove --platform flag - no Dockerfile or docker-compose files found +Patch skipped: Move .npmrc mounts from tmp/.npmrc to /tmp/.npmrc - no Buildkite files found + skuba update complete. Refreshed .gitignore. refresh-config-files @@ -234,6 +240,8 @@ Patch skipped: Remove version field from docker-compose files - no docker-compos Patch skipped: Update docker image references to use public.ecr.aws and remove --platform flag - no Dockerfile or docker-compose files found +Patch skipped: Move .npmrc mounts from tmp/.npmrc to /tmp/.npmrc - no Buildkite files found + skuba update complete. Refreshed .gitignore. refresh-config-files diff --git a/src/cli/lint/internalLints/upgrade/patches/8.2.1/collapseDuplicateMergeKeys.ts b/src/cli/lint/internalLints/upgrade/patches/8.2.1/collapseDuplicateMergeKeys.ts index dbbde40fe..9766cabac 100644 --- a/src/cli/lint/internalLints/upgrade/patches/8.2.1/collapseDuplicateMergeKeys.ts +++ b/src/cli/lint/internalLints/upgrade/patches/8.2.1/collapseDuplicateMergeKeys.ts @@ -10,7 +10,7 @@ const collapseDuplicateMergeKeys: PatchFunction = async ({ mode, }): Promise => { const buildkiteFiles = await glob( - ['.buildkite/**/*.yml', '.buildkite/**/*.yaml'], + ['{apps/*/,packages/*/,./}.buildkite/**/*.y*ml'], { onlyFiles: true }, ); diff --git a/src/cli/lint/internalLints/upgrade/patches/8.2.1/index.ts b/src/cli/lint/internalLints/upgrade/patches/8.2.1/index.ts index 7a26eaff0..225704820 100644 --- a/src/cli/lint/internalLints/upgrade/patches/8.2.1/index.ts +++ b/src/cli/lint/internalLints/upgrade/patches/8.2.1/index.ts @@ -1,6 +1,7 @@ import type { Patches } from '../..'; import { tryCollapseDuplicateMergeKeys } from './collapseDuplicateMergeKeys'; +import { tryMoveNpmrcMounts } from './moveNpmrcMounts'; import { tryPatchDockerComposeFiles } from './patchDockerCompose'; import { tryPatchDockerImages } from './patchDockerImages'; import { tryUpgradeESLint } from './upgradeESLint'; @@ -23,4 +24,8 @@ export const patches: Patches = [ description: 'Update docker image references to use public.ecr.aws and remove --platform flag', }, + { + apply: tryMoveNpmrcMounts, + description: 'Move .npmrc mounts from tmp/.npmrc to /tmp/.npmrc', + }, ]; diff --git a/src/cli/lint/internalLints/upgrade/patches/8.2.1/moveNpmrcMounts.test.ts b/src/cli/lint/internalLints/upgrade/patches/8.2.1/moveNpmrcMounts.test.ts new file mode 100644 index 000000000..30e1d5893 --- /dev/null +++ b/src/cli/lint/internalLints/upgrade/patches/8.2.1/moveNpmrcMounts.test.ts @@ -0,0 +1,289 @@ +// eslint-disable-next-line no-restricted-imports -- fs-extra is mocked +import fsp from 'fs/promises'; + +import memfs, { vol } from 'memfs'; + +import type { PatchConfig } from '../..'; +import { configForPackageManager } from '../../../../../../utils/packageManager'; + +import { tryMoveNpmrcMounts } from './moveNpmrcMounts'; + +const volToJson = () => vol.toJSON(process.cwd(), undefined, true); + +jest.mock('fs-extra', () => memfs); +jest.mock('fast-glob', () => ({ + glob: (pat: any, opts: any) => + jest.requireActual('fast-glob').glob(pat, { ...opts, fs: memfs }), +})); + +beforeEach(() => vol.reset()); + +describe('moveNpmrcMounts', () => { + const baseArgs = { + manifest: {} as PatchConfig['manifest'], + packageManager: configForPackageManager('pnpm'), + }; + + afterEach(() => jest.resetAllMocks()); + + describe.each(['lint', 'format'] as const)('%s', (mode) => { + it('should not need to modify any of the template pipelines', async () => { + for (const template of await fsp.readdir('template')) { + const pipelineFile = `template/${template}/.buildkite/pipeline.yml`; + try { + await fsp.stat(pipelineFile); + } catch { + continue; + } + + const contents = await fsp.readFile(pipelineFile, 'utf-8'); + + vol.fromJSON({ + '.buildkite/pipeline.yml': contents, + }); + + await expect( + tryMoveNpmrcMounts({ + ...baseArgs, + mode, + }), + ).resolves.toEqual({ + result: 'skip', + reason: 'no .npmrc mounts found need to be updated', + }); + + expect(volToJson()).toEqual({ + '.buildkite/pipeline.yml': contents, + }); + } + }); + + it('should skip if no Buildkite files are found', async () => { + await expect( + tryMoveNpmrcMounts({ + ...baseArgs, + mode, + }), + ).resolves.toEqual({ + result: 'skip', + reason: 'no Buildkite files found', + }); + + expect(volToJson()).toEqual({}); + }); + + it('should skip on a pipeline without mounts', async () => { + const input = `steps: + - label: 'My Step' + command: echo 'Hello, world!' +`; + + vol.fromJSON({ + '.buildkite/pipeline.yml': input, + }); + + await expect( + tryMoveNpmrcMounts({ + ...baseArgs, + mode, + }), + ).resolves.toEqual({ + result: 'skip', + reason: 'no .npmrc mounts found need to be updated', + }); + + expect(volToJson()).toEqual({ + '.buildkite/pipeline.yml': input, + }); + }); + + it('should fix an incorrect mount', async () => { + const input = `configs: + plugins: + - &docker-ecr-cache + seek-oss/docker-ecr-cache#v2.2.0: + cache-on: + - .npmrc + - package.json#.packageManager + - pnpm-lock.yaml + dockerfile: Dockerfile.dev-deps + ecr-name: build-cache/my-service + secrets: id=npm,src=tmp/.npmrc + + - &private-npm + seek-oss/private-npm#v1.2.0: + env: NPM_READ_TOKEN + output-path: tmp/ + +steps: []`; + + vol.fromJSON({ + '.buildkite/pipeline.yml': input, + }); + + await expect( + tryMoveNpmrcMounts({ + ...baseArgs, + mode, + }), + ).resolves.toEqual({ result: 'apply' }); + + expect(volToJson()).toEqual({ + '.buildkite/pipeline.yml': + mode === 'lint' + ? input + : `configs: + plugins: + - &docker-ecr-cache + seek-oss/docker-ecr-cache#v2.2.0: + cache-on: + - .npmrc + - package.json#.packageManager + - pnpm-lock.yaml + dockerfile: Dockerfile.dev-deps + ecr-name: build-cache/my-service + secrets: id=npm,src=/tmp/.npmrc + + - &private-npm + seek-oss/private-npm#v1.2.0: + env: NPM_READ_TOKEN + output-path: /tmp/ + +steps: []`, + }); + }); + + it('should fix an incorrect mount with comments', async () => { + const input = `configs: + plugins: + - &docker-ecr-cache + seek-oss/docker-ecr-cache#v2.2.0: + cache-on: + - .npmrc + - package.json#.packageManager + - pnpm-lock.yaml + dockerfile: Dockerfile.dev-deps + ecr-name: build-cache/my-service + secrets: id=npm,src=tmp/.npmrc # hello + + - &private-npm + seek-oss/private-npm#v1.2.0: + env: NPM_READ_TOKEN + output-path: tmp/#world + +steps: []`; + + vol.fromJSON({ + '.buildkite/pipeline.yml': input, + }); + + await expect( + tryMoveNpmrcMounts({ + ...baseArgs, + mode, + }), + ).resolves.toEqual({ result: 'apply' }); + + expect(volToJson()).toEqual({ + '.buildkite/pipeline.yml': + mode === 'lint' + ? input + : `configs: + plugins: + - &docker-ecr-cache + seek-oss/docker-ecr-cache#v2.2.0: + cache-on: + - .npmrc + - package.json#.packageManager + - pnpm-lock.yaml + dockerfile: Dockerfile.dev-deps + ecr-name: build-cache/my-service + secrets: id=npm,src=/tmp/.npmrc # hello + + - &private-npm + seek-oss/private-npm#v1.2.0: + env: NPM_READ_TOKEN + output-path: /tmp/#world + +steps: []`, + }); + }); + + it('should skip a mount without tmp', async () => { + const input = `configs: + plugins: + - &docker-ecr-cache + seek-oss/docker-ecr-cache#v2.2.0: + cache-on: + - .npmrc + - package.json#.packageManager + - pnpm-lock.yaml + dockerfile: Dockerfile.dev-deps + ecr-name: build-cache/my-service + secrets: id=npm,src=.npmrc + + - &private-npm + seek-oss/private-npm#v1.2.0: + env: NPM_READ_TOKEN + +steps: []`; + + vol.fromJSON({ + '.buildkite/pipeline.yml': input, + }); + + await expect( + tryMoveNpmrcMounts({ + ...baseArgs, + mode, + }), + ).resolves.toEqual({ + result: 'skip', + reason: 'no .npmrc mounts found need to be updated', + }); + + expect(volToJson()).toEqual({ + '.buildkite/pipeline.yml': input, + }); + }); + + it('should skip a /tmp/ mount', async () => { + const input = `configs: + plugins: + - &docker-ecr-cache + seek-oss/docker-ecr-cache#v2.2.0: + cache-on: + - .npmrc + - package.json#.packageManager + - pnpm-lock.yaml + dockerfile: Dockerfile.dev-deps + ecr-name: build-cache/my-service + secrets: id=npm,src=/tmp/.npmrc + + - &private-npm + seek-oss/private-npm#v1.2.0: + env: NPM_READ_TOKEN + output-path: /tmp/ + +steps: []`; + + vol.fromJSON({ + 'packages/stuff/.buildkite/pipeline.yaml': input, + }); + + await expect( + tryMoveNpmrcMounts({ + ...baseArgs, + mode, + }), + ).resolves.toEqual({ + result: 'skip', + reason: 'no .npmrc mounts found need to be updated', + }); + + expect(volToJson()).toEqual({ + 'packages/stuff/.buildkite/pipeline.yaml': input, + }); + }); + }); +}); diff --git a/src/cli/lint/internalLints/upgrade/patches/8.2.1/moveNpmrcMounts.ts b/src/cli/lint/internalLints/upgrade/patches/8.2.1/moveNpmrcMounts.ts new file mode 100644 index 000000000..33b26efe2 --- /dev/null +++ b/src/cli/lint/internalLints/upgrade/patches/8.2.1/moveNpmrcMounts.ts @@ -0,0 +1,69 @@ +import { inspect } from 'util'; + +import { glob } from 'fast-glob'; +import { promises as fs } from 'fs-extra'; + +import type { PatchFunction, PatchReturnType } from '../..'; +import { log } from '../../../../../../utils/logging'; + +const moveNpmrcMounts: PatchFunction = async ({ + mode, +}): Promise => { + const buildkiteFiles = await glob( + ['{apps/*/,packages/*/,./}.buildkite/**/*.y*ml'], + { onlyFiles: true }, + ); + + if (buildkiteFiles.length === 0) { + return { result: 'skip', reason: 'no Buildkite files found' }; + } + + const input = await Promise.all( + buildkiteFiles.map((name) => fs.readFile(name, 'utf-8')), + ); + + const replaced = input.map(moveNpmrcMountsInFile); + + if (replaced.every((r, i) => r === input[i])) { + return { + result: 'skip', + reason: 'no .npmrc mounts found need to be updated', + }; + } + + if (mode === 'lint') { + return { result: 'apply' }; + } + + await Promise.all( + buildkiteFiles.flatMap((name, i) => + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + replaced[i] !== input[i] ? [fs.writeFile(name, replaced[i]!)] : [], + ), + ); + + return { result: 'apply' }; +}; + +const secret = /^(\s*)secrets: id=npm,src=tmp\/\.npmrc(\s*#?.*)$/gm; +const outputPath = /^(\s*)output-path: tmp\/(\s*#?.*)$/gm; + +const moveNpmrcMountsInFile = (input: string) => { + if (!secret.test(input) || !outputPath.test(input)) { + return input; + } + + return input + .replaceAll(secret, '$1secrets: id=npm,src=/tmp/.npmrc$2') + .replaceAll(outputPath, '$1output-path: /tmp/$2'); +}; + +export const tryMoveNpmrcMounts: PatchFunction = async (config) => { + try { + return await moveNpmrcMounts(config); + } catch (err) { + log.warn('Failed to move .npmrc mounts'); + log.subtle(inspect(err)); + return { result: 'skip', reason: 'due to an error' }; + } +}; diff --git a/template/express-rest-api/.buildkite/pipeline.yml b/template/express-rest-api/.buildkite/pipeline.yml index b4918355a..25f32b5bd 100644 --- a/template/express-rest-api/.buildkite/pipeline.yml +++ b/template/express-rest-api/.buildkite/pipeline.yml @@ -15,12 +15,12 @@ configs: - package.json#.packageManager - pnpm-lock.yaml dockerfile: Dockerfile.dev-deps - secrets: id=npm,src=tmp/.npmrc + secrets: id=npm,src=/tmp/.npmrc - &private-npm seek-oss/private-npm#v1.2.0: env: NPM_READ_TOKEN - output-path: tmp/ + output-path: /tmp/ base-steps: - &deploy diff --git a/template/greeter/.buildkite/pipeline.yml b/template/greeter/.buildkite/pipeline.yml index 2c1701a8a..29dc9ea55 100644 --- a/template/greeter/.buildkite/pipeline.yml +++ b/template/greeter/.buildkite/pipeline.yml @@ -16,12 +16,12 @@ configs: - .npmrc - package.json#.packageManager - pnpm-lock.yaml - secrets: id=npm,src=tmp/.npmrc + secrets: id=npm,src=/tmp/.npmrc - &private-npm seek-oss/private-npm#v1.2.0: env: NPM_READ_TOKEN - output-path: tmp/ + output-path: /tmp/ steps: - label: 🧪 Test & Lint diff --git a/template/koa-rest-api/.buildkite/pipeline.yml b/template/koa-rest-api/.buildkite/pipeline.yml index b4918355a..25f32b5bd 100644 --- a/template/koa-rest-api/.buildkite/pipeline.yml +++ b/template/koa-rest-api/.buildkite/pipeline.yml @@ -15,12 +15,12 @@ configs: - package.json#.packageManager - pnpm-lock.yaml dockerfile: Dockerfile.dev-deps - secrets: id=npm,src=tmp/.npmrc + secrets: id=npm,src=/tmp/.npmrc - &private-npm seek-oss/private-npm#v1.2.0: env: NPM_READ_TOKEN - output-path: tmp/ + output-path: /tmp/ base-steps: - &deploy diff --git a/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml b/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml index 7100b3207..39db82a4b 100644 --- a/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml +++ b/template/lambda-sqs-worker-cdk/.buildkite/pipeline.yml @@ -14,12 +14,12 @@ configs: - .npmrc - package.json#.packageManager - pnpm-lock.yaml - secrets: id=npm,src=tmp/.npmrc + secrets: id=npm,src=/tmp/.npmrc - &private-npm seek-oss/private-npm#v1.2.0: env: NPM_READ_TOKEN - output-path: tmp/ + output-path: /tmp/ base-steps: - &deploy diff --git a/template/lambda-sqs-worker/.buildkite/pipeline.yml b/template/lambda-sqs-worker/.buildkite/pipeline.yml index 83175468c..670cc3016 100644 --- a/template/lambda-sqs-worker/.buildkite/pipeline.yml +++ b/template/lambda-sqs-worker/.buildkite/pipeline.yml @@ -14,12 +14,12 @@ configs: - .npmrc - package.json#.packageManager - pnpm-lock.yaml - secrets: id=npm,src=tmp/.npmrc + secrets: id=npm,src=/tmp/.npmrc - &private-npm seek-oss/private-npm#v1.2.0: env: NPM_READ_TOKEN - output-path: tmp/ + output-path: /tmp/ base-steps: - &deploy