feat(cli): wing pack (winglang#3938)

Introduces the `wing pack` command - a standard way to package libraries in Wing.

`wing pack` can be run in any Wing project directory that has a `package.json` in it. `wing pack` will validate your `package.json` produce a tarball that contains the libraries source files. For more documentation, see https://winglang.io/docs/libraries.

## Implementation

When you `bring "foo"`, the parser now performs some Node module resolution to look up the `foo` package and see if it's a JSII library or a Wing library based on its `package.json`. If `bring "foo"` refers to a Wing library, then it's equivalent to bringing that library's root directory, per winglang#4210.

## Future related work
- winglang#4415
- winglang#4294
- winglang#2171

## Checklist

- [x] Title matches [Winglang's style guide](https://www.winglang.io/contributing/start-here/pull_requests#how-are-pull-request-titles-formatted)
- [x] Description explains motivation and solution
- [x] Tests added (always)
- [x] Docs updated (only required for features)
- [ ] Added `pr/e2e-full` label if this feature requires end-to-end testing

*By submitting this pull request, I confirm that my contribution is made under the terms of the [Wing Cloud Contribution License](https://github.com/winglang/wing/blob/main/CONTRIBUTION_LICENSE.md)*.

apps/wing/fixtures/.gitignore

@@ -0,0 +1 @@
+*.tgz

apps/wing/fixtures/invalid1/main.w

@@ -0,0 +1,3 @@
+// example of a project missing a package.json file
+bring cloud;
+new cloud.Bucket();

apps/wing/fixtures/invalid2/main.w

@@ -0,0 +1,3 @@
+// example of a project missing "license" in package.json
+bring cloud;
+new cloud.Bucket();

apps/wing/fixtures/invalid2/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "invalid2",
+  "version": "0.0.0",
+  "description": "description",
+  "author": "author"
+}

apps/wing/fixtures/invalid3/file1.w

@@ -0,0 +1,6 @@
+// example of a project that doesn't compile
+class A {
+  init() {
+    log("Hello world!") // missing ';'
+  }
+}

apps/wing/package.json

@@ -31,6 +31,7 @@
     "package": "bump-pack -b"
   },
   "dependencies": {
+    "@npmcli/arborist": "^7.2.0",
     "@segment/analytics-node": "^1.1.0",
     "@wingconsole/app": "workspace:^",
     "@wingconsole/server": "workspace:^",
@@ -41,9 +42,12 @@
     "commander": "^10.0.1",
     "compare-versions": "^5.0.3",
     "debug": "^4.3.4",
+    "minimatch": "^9.0.3",
     "nanoid": "^3.3.6",
+    "npm-packlist": "^8.0.0",
     "open": "^8.4.2",
     "ora": "^5.4.1",
+    "tar": "^6.2.0",
     "tiny-updater": "^3.5.1",
     "uuid": "^8.3.2",
     "vscode-languageserver": "^8.1.0"
@@ -52,6 +56,9 @@
     "@types/debug": "^4.1.8",
     "@types/node": "^18.17.13",
     "@types/node-persist": "^3.1.4",
+    "@types/npmcli__arborist": "^5.6.2",
+    "@types/npm-packlist": "^7.0.1",
+    "@types/tar": "^6.1.6",
     "@types/semver-utils": "^1.1.1",
     "@types/uuid": "^8.3.4",
     "@typescript-eslint/eslint-plugin": "^6.7.4",

apps/wing/src/cli.ts

@@ -163,6 +163,13 @@ async function main() {
     .hook("preAction", collectAnalyticsHook)
     .action(runSubCommand("test"));
 
+  program
+    .command("pack")
+    .description("Package the current directory into an npm library (gzipped tarball).")
+    .addOption(new Option("-o --out-file <filename>", "Output filename"))
+    .hook("preAction", collectAnalyticsHook)
+    .action(runSubCommand("pack"));
+
   program
     .command("docs")
     .description("Open the Wing documentation")

apps/wing/src/commands/pack.test.ts

@@ -0,0 +1,133 @@
+import * as fs from "fs/promises";
+import { join } from "path";
+import { describe, it, expect, afterEach, vi } from "vitest";
+import { pack } from "./pack";
+import { exec, generateTmpDir } from "src/util";
+
+const fixturesDir = join(__dirname, "..", "..", "fixtures");
+const goodFixtureDir = join(__dirname, "..", "..", "..", "..", "examples", "wing-fixture");
+
+console.log = vi.fn();
+
+describe("wing pack", () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it("throws an error if a project is missing package.json", async () => {
+    const projectDir = join(fixturesDir, "invalid1");
+    const outdir = await generateTmpDir();
+    process.chdir(projectDir);
+    await expect(pack({ outfile: join(outdir, "tarball.tgz") })).rejects.toThrow(
+      /No package.json found in the current directory./
+    );
+    await expectNoTarball(outdir);
+  });
+
+  it("throws an error if package.json is missing a required field", async () => {
+    const projectDir = join(fixturesDir, "invalid2");
+    const outdir = await generateTmpDir();
+    process.chdir(projectDir);
+
+    await expect(pack({ outfile: join(outdir, "tarball.tgz") })).rejects.toThrow(
+      /Missing required field "license" in package.json/
+    );
+    await expectNoTarball(outdir);
+  });
+
+  it("throws an error if the project doesn't compile", async () => {
+    const projectDir = join(fixturesDir, "invalid3");
+    const outdir = await generateTmpDir();
+    process.chdir(projectDir);
+
+    await expect(pack({ outfile: join(outdir, "tarball.tgz") })).rejects.toThrow(/Expected ';'/);
+    await expectNoTarball(outdir);
+  });
+
+  it("packages a valid Wing project to a default path", async () => {
+    // GIVEN
+    const outdir = await generateTmpDir();
+    // copy everything to the output directory to sandbox this test
+    await exec(`cp -r ${goodFixtureDir}/* ${outdir}`);
+    process.chdir(outdir);
+
+    // WHEN
+    await pack();
+
+    // THEN
+    const files = await fs.readdir(outdir);
+    expect(files.filter((path) => path.endsWith(".tgz")).length).toEqual(1);
+    const tarballPath = files.find((path) => path.endsWith(".tgz"))!;
+    const tarballContents = await extractTarball(join(outdir, tarballPath), outdir);
+
+    const expectedFiles = ["index.js", "README.md", "package.json", "store.w"];
+    for (const file of expectedFiles) {
+      expect(tarballContents[file]).toBeDefined();
+    }
+
+    const pkgJson = JSON.parse(tarballContents["package.json"]);
+    expect(pkgJson.name).toEqual("wing-fixture");
+    expect(pkgJson.keywords.includes("winglang")).toBe(true);
+    expect(pkgJson.engines.wing).toEqual("*");
+    expect(pkgJson.wing).toEqual(true);
+  });
+
+  it("packages a valid Wing project to a user-specified path", async () => {
+    // GIVEN
+    const projectDir = goodFixtureDir;
+    const outdir = await generateTmpDir();
+    process.chdir(projectDir);
+
+    // WHEN
+    await pack({ outfile: join(outdir, "tarball.tgz") });
+
+    // THEN
+    const files = await fs.readdir(outdir);
+    expect(files.filter((path) => path.endsWith(".tgz")).length).toEqual(1);
+    const tarballPath = files.find((path) => path.endsWith(".tgz"))!;
+    const tarballContents = await extractTarball(join(outdir, tarballPath), outdir);
+
+    const expectedFiles = ["index.js", "README.md", "package.json", "store.w"];
+    for (const file of expectedFiles) {
+      expect(tarballContents[file]).toBeDefined();
+    }
+
+    const pkgJson = JSON.parse(tarballContents["package.json"]);
+    expect(pkgJson.name).toEqual("wing-fixture");
+    expect(pkgJson.keywords.includes("winglang")).toBe(true);
+    expect(pkgJson.engines.wing).toEqual("*");
+    expect(pkgJson.wing).toEqual(true);
+  });
+});
+
+/**
+ * Asserts that no tarball was created in the specified directory.
+ */
+async function expectNoTarball(projectDir: string) {
+  const files = await fs.readdir(projectDir);
+  expect(files.findIndex((path) => path.endsWith(".tgz"))).toEqual(-1);
+}
+
+async function extractTarball(src: string, outdir: string): Promise<Record<string, string>> {
+  await exec(`tar -xzf ${src} -C ${outdir}`);
+  const contents: Record<string, string> = {};
+
+  // when you extract an npm tarball, it creates a directory called "package"
+  const base = join(outdir, "package");
+
+  async function readDir(dir: string) {
+    const files = await fs.readdir(join(base, dir));
+    for (const file of files) {
+      const path = join(base, dir, file);
+      const stat = await fs.stat(path);
+      if (stat.isDirectory()) {
+        await readDir(join(dir, file));
+      } else {
+        contents[join(dir, file)] = (await fs.readFile(path)).toString();
+      }
+    }
+  }
+  await readDir(".");
+
+  return contents;
+}

apps/wing/src/commands/pack.ts

@@ -0,0 +1,196 @@
+import { constants } from "fs";
+import * as fs from "fs/promises";
+import * as os from "os";
+import * as path from "path";
+import { resolve } from "path";
+import Arborist from "@npmcli/arborist";
+import { Target } from "@winglang/compiler";
+import { minimatch } from "minimatch";
+import packlist from "npm-packlist";
+import * as tar from "tar";
+import { compile } from "./compile";
+
+// TODO: add --dry-run option?
+// TODO: let the user specify library's supported targets in package.json, and compile to each before packaging
+// TODO: print information about the generated library? (e.g. size, dependencies, number of public APIs)
+
+export interface PackageOptions {
+  /**
+   * Output filename.
+   */
+  readonly outfile?: string;
+}
+
+export async function pack(options: PackageOptions = {}): Promise<string> {
+  // check that the library compiles to the "sim" target
+  console.log('Compiling to the "sim" target...');
+
+  // TODO: workaround for https://github.com/winglang/wing/issues/4431
+  // await compile(".", { target: Target.SIM });
+  await compile(path.join("..", path.basename(process.cwd())), { target: Target.SIM });
+
+  const userDir = process.cwd();
+  const outfile = options.outfile ? resolve(options.outfile) : undefined;
+  const outdir = outfile ? path.dirname(outfile) : userDir;
+
+  // check package.json exists
+  const originalPkgJsonPath = path.join(userDir, "package.json");
+  if (!(await exists(originalPkgJsonPath))) {
+    throw new Error(`No package.json found in the current directory. Run \`npm init\` first.`);
+  }
+
+  const originalPkgJson = JSON.parse(await fs.readFile(originalPkgJsonPath, "utf8"));
+  const originalPkgJsonFiles: Set<string> = new Set(originalPkgJson.files ?? []);
+
+  // perform our work in a staging directory to avoid making a mess in the user's current directory
+  return withTempDir(async (workdir) => {
+    const excludeGlobs = ["/target/**", "/node_modules/**", "/.git/**", "/.wing/**"];
+    const includeGlobs = [
+      ...originalPkgJsonFiles,
+      "README*",
+      "package.json",
+      "**/*.w",
+      "**/*.js",
+      "LICENSE*",
+    ];
+
+    // copy the user's directory to the staging directory
+    await copyDir(userDir, workdir, excludeGlobs, includeGlobs);
+
+    // check package.json exists
+    const pkgJsonPath = path.join(workdir, "package.json");
+    if (!(await exists(pkgJsonPath))) {
+      throw new Error(`No package.json found in the current directory. Run \`npm init\` first.`);
+    }
+
+    const pkgJson = JSON.parse(await fs.readFile(pkgJsonPath, "utf8"));
+
+    // check package.json has required fields
+    const requiredFields = ["name", "version", "description", "author", "license"];
+    for (const field of requiredFields) {
+      if (pkgJson[field] === undefined) {
+        throw new Error(`Missing required field "${field}" in package.json.`);
+      }
+    }
+
+    // check that Wing source files will be included in the tarball
+    const pkgJsonFiles = new Set(pkgJson.files ?? []);
+    const expectedGlobs = ["**/*.js", "**/*.w"];
+    for (const glob of expectedGlobs) {
+      if (!pkgJsonFiles.has(glob)) {
+        pkgJsonFiles.add(glob);
+      }
+    }
+    pkgJson.files = [...pkgJsonFiles];
+
+    // check if "main" points to a valid file, and if not, create a dummy file
+    let main = pkgJson.main ?? "index.js";
+    if (!(await exists(main))) {
+      const lines = [];
+      lines.push("// This file was generated by `wing pack`");
+      lines.push(
+        `console.log("${pkgJson.name} is a winglang library and cannot be used from javascript (yet). See winglang.io/docs/libraries")`
+      );
+      lines.push();
+      await fs.writeFile(main, lines.join("\n"));
+    }
+    pkgJson.main = main;
+
+    // add "winglang" to "keywords"
+    const keywords = new Set(pkgJson.keywords ?? []);
+    keywords.add("winglang");
+    pkgJson.keywords = [...keywords];
+
+    // add "wing" to "engines"
+    pkgJson.engines = { wing: "*" };
+
+    // add "wing" top-level field
+    pkgJson.wing = true;
+
+    // write package.json
+    await fs.writeFile(pkgJsonPath, JSON.stringify(pkgJson, null, 2) + "\n");
+
+    // make the tarball
+    const arborist = new Arborist({ path: workdir });
+    const tree = await arborist.loadActual();
+    const pkg = tree.package;
+    const tarballPath = outfile ?? path.join(outdir, `${pkg.name}-${pkg.version}.tgz`);
+    const files = await packlist(tree);
+    await tar.create(
+      {
+        gzip: true,
+        file: tarballPath,
+        cwd: workdir,
+        prefix: "package/",
+        portable: true,
+        noPax: true,
+      },
+      files
+    );
+
+    console.log("Created tarball:", tarballPath);
+    return tarballPath;
+  });
+}
+
+async function copyDir(src: string, dest: string, excludeGlobs: string[], includeGlobs: string[]) {
+  const files = await fs.readdir(src);
+  for (const file of files) {
+    const srcPath = path.join(src, file);
+    const destPath = path.join(dest, file);
+    const stat = await fs.stat(srcPath);
+    if (stat.isDirectory()) {
+      if (shouldInclude(srcPath, excludeGlobs, includeGlobs)) {
+        await copyDir(srcPath, destPath, excludeGlobs, includeGlobs);
+      }
+    } else {
+      await fs.copyFile(srcPath, destPath);
+    }
+  }
+}
+
+function shouldInclude(srcPath: string, excludeGlobs: string[], includeGlobs: string[]): boolean {
+  for (const glob of excludeGlobs) {
+    if (minimatch(srcPath, glob)) {
+      return false;
+    }
+  }
+  for (const glob of includeGlobs) {
+    if (minimatch(srcPath, glob)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Run some work in a temporary directory.
+ */
+async function withTempDir<T>(work: (workdir: string) => Promise<T>): Promise<T> {
+  const workdir = await fs.mkdtemp(path.join(os.tmpdir(), "wing-pack-"));
+  const cwd = process.cwd();
+  try {
+    process.chdir(workdir);
+    // wait for the work to be completed before
+    // we cleanup the work environment.
+    return await work(workdir);
+  } finally {
+    process.chdir(cwd);
+    await fs.rm(workdir, { recursive: true });
+  }
+}
+
+/**
+ * Check if a file exists for an specific path
+ */
+export async function exists(filePath: string): Promise<boolean> {
+  try {
+    await fs.access(
+      filePath,
+      constants.F_OK | constants.R_OK | constants.W_OK //eslint-disable-line no-bitwise
+    );
+    return true;
+  } catch (er) {
+    return false;
+  }
+}