Auto generate model folder (#1255)

internals/scripts/package.json

@@ -18,10 +18,13 @@
     "write:docs:guides": "wireit",
     "write:docs:models": "wireit",
     "write:docs:api": "wireit",
+    "write:model:docs": "wireit",
+    "write:model:folder": "wireit",
     "start:guide": "wireit",
     "validate": "wireit",
     "update:npm-dependencies": "wireit",
     "build": "wireit",
+    "copy-files": "wireit",
     "test:run": "wireit",
     "test": "wireit"
   },
@@ -58,6 +61,18 @@
         "build"
       ]
     },
+    "write:model:docs": {
+      "command": "node ./dist/bin/write/model/docs/index.js",
+      "dependencies": [
+        "build"
+      ]
+    },
+    "write:model:folder": {
+      "command": "node ./dist/bin/write/model/folder/index.js",
+      "dependencies": [
+        "build"
+      ]
+    },
     "start:guide": {
       "command": "node ./dist/bin/start/guide.js",
       "dependencies": [
@@ -71,15 +86,14 @@
       ]
     },
     "build": {
-      "command": "tsc -p ./tsconfig.json",
+      "command": "pnpm copy-files && tsc -p ./tsconfig.json",
       "dependencies": [
         "../common:build"
       ],
       "files": [
         "src/**/*.ts",
-        "src/**/*.mts",
         "!src/**/*.test.ts",
-        "!src/**/*.test.mts",
+        "**/_templates/**/*.ejs",
         "package.json",
         "vite.config.ts",
         "tsconfig.json"
@@ -88,6 +102,9 @@
         "dist/**"
       ]
     },
+    "copy-files": {
+      "command": "rsync -avmq --include='*.ejs' -f 'hide,! */' ./src/ ./dist"
+    },
     "test:run": {
       "command": "vitest run --config ./vite.config.ts",
       "dependencies": [

internals/scripts/src/bin/write/model/docs/index.ts

@@ -0,0 +1,116 @@
+import path from 'path';
+import { exists, readFile, readdir, writeFile } from '@internals/common/fs';
+import { MODELS_DIR, SHARED_DIR } from '@internals/common/constants';
+import { error, info } from '@internals/common/logger';
+import { JSONSchema, getPackageJSON } from '@internals/common/package-json';
+import { getModels } from '../shared/getModels.js';
+
+const getModelFamily = (packageJSON: JSONSchema) => packageJSON['@upscalerjs']?.['modelFamily'];
+
+const getSharedDoc = async (modelFamily: string) => {
+  const sharedDoc = path.resolve(SHARED_DIR, 'src', modelFamily, 'DOC.mdx');
+  if (!await exists(sharedDoc)) {
+    throw new Error(`File does not exist: ${sharedDoc}`)
+  }
+  return await readFile(sharedDoc);
+};
+
+const getSnippets = async (model: string): Promise<Record<string, string>> => {
+  const snippets: Record<string, string> = {};
+  const docSnippetsPath = path.resolve(MODELS_DIR, model, 'doc-snippets');
+  if (!await exists(docSnippetsPath)) {
+    throw new Error(`doc snippets folder does not exist at "${docSnippetsPath}"`)
+  }
+  const snippetPaths = await readdir(docSnippetsPath);
+
+  for (const snippetPath of snippetPaths) {
+    const snippet = await readFile(path.resolve(docSnippetsPath, snippetPath)) ?? '';
+    const snippetKey = snippetPath.split('.').slice(0, -1).join('.');
+    if (typeof snippetKey !== 'string') {
+      throw new Error(`Bad snippet key: ${snippetKey}`)
+    }
+    snippets[`snippets/${snippetKey}`] = snippet.trim();
+  }
+  return snippets;
+};
+
+const getPackageJSONArgs = async (model: string, packageJSON: JSONSchema): Promise<Record<string, string | undefined>> => {
+  const name = packageJSON.name;
+  if (!name) {
+    throw new Error(`No name defined for packageJSON for model ${model}`);
+  }
+
+  return {
+    key: name.split("@upscalerjs/").pop(),
+    description: `Overview of @upscalerjs/${model} model`,
+    title: packageJSON['@upscalerjs']?.title,
+    ...(await getSnippets(model)),
+  };
+};
+
+const getKey = (match: string) => match.match(/<%(.*)%>/)?.[1].trim();
+
+const getPreparedDoc = async (model: string) => {
+  const packageJSON = await getPackageJSON(path.resolve(MODELS_DIR, model, 'package.json'));
+  const modelFamily = getModelFamily(packageJSON);
+  if (!modelFamily) {
+    throw new Error(`No explicit model family defined in package JSON: ${model}`)
+  }
+
+  const sharedDoc = await getSharedDoc(modelFamily);
+  const args = await getPackageJSONArgs(model, packageJSON);
+  const matches = sharedDoc.matchAll(/<%.+?%>/g);
+  const chunks: (string | undefined)[] = [];
+  let start = 0;
+  for (const match of matches) {
+    const key = getKey(match[0]);
+    if (key === undefined) {
+      throw new Error(`An undefined key was returned from the match "${match[0]}" for model ${model}`);
+    } else if (!(key in args)) {
+      throw new Error(`Key "${key}" for model family ${modelFamily} and model ${model} was not found in args. Did you mean to prepend it with 'snippets/'? Args is: ${JSON.stringify(args, null, 2)}}`);
+    } else if (typeof args[key] !== 'string') {
+      throw new Error(`Key "${key}" for model family ${modelFamily} and model ${model} is not a string, it is: ${typeof args[key]}`)
+    } else {
+      const matchStart = match?.index ?? 0;
+      const matchEnd = matchStart + (match[0]?.length ?? 0);
+
+      chunks.push(sharedDoc.slice(start, matchStart));
+      chunks.push(args[key])
+      start = matchEnd;
+    }
+  }
+  chunks.push(sharedDoc.slice(start));
+  return chunks.join('');
+}
+
+/****
+ * Main function
+ */
+
+const writeModelDocs = async (
+  models: Array<string>,
+) => {
+  await Promise.all(models.map(async model => {
+    const updatedDoc = await getPreparedDoc(model);
+    const targetPath = path.resolve(MODELS_DIR, model, 'DOC.mdx');
+
+    await readFile(targetPath);
+
+    await writeFile(targetPath, updatedDoc);
+  }));
+}
+
+const main = async () => {
+  const validModels = await getModels();
+
+  if (validModels.length === 0) {
+    error('No models selected, nothing to do.')
+    return;
+  }
+
+
+  info(`Writing model docs for models:\n${validModels.map(m => `- ${m}`).join('\n')}`);
+  await writeModelDocs(validModels);
+};
+
+main();

internals/scripts/src/bin/write/model/folder/_templates/package.json.ejs

@@ -0,0 +1,82 @@
+{
+  "name": "<%- name %>",
+  "version": "<%- version %>",
+  "description": "<%- description %>",
+  "keywords": <%- JSON.stringify(keywords, null, 2) %>,
+  "author": "Kevin Scott",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/thekevinscott/upscaler.git"
+  },
+  "exports": <%- JSON.stringify(exports, null, 2) %>,
+  "module": "<%- module %>",
+  "types": "<%- types %>",
+  "umd:main": "<%- umdMain %>",
+  "scripts": <%- JSON.stringify(Object.keys(scripts).reduce((obj, scriptName) => ({
+    ...obj,
+    [scriptName]: 'wireit',
+  }), {
+    "lint": "wireit",
+    "prepublishOnly": "wireit",
+    "validate:build": "wireit",
+    "scaffold": "wireit",
+    "build:umd:tsc": "wireit",
+    "build:umd:index": "wireit",
+    "build:umd:rollup:index": "wireit",
+    "build:umd:uglify:index": "wireit",
+  }), null, 2) %>,
+  "files": [
+    "assets/**/*",
+    "license",
+    "models/**/*",
+    "dist/**/*"
+  ],
+  "peerDependencies": {
+    "@tensorflow/tfjs": "~4.11.0"
+  },
+  "devDependencies": {
+    "@tensorflow/tfjs": "~4.11.0",
+    "@tensorflow/tfjs-core": "~4.11.0",
+    "@tensorflow/tfjs-layers": "~4.11.0",
+    "@tensorflow/tfjs-node": "~4.11.0",
+    "@tensorflow/tfjs-node-gpu": "~4.11.0",
+    "seedrandom": "3.0.5",
+    "wireit": "^0.14.0"
+  },
+  "@upscalerjs": {
+    "title": "<%- title %>",
+    <% if (!!locals.models) { %>"models": <%- JSON.stringify(models, null, 2) %>,<% } %>
+    "modelFamily": "<%- modelFamily %>"
+  },
+  "wireit": <%- JSON.stringify(Object.entries(scripts).reduce((obj, [name, value]) => ({
+    ...obj,
+    [name]: value,
+  }), {
+    "lint": {
+      "command": "eslint -c ../.eslintrc.js src --ext .ts",
+      "dependencies": [
+        "scaffold"
+      ]
+    },
+    "prepublishOnly": {
+      "command": "pnpm lint && pnpm build && pnpm validate:build"
+    },
+    "scaffold": {
+      "command": "node -e 'const fs = require(\"fs\"); const {name, version} = require(\"./package.json\"); fs.writeFileSync(\"./src/constants.generated.ts\", `export const NAME = \"${name}\";\\nexport const VERSION=\"${version}\"`, \"utf-8\");'",
+      "files": [
+        "package.json"
+      ],
+      "output": [
+        "./src/constants.generated.ts"
+      ]
+    },
+    "build:umd:tsc": {
+      "command": "tsc -p ./tsconfig.umd.json --outDir ./dist/umd-tmp"
+    },
+    "build:umd:index": {
+      "command": "pnpm build:umd:rollup:index && pnpm build:umd:uglify:index"
+    },
+  }), null, 2) %>
+}
+