diff --git a/internals/scripts/package.json b/internals/scripts/package.json index acdc393f7..447476daf 100644 --- a/internals/scripts/package.json +++ b/internals/scripts/package.json @@ -18,10 +18,13 @@ "write:docs:guides": "wireit", "write:docs:models": "wireit", "write:docs:api": "wireit", + "write:model:docs": "wireit", + "write:model:folder": "wireit", "start:guide": "wireit", "validate": "wireit", "update:npm-dependencies": "wireit", "build": "wireit", + "copy-files": "wireit", "test:run": "wireit", "test": "wireit" }, @@ -58,6 +61,18 @@ "build" ] }, + "write:model:docs": { + "command": "node ./dist/bin/write/model/docs/index.js", + "dependencies": [ + "build" + ] + }, + "write:model:folder": { + "command": "node ./dist/bin/write/model/folder/index.js", + "dependencies": [ + "build" + ] + }, "start:guide": { "command": "node ./dist/bin/start/guide.js", "dependencies": [ @@ -71,15 +86,14 @@ ] }, "build": { - "command": "tsc -p ./tsconfig.json", + "command": "pnpm copy-files && tsc -p ./tsconfig.json", "dependencies": [ "../common:build" ], "files": [ "src/**/*.ts", - "src/**/*.mts", "!src/**/*.test.ts", - "!src/**/*.test.mts", + "**/_templates/**/*.ejs", "package.json", "vite.config.ts", "tsconfig.json" @@ -88,6 +102,9 @@ "dist/**" ] }, + "copy-files": { + "command": "rsync -avmq --include='*.ejs' -f 'hide,! */' ./src/ ./dist" + }, "test:run": { "command": "vitest run --config ./vite.config.ts", "dependencies": [ diff --git a/internals/scripts/src/bin/write/model/docs/index.ts b/internals/scripts/src/bin/write/model/docs/index.ts new file mode 100644 index 000000000..57f7abc04 --- /dev/null +++ b/internals/scripts/src/bin/write/model/docs/index.ts @@ -0,0 +1,116 @@ +import path from 'path'; +import { exists, readFile, readdir, writeFile } from '@internals/common/fs'; +import { MODELS_DIR, SHARED_DIR } from '@internals/common/constants'; +import { error, info } from '@internals/common/logger'; +import { JSONSchema, getPackageJSON } from '@internals/common/package-json'; +import { getModels } from '../shared/getModels.js'; + +const getModelFamily = (packageJSON: JSONSchema) => packageJSON['@upscalerjs']?.['modelFamily']; + +const getSharedDoc = async (modelFamily: string) => { + const sharedDoc = path.resolve(SHARED_DIR, 'src', modelFamily, 'DOC.mdx'); + if (!await exists(sharedDoc)) { + throw new Error(`File does not exist: ${sharedDoc}`) + } + return await readFile(sharedDoc); +}; + +const getSnippets = async (model: string): Promise> => { + const snippets: Record = {}; + const docSnippetsPath = path.resolve(MODELS_DIR, model, 'doc-snippets'); + if (!await exists(docSnippetsPath)) { + throw new Error(`doc snippets folder does not exist at "${docSnippetsPath}"`) + } + const snippetPaths = await readdir(docSnippetsPath); + + for (const snippetPath of snippetPaths) { + const snippet = await readFile(path.resolve(docSnippetsPath, snippetPath)) ?? ''; + const snippetKey = snippetPath.split('.').slice(0, -1).join('.'); + if (typeof snippetKey !== 'string') { + throw new Error(`Bad snippet key: ${snippetKey}`) + } + snippets[`snippets/${snippetKey}`] = snippet.trim(); + } + return snippets; +}; + +const getPackageJSONArgs = async (model: string, packageJSON: JSONSchema): Promise> => { + const name = packageJSON.name; + if (!name) { + throw new Error(`No name defined for packageJSON for model ${model}`); + } + + return { + key: name.split("@upscalerjs/").pop(), + description: `Overview of @upscalerjs/${model} model`, + title: packageJSON['@upscalerjs']?.title, + ...(await getSnippets(model)), + }; +}; + +const getKey = (match: string) => match.match(/<%(.*)%>/)?.[1].trim(); + +const getPreparedDoc = async (model: string) => { + const packageJSON = await getPackageJSON(path.resolve(MODELS_DIR, model, 'package.json')); + const modelFamily = getModelFamily(packageJSON); + if (!modelFamily) { + throw new Error(`No explicit model family defined in package JSON: ${model}`) + } + + const sharedDoc = await getSharedDoc(modelFamily); + const args = await getPackageJSONArgs(model, packageJSON); + const matches = sharedDoc.matchAll(/<%.+?%>/g); + const chunks: (string | undefined)[] = []; + let start = 0; + for (const match of matches) { + const key = getKey(match[0]); + if (key === undefined) { + throw new Error(`An undefined key was returned from the match "${match[0]}" for model ${model}`); + } else if (!(key in args)) { + throw new Error(`Key "${key}" for model family ${modelFamily} and model ${model} was not found in args. Did you mean to prepend it with 'snippets/'? Args is: ${JSON.stringify(args, null, 2)}}`); + } else if (typeof args[key] !== 'string') { + throw new Error(`Key "${key}" for model family ${modelFamily} and model ${model} is not a string, it is: ${typeof args[key]}`) + } else { + const matchStart = match?.index ?? 0; + const matchEnd = matchStart + (match[0]?.length ?? 0); + + chunks.push(sharedDoc.slice(start, matchStart)); + chunks.push(args[key]) + start = matchEnd; + } + } + chunks.push(sharedDoc.slice(start)); + return chunks.join(''); +} + +/**** + * Main function + */ + +const writeModelDocs = async ( + models: Array, +) => { + await Promise.all(models.map(async model => { + const updatedDoc = await getPreparedDoc(model); + const targetPath = path.resolve(MODELS_DIR, model, 'DOC.mdx'); + + await readFile(targetPath); + + await writeFile(targetPath, updatedDoc); + })); +} + +const main = async () => { + const validModels = await getModels(); + + if (validModels.length === 0) { + error('No models selected, nothing to do.') + return; + } + + + info(`Writing model docs for models:\n${validModels.map(m => `- ${m}`).join('\n')}`); + await writeModelDocs(validModels); +}; + +main(); diff --git a/internals/scripts/src/bin/write/model/folder/_templates/package.json.ejs b/internals/scripts/src/bin/write/model/folder/_templates/package.json.ejs new file mode 100644 index 000000000..05c41f35e --- /dev/null +++ b/internals/scripts/src/bin/write/model/folder/_templates/package.json.ejs @@ -0,0 +1,82 @@ +{ + "name": "<%- name %>", + "version": "<%- version %>", + "description": "<%- description %>", + "keywords": <%- JSON.stringify(keywords, null, 2) %>, + "author": "Kevin Scott", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/thekevinscott/upscaler.git" + }, + "exports": <%- JSON.stringify(exports, null, 2) %>, + "module": "<%- module %>", + "types": "<%- types %>", + "umd:main": "<%- umdMain %>", + "scripts": <%- JSON.stringify(Object.keys(scripts).reduce((obj, scriptName) => ({ + ...obj, + [scriptName]: 'wireit', + }), { + "lint": "wireit", + "prepublishOnly": "wireit", + "validate:build": "wireit", + "scaffold": "wireit", + "build:umd:tsc": "wireit", + "build:umd:index": "wireit", + "build:umd:rollup:index": "wireit", + "build:umd:uglify:index": "wireit", + }), null, 2) %>, + "files": [ + "assets/**/*", + "license", + "models/**/*", + "dist/**/*" + ], + "peerDependencies": { + "@tensorflow/tfjs": "~4.11.0" + }, + "devDependencies": { + "@tensorflow/tfjs": "~4.11.0", + "@tensorflow/tfjs-core": "~4.11.0", + "@tensorflow/tfjs-layers": "~4.11.0", + "@tensorflow/tfjs-node": "~4.11.0", + "@tensorflow/tfjs-node-gpu": "~4.11.0", + "seedrandom": "3.0.5", + "wireit": "^0.14.0" + }, + "@upscalerjs": { + "title": "<%- title %>", + <% if (!!locals.models) { %>"models": <%- JSON.stringify(models, null, 2) %>,<% } %> + "modelFamily": "<%- modelFamily %>" + }, + "wireit": <%- JSON.stringify(Object.entries(scripts).reduce((obj, [name, value]) => ({ + ...obj, + [name]: value, + }), { + "lint": { + "command": "eslint -c ../.eslintrc.js src --ext .ts", + "dependencies": [ + "scaffold" + ] + }, + "prepublishOnly": { + "command": "pnpm lint && pnpm build && pnpm validate:build" + }, + "scaffold": { + "command": "node -e 'const fs = require(\"fs\"); const {name, version} = require(\"./package.json\"); fs.writeFileSync(\"./src/constants.generated.ts\", `export const NAME = \"${name}\";\\nexport const VERSION=\"${version}\"`, \"utf-8\");'", + "files": [ + "package.json" + ], + "output": [ + "./src/constants.generated.ts" + ] + }, + "build:umd:tsc": { + "command": "tsc -p ./tsconfig.umd.json --outDir ./dist/umd-tmp" + }, + "build:umd:index": { + "command": "pnpm build:umd:rollup:index && pnpm build:umd:uglify:index" + }, + }), null, 2) %> +} + diff --git a/internals/scripts/src/bin/write/model/folder/index.ts b/internals/scripts/src/bin/write/model/folder/index.ts new file mode 100644 index 000000000..4c3e8d195 --- /dev/null +++ b/internals/scripts/src/bin/write/model/folder/index.ts @@ -0,0 +1,235 @@ +import path from 'path'; +import * as url from 'url'; +import asyncPool from "tiny-async-pool"; +import { readFile, writeFile, } from '@internals/common/fs'; +import { MODELS_DIR, } from '@internals/common/constants'; +import { error, info, } from '@internals/common/logger'; +import { getPackageJSON } from '@internals/common/package-json'; +import { getTemplate as _getTemplate, } from '@internals/common/get-template'; +import { getModels } from '../shared/getModels.js'; + +interface ModelDefinition { + exportKey: string; + umdPath: string; + requirePath: string; + importPath: string; +} + +const NUMBER_OF_CONCURRENT_THREADS = 5; + +const __dirname = url.fileURLToPath(new URL('.', import.meta.url)); +const TEMPLATES_DIR = path.resolve(__dirname, '_templates'); + +const getTemplate = ( + templateName: string, + args: Parameters[1] = {} +) => _getTemplate(path.resolve(TEMPLATES_DIR, templateName), args); + +const validateExport = (entry: [string, unknown]): entry is [string, { + umd: string; + import: string; + require: string; +}] => { + const xport = entry[1]; + return typeof xport === 'object' && xport !== null && 'umd' in xport && 'require' in xport && 'import' in xport; +}; + +const getUMDNames = async (modelDirectoryPath: string) => JSON.parse(await readFile(path.resolve(modelDirectoryPath, 'umd-names.json'))); +const getUmdKey = (umdPath: string) => umdPath.split('./').pop(); + +const writeModelFolder = async (modelDirectoryName: string) => { + const modelDirectoryPath = path.resolve(MODELS_DIR, modelDirectoryName); + const { + name, + version, + description, + keywords, + module, + types, + 'umd:main': umdMain, + exports: xports, + '@upscalerjs': { + title, + modelFamily, + models: supplementaryModelInformation, + }, + } = await getPackageJSON(modelDirectoryPath); + if (!umdMain) { + throw new Error(`No umd-main found in ${modelDirectoryName}/package.json`) + } + const umdNames = await getUMDNames(modelDirectoryPath); + const umdIndexName = umdNames['.']; + if (!umdIndexName) { + throw new Error(`No UMD index name defined for . in ${modelDirectoryName}`) + } + + const validExports = Object.entries(xports).filter(validateExport); + if (validExports.length === 0) { + throw new Error(`No valid exports for model ${modelDirectoryName}. Exports: ${JSON.stringify(xports, null, 2)}`); + } + + const definedModels: ModelDefinition[] = Object.entries(xports).filter(validateExport).map(([exportKey, { + umd: umdPath, + require: requirePath, + import: importPath, + }]) => ({ + exportKey, + umdPath, + requirePath, + importPath, + })); + if (definedModels.length === 0) { + throw new Error(`No defined models for model ${modelDirectoryName}`); + } + const nonIndexModels = definedModels.filter(({ exportKey }) => exportKey !== '.'); + const sharedFileDependenies = [ + "src/**/*.ts", + "!src/**/*.test.ts", + "!src/**/*.generated.ts", + "../../packages/shared/src/esrgan/**/*.ts", + "package.json", + "tsconfig.json", + "../tsconfig.json" + ]; + const umdFileDependencies = [ + "tsconfig.umd.json", + "umd-names.json", + "../tsconfig.umd.json", + ]; + const esmFileDependencies = [ + "tsconfig.esm.json", + "../tsconfig.esm.json", + ]; + const cjsFileDependencies = [ + "tsconfig.cjs.json", + "../tsconfig.cjs.json", + ]; + const exports = definedModels.reduce((obj, { exportKey, umdPath, requirePath, importPath }) => ({ + ...obj, + [exportKey]: { + umd: umdPath, + require: requirePath, + import: importPath, + }, + }), {}); + const vars = { + modelDirectoryName, + name, + version, + description, + keywords, + exports, + module, + types, + umdMain, + scripts: nonIndexModels.reduce((obj, { exportKey: key, umdPath }) => { + const umdName = umdNames[key]; + if (!umdName) { + throw new Error(`No UMD index name defined for ${key} in ${modelDirectoryName}`) + } + const nonMinifiedOut = umdPath.replace('.min', ''); + const originalPath = nonMinifiedOut.replace('dist/umd', 'dist/umd-tmp'); + return { + ...obj, + [`build:umd:${getUmdKey(key)}`]: { + "command": `pnpm build:umd:rollup:${getUmdKey(key)} && pnpm build:umd:uglify:${getUmdKey(key)}` + }, + [`build:umd:rollup:${getUmdKey(key)}`]: { + "command": `rollup -c ../rollup.config.cjs --input ${originalPath} --file ${nonMinifiedOut} --name ${umdName} --format umd` + }, + [`build:umd:uglify:${getUmdKey(key)}`]: { + "command": `uglifyjs ${nonMinifiedOut} --output ${umdPath} --compress --mangle --source-map` + }, + }; + }, { + "validate:build": { + "command": `ts-node ../../scripts/package-scripts/validate-build.ts models/${modelDirectoryName}` + }, + "build:umd:rollup:index": { + "command": `rollup -c ../rollup.config.cjs --input ./dist/umd-tmp/models/${modelDirectoryName}/src/umd.js --file ./dist/umd/models/${modelDirectoryName}/src/umd.js --name ${umdIndexName} --format umd` + }, + "build:umd:uglify:index": { + "command": `uglifyjs ./dist/umd/models/${modelDirectoryName}/src/umd.js --output ./dist/umd/models/${modelDirectoryName}/src/umd.min.js --compress --mangle --source-map` + }, + "build": { + "command": "pnpm build:cjs && pnpm build:esm && pnpm build:umd", + "files": [ + ...sharedFileDependenies, + ...umdFileDependencies, + ...esmFileDependencies, + ...cjsFileDependencies, + ], + "output": [ + "dist/**" + ] + }, + "build:cjs": { + "command": "tsc -p ./tsconfig.cjs.json --outDir ./dist/cjs --baseUrl ./src", + "dependencies": [ + "scaffold" + ], + "files": [ + ...sharedFileDependenies, + ...cjsFileDependencies, + ], + "output": [ + "dist/cjs/**" + ] + }, + "build:esm": { + "command": "tsc -p ./tsconfig.esm.json --outDir ./dist/esm --baseUrl ./src", + "dependencies": [ + "scaffold" + ], + "files": [ + ...sharedFileDependenies, + ...esmFileDependencies, + ], + "output": [ + "dist/esm/**" + ] + }, + "build:umd": { + "command": `pnpm run build:umd:tsc && pnpm run build:umd:index && ${nonIndexModels.map(m => `pnpm run build:umd:${getUmdKey(m.exportKey)}`).join(' && ')} && rimraf ./dist/umd-tmp`, + "dependencies": [ + "scaffold" + ], + "files": [ + ...sharedFileDependenies, + ...umdFileDependencies, + ], + "output": [ + "dist/umd/**" + ] + }, + }), + title, + modelFamily, + models: supplementaryModelInformation, + }; + const newPackageJSON = JSON.parse(await getTemplate('package.json.ejs', vars)); + await writeFile(path.resolve(modelDirectoryPath, 'package.json'), `${JSON.stringify(newPackageJSON, null, 2)}\n`); +}; + +const writeModelFolders = async ( + models: Array, +) => { + for await (const _ of asyncPool(NUMBER_OF_CONCURRENT_THREADS, models, writeModelFolder)) { + // empty + } +} + +const main = async () => { + const validModels = await getModels(); + + if (validModels.length === 0) { + error('No models selected, nothing to do.') + return; + } + + + info(`Writing model folders for models:\n${validModels.map(m => `- ${m}`).join('\n')}`); + await writeModelFolders(validModels); +}; + +main(); diff --git a/internals/scripts/src/bin/write/model/shared/getModels.ts b/internals/scripts/src/bin/write/model/shared/getModels.ts new file mode 100644 index 000000000..47087c39e --- /dev/null +++ b/internals/scripts/src/bin/write/model/shared/getModels.ts @@ -0,0 +1,57 @@ +import path from 'path'; +import { exists, readdir, stat } from '@internals/common/fs'; +import { MODELS_DIR } from '@internals/common/constants'; +import { warn } from '@internals/common/logger'; +import { parseArgs } from "node:util"; + +const isDirectory = async (path: string) => (await stat(path)).isDirectory(); + +const isValidModel = async (modelDirectoryName: string) => { + const modelDirectoryPath = path.resolve(MODELS_DIR, modelDirectoryName); + return await exists(modelDirectoryPath) && await isDirectory(modelDirectoryPath); +}; + +const getModelDirectories = async () => { + const modelDirectories: string[] = []; + for (const modelDirectoryName of await readdir(MODELS_DIR)) { + if (await isDirectory(path.resolve(MODELS_DIR, modelDirectoryName))) { + modelDirectories.push(modelDirectoryName); + } + }; + return modelDirectories; +} + +const expandModel = async (model: string): Promise => { + if (model.includes('*')) { + const modelNameMatch = model.split('*')[0]; + const models: string[] = []; + for (const modelDirectoryName of await getModelDirectories()) { + if (modelDirectoryName.startsWith(modelNameMatch)) { + models.push(modelDirectoryName); + } + } + return models; + } + return [model]; +}; + +export const getModels = async (): Promise => { + const { + positionals: models, + } = parseArgs({ + allowPositionals: true, + }); + + const validModels = new Set(); + for (const modelName of models) { + for (const model of await expandModel(modelName)) { + if (await isValidModel(model)) { + validModels.add(model); + } else { + warn(`Invalid model: ${model}`); + } + } + }; + + return Array.from(validModels); +}