Skip to content

Commit

Permalink
feat: add async transform support
Browse files Browse the repository at this point in the history
  • Loading branch information
makamekm committed Oct 30, 2024
1 parent 07fdda5 commit fc7bd30
Show file tree
Hide file tree
Showing 19 changed files with 588 additions and 287 deletions.
104 changes: 65 additions & 39 deletions src/cmd/build/index.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
import glob from 'glob';
import {Arguments, Argv} from 'yargs';
import {join, resolve} from 'path';
import shell from 'shelljs';
import glob from 'glob';

import OpenapiIncluder from '@diplodoc/openapi-extension/includer';

import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '../../constants';
import {argvValidator} from '../../validator';
import {ArgvService, Includers, SearchService} from '../../services';
import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '~/constants';
import {argvValidator} from '~/validator';
import {Includer, YfmArgv} from '~/models';
import {ArgvService, Includers, SearchService, TocService} from '~/services';
import {
initLinterWorkers,
processAssets,
Expand All @@ -17,10 +18,12 @@ import {
processLogs,
processPages,
processServiceFiles,
} from '../../steps';
import {prepareMapFile} from '../../steps/processMapFile';
import {copyFiles, logger} from '../../utils';
import {upload as publishFilesToS3} from '../../commands/publish/upload';
} from '~/steps';
import {prepareMapFile} from '~/steps/processMapFile';
import {copyFiles, logger} from '~/utils';
import {upload as publishFilesToS3} from '~/commands/publish/upload';
import {RevisionContext, makeRevisionContext} from '~/context/context';
import {FsContextCli} from '~/context/fs';

export const build = {
command: ['build', '$0'],
Expand All @@ -43,6 +46,12 @@ function builder<T>(argv: Argv<T>) {
type: 'string',
group: 'Build options:',
})
.option('plugins', {
alias: 'p',
describe: 'Path to plugins js file',
type: 'string',
group: 'Build options:',
})
.option('varsPreset', {
default: 'default',
describe: 'Target vars preset of documentation <external|internal>',
Expand Down Expand Up @@ -175,7 +184,8 @@ function builder<T>(argv: Argv<T>) {
);
}

async function handler(args: Arguments<any>) {
async function handler(args: Arguments<YfmArgv>) {
const userInputFolder = resolve(args.input);
const userOutputFolder = resolve(args.output);
const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER);
const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER);
Expand All @@ -185,14 +195,15 @@ async function handler(args: Arguments<any>) {
}

try {
// Init singletone services
ArgvService.init({
...args,
rootInput: args.input,
rootInput: userInputFolder,
input: tmpInputFolder,
output: tmpOutputFolder,
});
SearchService.init();
Includers.init([OpenapiIncluder as any]);
Includers.init([OpenapiIncluder as Includer]);

const {
output: outputFolderPath,
Expand All @@ -203,41 +214,64 @@ async function handler(args: Arguments<any>) {
addMapFile,
} = ArgvService.getConfig();

preparingTemporaryFolders(userOutputFolder);
const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER);

await processServiceFiles();
processExcludedFiles();
// Create build context that stores the information about the current build
const context = await makeRevisionContext(
userInputFolder,
userOutputFolder,
tmpInputFolder,
tmpOutputFolder,
outputBundlePath,
);

const fs = new FsContextCli(context);

// Creating temp .input & .output folder
await preparingTemporaryFolders(context);

// Read and prepare Preset & Toc data
await processServiceFiles(context, fs);

// Removes all content files that unspecified in toc files or ignored.
await processExcludedFiles();

// Write files.json
if (addMapFile) {
prepareMapFile();
await prepareMapFile();
}

const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER);
const navigationPaths = TocService.getNavigationPaths();

// 1. Linting
if (!lintDisabled) {
/* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */
await initLinterWorkers();
await initLinterWorkers(navigationPaths);
}

const processes = [
!lintDisabled && processLinter(),
!buildDisabled && processPages(outputBundlePath),
!lintDisabled && processLinter(context, navigationPaths),
!buildDisabled && processPages(fs, outputBundlePath, context),
].filter(Boolean) as Promise<void>[];

await Promise.all(processes);

// 2. Building
if (!buildDisabled) {
// process additional files
processAssets({
// Process assets
await processAssets({
args,
outputFormat,
outputBundlePath,
tmpOutputFolder,
userOutputFolder,
context,
fs,
});

// Process changelogs
await processChangelogs();

// Finish search service processing
await SearchService.release();

// Copy all generated files to user' output folder
Expand All @@ -246,6 +280,7 @@ async function handler(args: Arguments<any>) {
shell.cp('-r', join(tmpOutputFolder, '.*'), userOutputFolder);
}

// Upload the files to S3
if (publish) {
const DEFAULT_PREFIX = process.env.YFM_STORAGE_PREFIX ?? '';
const {
Expand Down Expand Up @@ -273,31 +308,22 @@ async function handler(args: Arguments<any>) {
} catch (err) {
logger.error('', err.message);
} finally {
// Print logs
processLogs(tmpInputFolder);

shell.rm('-rf', tmpInputFolder, tmpOutputFolder);
}
}

function preparingTemporaryFolders(userOutputFolder: string) {
const args = ArgvService.getConfig();

shell.mkdir('-p', userOutputFolder);
// Creating temp .input & .output folder
async function preparingTemporaryFolders(context: RevisionContext) {
shell.mkdir('-p', context.userOutputFolder);

// Create temporary input/output folders
shell.rm('-rf', args.input, args.output);
shell.mkdir(args.input, args.output);
shell.rm('-rf', context.tmpInputFolder, context.tmpOutputFolder);
shell.mkdir(context.tmpInputFolder, context.tmpOutputFolder);

copyFiles(
args.rootInput,
args.input,
glob.sync('**', {
cwd: args.rootInput,
nodir: true,
follow: true,
ignore: ['node_modules/**', '*/node_modules/**'],
}),
);
await copyFiles(context.userInputFolder, context.tmpInputFolder, context.files);

shell.chmod('-R', 'u+w', args.input);
shell.chmod('-R', 'u+w', context.tmpInputFolder);
}
34 changes: 34 additions & 0 deletions src/context/context.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import {RevisionContext as RevisionContextTransfrom} from '@diplodoc/transform/lib/typings';
import glob from 'glob';

export interface RevisionContext extends RevisionContextTransfrom {
userInputFolder: string;
userOutputFolder: string;
tmpInputFolder: string;
tmpOutputFolder: string;
outputBundlePath: string;
}

export async function makeRevisionContext(
userInputFolder: string,
userOutputFolder: string,
tmpInputFolder: string,
tmpOutputFolder: string,
outputBundlePath: string,
): Promise<RevisionContext> {
const files = glob.sync('**', {
cwd: userInputFolder,
nodir: true,
follow: true,
ignore: ['node_modules/**', '*/node_modules/**'],
});

return {
userInputFolder,
userOutputFolder,
tmpInputFolder,
tmpOutputFolder,
outputBundlePath,
files,
};
}
107 changes: 107 additions & 0 deletions src/context/fs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
import {readFileSync, statSync, writeFileSync} from 'fs';
import {readFile, stat, writeFile} from 'fs/promises';
import {resolve} from 'path';
import {FsContext} from '@diplodoc/transform/lib/typings';
import {RevisionContext} from './context';

export function isFileExists(file: string) {
try {
const stats = statSync(file);

return stats.isFile();
} catch (e) {
return false;
}
}

export async function isFileExistsAsync(file: string) {
try {
const stats = await stat(file);

return stats.isFile();
} catch (e) {
return false;
}
}

export class FsContextCli implements FsContext {
private context: RevisionContext;

constructor(context: RevisionContext) {
this.context = context;
}

getPaths(path: string) {
const arr = [path];

const isFromTmpInputFolder = path.startsWith(resolve(this.context.tmpInputFolder) + '/');
if (isFromTmpInputFolder) {
const assetPath = path.replace(resolve(this.context.tmpInputFolder) + '/', '');
const originPath = resolve(this.context.userInputFolder, assetPath);

arr.unshift(originPath);
}

return arr;
}

exist(path: string): boolean {
const paths = this.getPaths(path);

for (const path of paths) {
if (isFileExists(path)) {
return true;
}
}

return false;
}

read(path: string): string {
const paths = this.getPaths(path);

for (const path of paths) {
if (isFileExists(path)) {
return readFileSync(path, 'utf8');
}
}

throw Error(`File has not been found at: ${path}`);
}

write(path: string, content: string): void {
writeFileSync(path, content, {
encoding: 'utf8',
});
}

async existAsync(path: string): Promise<boolean> {
const paths = this.getPaths(path);

for (const path of paths) {
if (await isFileExistsAsync(path)) {
return true;
}
}

return false;
}

async readAsync(path: string): Promise<string> {
const paths = this.getPaths(path);

for (const path of paths) {
if (await isFileExistsAsync(path)) {
return await readFile(path, 'utf8');
}
}

throw Error(`File has not been found at: ${path}`);
}

async writeAsync(path: string, content: string): Promise<void> {
await writeFile(path, content, {
encoding: 'utf8',
});
}
}
Loading

0 comments on commit fc7bd30

Please sign in to comment.