Skip to content

Commit

Permalink
feat: add async transform support
Browse files Browse the repository at this point in the history
  • Loading branch information
makamekm committed Oct 30, 2024
1 parent 07fdda5 commit 2246641
Show file tree
Hide file tree
Showing 19 changed files with 581 additions and 287 deletions.
98 changes: 59 additions & 39 deletions src/cmd/build/index.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
import glob from 'glob';
import {Arguments, Argv} from 'yargs';
import {join, resolve} from 'path';
import shell from 'shelljs';
import glob from 'glob';

import OpenapiIncluder from '@diplodoc/openapi-extension/includer';

import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '../../constants';
import {argvValidator} from '../../validator';
import {ArgvService, Includers, SearchService} from '../../services';
import {BUNDLE_FOLDER, Stage, TMP_INPUT_FOLDER, TMP_OUTPUT_FOLDER} from '~/constants';
import {argvValidator} from '~/validator';
import {Includer, YfmArgv} from '~/models';
import {ArgvService, Includers, SearchService, TocService} from '~/services';
import {
initLinterWorkers,
processAssets,
Expand All @@ -17,10 +18,12 @@ import {
processLogs,
processPages,
processServiceFiles,
} from '../../steps';
import {prepareMapFile} from '../../steps/processMapFile';
import {copyFiles, logger} from '../../utils';
import {upload as publishFilesToS3} from '../../commands/publish/upload';
} from '~/steps';
import {prepareMapFile} from '~/steps/processMapFile';
import {copyFiles, logger} from '~/utils';
import {upload as publishFilesToS3} from '~/commands/publish/upload';
import {RevisionContext, makeRevisionContext} from '~/context/context';
import {FsContextCli} from '~/context/fs';

export const build = {
command: ['build', '$0'],
Expand Down Expand Up @@ -175,7 +178,8 @@ function builder<T>(argv: Argv<T>) {
);
}

async function handler(args: Arguments<any>) {
async function handler(args: Arguments<YfmArgv>) {
const userInputFolder = resolve(args.input);
const userOutputFolder = resolve(args.output);
const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER);
const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER);
Expand All @@ -185,14 +189,15 @@ async function handler(args: Arguments<any>) {
}

try {
// Init singletone services
ArgvService.init({
...args,
rootInput: args.input,
rootInput: userInputFolder,
input: tmpInputFolder,
output: tmpOutputFolder,
});
SearchService.init();
Includers.init([OpenapiIncluder as any]);
Includers.init([OpenapiIncluder as Includer]);

const {
output: outputFolderPath,
Expand All @@ -203,41 +208,64 @@ async function handler(args: Arguments<any>) {
addMapFile,
} = ArgvService.getConfig();

preparingTemporaryFolders(userOutputFolder);
const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER);

// Create build context that stores the information about the current build
const context = await makeRevisionContext(
userInputFolder,
userOutputFolder,
tmpInputFolder,
tmpOutputFolder,
outputBundlePath,
);

const fs = new FsContextCli(context);

// Creating temp .input & .output folder
await preparingTemporaryFolders(context);

// Read and prepare Preset & Toc data
await processServiceFiles(context, fs);

await processServiceFiles();
processExcludedFiles();
// Removes all content files that unspecified in toc files or ignored.
await processExcludedFiles();

// Write files.json
if (addMapFile) {
prepareMapFile();
await prepareMapFile();
}

const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER);
const navigationPaths = TocService.getNavigationPaths();

// 1. Linting
if (!lintDisabled) {
/* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */
await initLinterWorkers();
await initLinterWorkers(navigationPaths);
}

const processes = [
!lintDisabled && processLinter(),
!buildDisabled && processPages(outputBundlePath),
!lintDisabled && processLinter(context, navigationPaths),
!buildDisabled && processPages(fs, outputBundlePath, context),
].filter(Boolean) as Promise<void>[];

await Promise.all(processes);

// 2. Building
if (!buildDisabled) {
// process additional files
processAssets({
// Process assets
await processAssets({
args,
outputFormat,
outputBundlePath,
tmpOutputFolder,
userOutputFolder,
context,
fs,
});

// Process changelogs
await processChangelogs();

// Finish search service processing
await SearchService.release();

// Copy all generated files to user' output folder
Expand All @@ -246,6 +274,7 @@ async function handler(args: Arguments<any>) {
shell.cp('-r', join(tmpOutputFolder, '.*'), userOutputFolder);
}

// Upload the files to S3
if (publish) {
const DEFAULT_PREFIX = process.env.YFM_STORAGE_PREFIX ?? '';
const {
Expand Down Expand Up @@ -273,31 +302,22 @@ async function handler(args: Arguments<any>) {
} catch (err) {
logger.error('', err.message);
} finally {
// Print logs
processLogs(tmpInputFolder);

shell.rm('-rf', tmpInputFolder, tmpOutputFolder);
}
}

function preparingTemporaryFolders(userOutputFolder: string) {
const args = ArgvService.getConfig();

shell.mkdir('-p', userOutputFolder);
// Creating temp .input & .output folder
async function preparingTemporaryFolders(context: RevisionContext) {
shell.mkdir('-p', context.userOutputFolder);

// Create temporary input/output folders
shell.rm('-rf', args.input, args.output);
shell.mkdir(args.input, args.output);
shell.rm('-rf', context.tmpInputFolder, context.tmpOutputFolder);
shell.mkdir(context.tmpInputFolder, context.tmpOutputFolder);

copyFiles(
args.rootInput,
args.input,
glob.sync('**', {
cwd: args.rootInput,
nodir: true,
follow: true,
ignore: ['node_modules/**', '*/node_modules/**'],
}),
);
await copyFiles(context.userInputFolder, context.tmpInputFolder, context.files);

shell.chmod('-R', 'u+w', args.input);
shell.chmod('-R', 'u+w', context.tmpInputFolder);
}
34 changes: 34 additions & 0 deletions src/context/context.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import {RevisionContext as RevisionContextTransfrom} from '@diplodoc/transform/lib/typings';
import glob from 'glob';

export interface RevisionContext extends RevisionContextTransfrom {
userInputFolder: string;
userOutputFolder: string;
tmpInputFolder: string;
tmpOutputFolder: string;
outputBundlePath: string;
}

export async function makeRevisionContext(
userInputFolder: string,
userOutputFolder: string,
tmpInputFolder: string,
tmpOutputFolder: string,
outputBundlePath: string,
): Promise<RevisionContext> {
const files = glob.sync('**', {
cwd: userInputFolder,
nodir: true,
follow: true,
ignore: ['node_modules/**', '*/node_modules/**'],
});

return {
userInputFolder,
userOutputFolder,
tmpInputFolder,
tmpOutputFolder,
outputBundlePath,
files,
};
}
107 changes: 107 additions & 0 deletions src/context/fs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
import {readFileSync, statSync, writeFileSync} from 'fs';
import {readFile, stat, writeFile} from 'fs/promises';
import {resolve} from 'path';
import {FsContext} from '@diplodoc/transform/lib/typings';
import {RevisionContext} from './context';

export function isFileExists(file: string) {
try {
const stats = statSync(file);

return stats.isFile();
} catch (e) {
return false;
}
}

export async function isFileExistsAsync(file: string) {
try {
const stats = await stat(file);

return stats.isFile();
} catch (e) {
return false;
}
}

export class FsContextCli implements FsContext {
private context: RevisionContext;

constructor(context: RevisionContext) {
this.context = context;
}

getPaths(path: string) {
const arr = [path];

const isFromTmpInputFolder = path.startsWith(resolve(this.context.tmpInputFolder) + '/');
if (isFromTmpInputFolder) {
const assetPath = path.replace(resolve(this.context.tmpInputFolder) + '/', '');
const originPath = resolve(this.context.userInputFolder, assetPath);

arr.unshift(originPath);
}

return arr;
}

exist(path: string): boolean {
const paths = this.getPaths(path);

for (const path of paths) {
if (isFileExists(path)) {
return true;
}
}

return false;
}

read(path: string): string {
const paths = this.getPaths(path);

for (const path of paths) {
if (isFileExists(path)) {
return readFileSync(path, 'utf8');
}
}

throw Error(`File has not been found at: ${path}`);
}

write(path: string, content: string): void {
writeFileSync(path, content, {
encoding: 'utf8',
});
}

async existAsync(path: string): Promise<boolean> {
const paths = this.getPaths(path);

for (const path of paths) {
if (await isFileExistsAsync(path)) {
return true;
}
}

return false;
}

async readAsync(path: string): Promise<string> {
const paths = this.getPaths(path);

for (const path of paths) {
if (await isFileExistsAsync(path)) {
return await readFile(path, 'utf8');
}
}

throw Error(`File has not been found at: ${path}`);
}

async writeAsync(path: string, content: string): Promise<void> {
await writeFile(path, content, {
encoding: 'utf8',
});
}
}
16 changes: 14 additions & 2 deletions src/models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import {LintConfig} from '@diplodoc/transform/lib/yfmlint';

import {IncludeMode, Lang, ResourceType, Stage} from './constants';
import {FileContributors, VCSConnector, VCSConnectorConfig} from './vcs-connector/connector-models';
import {RevisionContext} from './context/context';
import {FsContext} from '@diplodoc/transform/lib/typings';

export type VarsPreset = 'internal' | 'external';

Expand All @@ -26,7 +28,10 @@ export type NestedContributorsForPathFunction = (
nestedContributors: Contributors,
) => void;
export type UserByLoginFunction = (login: string) => Promise<Contributor | null>;
export type CollectionOfPluginsFunction = (output: string, options: PluginOptions) => string;
export type CollectionOfPluginsFunction = (
output: string,
options: PluginOptions,
) => Promise<string>;
export type GetModifiedTimeByPathFunction = (filepath: string) => number | undefined;

/**
Expand Down Expand Up @@ -58,6 +63,7 @@ interface YfmConfig {
varsPreset: VarsPreset;
ignore: string[];
outputFormat: string;
plugins: string;
allowHTML: boolean;
vars: Record<string, string>;
applyPresets: boolean;
Expand Down Expand Up @@ -257,16 +263,20 @@ export interface PluginOptions {
changelogs?: ChangelogItem[];
extractChangelogs?: boolean;
included?: boolean;
context: RevisionContext;
fs?: FsContext;
}

export interface Plugin {
collect: (input: string, options: PluginOptions) => string | void;
collect: (input: string, options: PluginOptions) => Promise<string | void>;
}

export interface ResolveMd2MdOptions {
inputPath: string;
outputPath: string;
metadata: MetaDataOptions;
context: RevisionContext;
fs: FsContext;
}

export interface ResolverOptions {
Expand All @@ -278,6 +288,8 @@ export interface ResolverOptions {
outputPath: string;
outputBundlePath: string;
metadata?: MetaDataOptions;
context: RevisionContext;
fs: FsContext;
}

export interface PathData {
Expand Down
Loading

0 comments on commit 2246641

Please sign in to comment.