[DOP-4033]: Replicate shared.mk steps within Autobuilder code (#917)

* [DOP-4033]: Add initial commands

* [DOP-4033]: Create helpers for cli commands

* [DOP-4033]: Add default args

* [DOP-4033]: Move directories

* [DOP-4033]: Update spawn to work async and add code for getPatchId command

* [DOP-4033]: Refactor executeCliCommand

* [DOP-4033]: Wrap up nextgenparse refactor

* [DOP-4033]: Move commands directory out of enhanced

* [DOP-4033]: Add next gen html command

* [DOP-4033]: Use correct arg for createEnvProdFile

* [DOP-4033]: Add git hash

* [DOP-4033]: Move commands into shared directory

* [DOP-4033]: Refactor

* [DOP-4033]: Create base job for standard builds ie jobs with makefiles that don't override shared,mk

* [DOP-4033]: Add the ability to append to file using executeCliCommand

* [DOP-4033]: Add logic to write to .env.production file

* [DOP-4033]: Add more logic to write to files and start working creating test env

* [DOP-4033]: Add local testing app and make test build faster

* [DOP-4033]: Add test code to confirm commands work, and remove base dockerfile

* [DOP-4033]: Add next-gen-stage, and add support for arm64 dockerfile

* [DOP-4033]: Remove extra args from dockerfile

* [DOP-4033]: Add oas-page-build and persistence module

* [DOP-4033]: Add bundle path for oaspagebuild

* [DOP-4033]: Work on logic for deploy and ability to pipe output from one command to another

* [DOP-4033]: Work on logic for deploy and ability to pipe output from one command to another

* [DOP-4033]: Work on logic for deploy and ability to pipe output from one command to another

* [DOP-4033]: Finish up pipe logic

* [DOP-4033]: Add next-gen-deploy

* [DOP-4033]: Remove unused imports

* [DOP-4033]: Remove standard job handler

* [DOP-4033]: Add execution helper

* [DOP-4033]: Refactor localApp

* [DOP-4033]: Add stage job

* [DOP-4033]: Update persistence module

* [DOP-4033]: Add next-gen-deploy

* [DOP-4033]: Update next-gen-deploy to work

* [DOP-4033]: Add comments

* [DOP-4033]: Add more comments

* [DOP-4033]: Remove index.ts

* [DOP-4033]: Refactor to use index file

* [DOP-4033]: Use correct values for nextgenstage

* [DOP-4033]: Respond to review comments

* [DOP-4033]: Use PATH_PREFIX

* [DOP-4033]: Remove param0

* [DOP-4033]: Respond to review feedback

* [DOP-4033]: Respond to review feedback

* [DOP-4033]: Remove extra log

* [DOP-4033]: Add empty string to join to remove commas

* [DOP-4033]: Use console.log

* [DOP-4033]: Check array length

* [DOP-4033]: Add error handling for dependency helper

Dockerfile.local

@@ -0,0 +1,101 @@
+FROM arm64v8/ubuntu:20.04
+ARG NPM_BASE_64_AUTH
+ARG NPM_EMAIL
+ARG SNOOTY_PARSER_VERSION=0.14.9
+ARG SNOOTY_FRONTEND_VERSION=0.14.18
+ARG MUT_VERSION=0.10.7
+ARG REDOC_CLI_VERSION=1.2.2
+ARG NPM_BASE_64_AUTH
+ARG NPM_EMAIL
+ENV DEBIAN_FRONTEND=noninteractive
+
+# helper libraries for docs builds
+RUN apt-get update && apt-get install -y vim git unzip zip
+
+# get node 18
+# https://gist.github.com/RinatMullayanov/89687a102e696b1d4cab
+RUN apt-get install --yes curl
+RUN curl --location https://deb.nodesource.com/setup_18.x | bash -
+RUN apt-get install --yes nodejs
+RUN apt-get install --yes build-essential
+RUN apt-get install --yes python3-pip libxml2-dev libxslt-dev python-dev pkg-config
+
+WORKDIR /app
+
+RUN python3 -m pip install poetry
+
+# install snooty parser
+RUN git clone -b v${SNOOTY_PARSER_VERSION} --depth 1 https://github.com/mongodb/snooty-parser.git \
+    && cd snooty-parser \
+    && python3 -m poetry install \
+    && make package \
+    && mv dist/snooty /opt/
+
+# install mut
+
+RUN git clone -b v${MUT_VERSION} --depth 1 https://github.com/mongodb/mut.git \
+    && cd mut \
+    && python3 -m poetry install \
+    && make package \
+    && mv dist/mut /opt/
+
+RUN curl  -L -o redoc.zip https://github.com/mongodb-forks/redoc/archive/refs/tags/v${REDOC_CLI_VERSION}.zip \
+    && unzip redoc.zip \
+    && mv redoc-${REDOC_CLI_VERSION} redoc/ 
+
+ENV PATH="${PATH}:/opt/snooty:/opt/mut:/app/.local/bin"
+
+# setup user and root directory
+RUN useradd -ms /bin/bash docsworker
+RUN chmod 755 -R /app
+RUN chown -Rv docsworker /app
+USER docsworker
+
+# install snooty frontend and docs-tools
+RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git       \
+    && cd snooty                                                                                   \
+    && npm ci --legacy-peer-deps --omit=dev                                                        
+
+RUN mkdir -p modules/persistence && chmod 755 modules/persistence
+COPY modules/persistence/package*.json ./modules/persistence/
+RUN cd ./modules/persistence \
+    && npm ci --legacy-peer-deps 
+
+RUN  mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder
+COPY modules/oas-page-builder/package*.json ./modules/oas-page-builder/
+RUN cd ./modules/oas-page-builder \
+    && npm ci --legacy-peer-deps 
+
+# Root project build
+COPY package*.json ./
+RUN npm ci --legacy-peer-deps
+# Build persistence module
+
+COPY --chown=docsworker modules/persistence/tsconfig*.json ./modules/persistence
+COPY --chown=docsworker modules/persistence/src ./modules/persistence/src/
+COPY --chown=docsworker modules/persistence/index.ts ./modules/persistence
+
+RUN cd ./modules/persistence \
+    && npm run build
+
+# Build modules
+# OAS Page Builder
+COPY --chown=docsworker modules/oas-page-builder/tsconfig*.json ./modules/oas-page-builder
+COPY --chown=docsworker modules/oas-page-builder/src ./modules/oas-page-builder/src/
+COPY --chown=docsworker modules/oas-page-builder/index.ts ./modules/oas-page-builder
+
+RUN cd ./modules/oas-page-builder \
+    && npm run build
+
+COPY tsconfig*.json ./
+COPY config config/
+COPY api api/
+COPY src src/
+
+RUN npm run build:esbuild
+
+RUN mkdir repos && chmod 755 repos
+
+EXPOSE 3000
+
+CMD ["node", "--enable-source-maps", "dist/entrypoints/localApp.js"]

package.json

@@ -8,6 +8,7 @@
     "start": "node index.js",
     "clean": "node maintain.js",
     "build": "tsc",
+    "build:esbuild": "esbuild src/entrypoints/localApp.ts --bundle --platform=node --outdir=./dist/entrypoints --allow-overwrite --sourcemap",
     "format": "npm run prettier -- --check",
     "format:fix": "npm run prettier -- --write",
     "lint": "eslint --ext .ts .",

src/commands/index.ts

@@ -0,0 +1,17 @@
+import { prepareBuildAndGetDependencies } from './src/helpers/dependency-helpers';
+import { nextGenDeploy } from './src/shared/next-gen-deploy';
+import { nextGenHtml } from './src/shared/next-gen-html';
+import { nextGenParse } from './src/shared/next-gen-parse';
+import { nextGenStage } from './src/shared/next-gen-stage';
+import { oasPageBuild } from './src/shared/oas-page-build';
+import { persistenceModule } from './src/shared/persistence-module';
+
+export {
+  nextGenParse,
+  nextGenHtml,
+  nextGenStage,
+  persistenceModule,
+  oasPageBuild,
+  nextGenDeploy,
+  prepareBuildAndGetDependencies,
+};

src/commands/src/helpers/dependency-helpers.ts

@@ -0,0 +1,65 @@
+import path from 'path';
+import fs from 'fs';
+import { executeCliCommand, getCommitBranch, getCommitHash, getPatchId, getRepoDir } from '.';
+import { promisify } from 'util';
+
+const existsAsync = promisify(fs.exists);
+const writeFileAsync = promisify(fs.writeFile);
+
+async function cloneRepo(repoName: string) {
+  await executeCliCommand({
+    command: 'git',
+    args: ['clone', `https://github.com/mongodb/${repoName}`],
+    options: { cwd: `${process.cwd()}/repos` },
+  });
+}
+async function createEnvProdFile(repoDir: string, projectName: string, baseUrl: string, prefix = '') {
+  const prodFileName = `${process.cwd()}/snooty/.env.production`;
+
+  try {
+    await writeFileAsync(
+      prodFileName,
+      `GATSBY_SITE=${projectName}
+      GATSBY_MANIFEST_PATH=${repoDir}/bundle.zip
+      GATSBY_PARSER_USER=${process.env.USER}
+      GATSBY_BASE_URL=${baseUrl}
+      PATH_PREFIX=${prefix}`,
+      'utf8'
+    );
+  } catch (e) {
+    console.error(`ERROR! Could not write to .env.production`);
+    throw e;
+  }
+}
+
+export async function prepareBuildAndGetDependencies(repoName: string, projectName: string, baseUrl: string) {
+  // before we get build dependencies, we need to clone the repo
+  await cloneRepo(repoName);
+
+  const repoDir = getRepoDir(repoName);
+
+  // doing these in parallel
+  const commandPromises = [
+    getCommitHash(repoDir),
+    getCommitBranch(repoDir),
+    getPatchId(repoDir),
+    existsAsync(path.join(process.cwd(), 'config/redirects')),
+    createEnvProdFile(repoDir, projectName, baseUrl),
+  ];
+
+  try {
+    const dependencies = await Promise.all(commandPromises);
+
+    return {
+      commitHash: dependencies[0] as string,
+      commitBranch: dependencies[1] as string,
+      patchId: dependencies[2] as string | undefined,
+      hasRedirects: dependencies[3] as boolean,
+      bundlePath: `${repoDir}/bundle.zip`,
+      repoDir,
+    };
+  } catch (error) {
+    console.error('ERROR! Could not get build dependencies');
+    throw error;
+  }
+}