DOP-3863: Persistence module inserts github_username to Atlas (#861)

* add github_username to documents collection and metadata

* use github username on updated_documents collection, use docs-builder-bot as fallback

* fix tests

* point toward 3863-meta branch, set off preprd build

* empty-build

* change back test conditions

modules/persistence/README.md

@@ -22,12 +22,14 @@ Compiles the module using `tsc`. By default, compiles to the `./dist` directory.
 
 Runs the contents of the `./dist` directory.
 Requires usage of `-- -path` argument, eg `npm run start -- --path ./build/artifacts.zip`.
+An optional argument to specify the github user and link the build to the user will require usage of `--githubUser <github_username>`.
 Recommended command for running this module in higher than local environments.
 Requires parser output artifacts to be present in specified directory and zip file at `--path` value specified.
 
 ### `npm run dev`
 
 Cleans dist, compiles, and runs with arguments `-path ./build/artifacts.zip`.
+Optionally, add the usage of this argument to the command to link build to a desired github user `-- --githubUser <github_username>`.
 Requires parser output artifacts to be present in specified directory and zip file at `./build/artifacts.zip`.
 
 ## Available Arguments

modules/persistence/index.ts

@@ -17,6 +17,7 @@ import { upsertAssets } from './src/services/assets';
 
 interface ModuleArgs {
   path: string;
+  githubUser: string;
   strict: string;
   [props: string | number | symbol]: unknown;
 }
@@ -29,15 +30,19 @@ const missingPathMessage = 'No path specified in arguments - please specify a bu
 // Load command line args into a parameterized argv
 const argv: ModuleArgs = minimist(process.argv.slice(2));
 
-const app = async (path: string) => {
+const app = async (path: string, githubUser: string) => {
   try {
     if (!path) throw missingPathMessage;
     const zip = new AdmZip(path);
     // atomic buildId for all artifacts read by this module - fundamental assumption
     // that only one build will be used per run of this module.
     const buildId = new mongodb.ObjectId();
-    const metadata = await metadataFromZip(zip);
-    await Promise.all([insertAndUpdatePages(buildId, zip), insertMetadata(buildId, metadata), upsertAssets(zip)]);
+    const metadata = await metadataFromZip(zip, githubUser);
+    await Promise.all([
+      insertAndUpdatePages(buildId, zip, githubUser),
+      insertMetadata(buildId, metadata),
+      upsertAssets(zip),
+    ]);
     await insertMergedMetadataEntries(buildId, metadata);
     // DOP-3447 clean up stale metadata
     await deleteStaleMetadata(metadata);
@@ -52,7 +57,7 @@ const app = async (path: string) => {
 
 try {
   console.log(argv);
-  app(argv['path']);
+  app(argv['path'], argv['githubUser']);
 } catch (error) {
   console.log('caught in terminal handling');
   // only exit with non zero error code if running with strict mode on

modules/persistence/src/services/connector/index.ts

@@ -44,7 +44,11 @@ export const insert = async (docs: any[], collection: string, buildId: ObjectId)
   const insertSession = await db();
   try {
     return insertSession.collection(collection).insertMany(
-      docs.map((d) => ({ ...d, build_id: buildId, created_at: buildId.getTimestamp() })),
+      docs.map((d) => ({
+        ...d,
+        build_id: buildId,
+        created_at: buildId.getTimestamp(),
+      })),
       { ordered: false }
     );
   } catch (error) {

modules/persistence/src/services/metadata/index.ts

@@ -14,17 +14,19 @@ export interface Metadata {
   associated_products?: AssociatedProduct[];
   toctree: ToC;
   toctreeOrder: any[];
+  github_username?: string;
   [key: string]: any;
 }
 // Service responsible for memoization of metadata entries.
 // Any extraneous logic performed on metadata entries as part of upload should be added here
 // or within subfolders of this module
-export const metadataFromZip = async (zip: AdmZip) => {
+export const metadataFromZip = async (zip: AdmZip, githubUser?: string) => {
   const zipEntries = zip.getEntries();
   const metadata = zipEntries
     .filter((entry) => entry.entryName === 'site.bson')
     .map((entry) => deserialize(entry.getData()))[0] as Metadata;
   await verifyMetadata(metadata);
+  metadata.github_username = githubUser || 'docs-builder-bot';
   return metadata;
 };
 

modules/persistence/src/services/pages/index.ts

@@ -43,11 +43,15 @@ const UPDATED_AST_COLL_NAME = 'updated_documents';
 // Service responsible for memoization of page level documents.
 // Any extraneous logic performed on page level documents as part of upload should be added here
 // or within subfolders of this module
-const pagesFromZip = (zip: AdmZip) => {
+const pagesFromZip = (zip: AdmZip, githubUser?: string) => {
   const zipPages = zip.getEntries();
   return zipPages
     .filter((entry) => entry.entryName?.startsWith('documents/'))
-    .map((entry) => deserialize(entry.getData()));
+    .map((entry) => {
+      const document = deserialize(entry.getData());
+      document.github_username = githubUser || 'docs-builder-bot';
+      return document;
+    });
 };
 
 /**
@@ -144,6 +148,7 @@ class UpdatedPagesManager {
                 static_assets: this.findUpdatedAssets(page.static_assets, prevPageData?.static_assets),
                 updated_at: this.updateTime,
                 deleted: false,
+                github_username: page.github_username || 'docs-builder-bot',
               },
               $setOnInsert: {
                 created_at: this.updateTime,
@@ -278,9 +283,9 @@ const updatePages = async (pages: Document[], collection: string) => {
   }
 };
 
-export const insertAndUpdatePages = async (buildId: ObjectId, zip: AdmZip) => {
+export const insertAndUpdatePages = async (buildId: ObjectId, zip: AdmZip, githubUser?: string) => {
   try {
-    const pages = pagesFromZip(zip);
+    const pages = pagesFromZip(zip, githubUser);
     const ops: PromiseLike<any>[] = [insert(pages, COLLECTION_NAME, buildId)];
 
     const featureEnabled = process.env.FEATURE_FLAG_UPDATE_PAGES;

modules/persistence/tests/metadata/metadata.test.ts

@@ -49,7 +49,13 @@ describe('metadata module', () => {
   describe('metadataFromZip', () => {
     it('should get metadata from site.bson', async () => {
       const metaFromZip = await _metadataFromZip(zip);
-      expect(metaFromZip).toEqual(meta);
+      expect(metaFromZip).toEqual({ ...meta, github_username: 'docs-builder-bot' });
+    });
+
+    it('should add github username to metadata', async () => {
+      const githubUser = 'gritty';
+      const metaFromZip = await _metadataFromZip(zip, githubUser);
+      expect(metaFromZip).toEqual({ ...meta, github_username: githubUser });
     });
   });
 

src/job/stagingJobHandler.ts

@@ -55,7 +55,7 @@ export class StagingJobHandler extends JobHandler {
   prepStageSpecificNextGenCommands(): void {
     if (this.currJob.buildCommands) {
       this.currJob.buildCommands[this.currJob.buildCommands.length - 1] = 'make next-gen-parse';
-      this.currJob.buildCommands.push('make next-gen-html');
+      this.currJob.buildCommands.push(`make next-gen-html GH_USER=${this.currJob.payload.repoOwner}`);
       const project = this.currJob.payload.project === 'cloud-docs' ? this.currJob.payload.project : '';
       const branchName = /^[a-zA-Z0-9_\-\./]+$/.test(this.currJob.payload.branchName)
         ? this.currJob.payload.branchName

tests/data/data.ts

@@ -162,7 +162,7 @@ export class TestDataProvider {
     const genericCommands = TestDataProvider.getCommonBuildCommands(job);
     const commands = Array<string>().concat(genericCommands.slice(0, genericCommands.length - 1), [
       'make next-gen-parse',
-      'make next-gen-html',
+      `make next-gen-html GH_USER=${job.payload.repoOwner}`,
     ]);
     const project = job.payload.project === 'cloud-docs' ? job.payload.project : '';
     const branchName = /^[a-zA-Z0-9_\-\./]+$/.test(job.payload.branchName) ? job.payload.branchName : '';