DOP-4020: repos_branches to docsets (#909)

* find todos in main RepoBranchesRepository, consolidate slack helpers * delete unused RepoBranchesRepository method * replace getRepo everywhere with new DocsetsRepository getRepo * add docsetsRepository to each JobHandler and test * add docsetsRepo to JobHandlers * more test finds, missing references * added env vars for docsetCollection * modify tests * set to stage ecs * add docsets to parameter store and serverless * more env vars * persistence module repos_branches migrate to docsets * work on tests in persistence module * dont use preprd * fixed persistence tests * merge two separate docsetsRepos * deploy preprd * use .env * debug failing test * all tests passing * PR feedback * fix test * fix v2 to match v1, double query * add warning to checkSnootyTomlPath
mongodb · Sep 22, 2023 · 01be34b · 01be34b
1 parent c007d0f
commit 01be34b
Show file tree

Hide file tree

Showing 51 changed files with 592 additions and 301 deletions.
diff --git a/api/config/custom-environment-variables.json b/api/config/custom-environment-variables.json
@@ -20,6 +20,7 @@
   "entitlementCollection": "USER_ENTITLEMENT_COL_NAME",
   "dashboardUrl": "DASHBOARD_URL",
   "repoBranchesCollection": "REPO_BRANCHES_COL_NAME",
+  "docsetsCollection": "DOCSETS_COL_NAME",
   "taskDefinitionFamily": "TASK_DEFINITION_FAMILY",
   "jobsQueueUrl": "JOBS_QUEUE_URL",
   "jobUpdatesQueueUrl": "JOB_UPDATES_QUEUE_URL",

diff --git a/api/config/default.json b/api/config/default.json
@@ -19,6 +19,7 @@
   "jobQueueCollection": "JOB_QUEUE_COL_NAME",
   "entitlementCollection": "USER_ENTITLEMENT_COL_NAME",
   "repoBranchesCollection": "REPO_BRANCHES_COL_NAME",
+  "docsetsCollection": "DOCSETS_COL_NAME",
   "MONGO_TIMEOUT_S": 15,
   "JOB_TIMEOUT_S": 900,
   "RETRY_TIMEOUT_MS": 5000,

diff --git a/api/controllers/v1/github.ts b/api/controllers/v1/github.ts
@@ -4,13 +4,19 @@ import { JobRepository } from '../../../src/repositories/jobRepository';
 import { ConsoleLogger } from '../../../src/services/logger';
 import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository';
 import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github';
+import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
+import { ReposBranchesDocument } from '../../../modules/persistence/src/services/metadata/repos_branches';
 
-async function prepGithubPushPayload(githubEvent: any, repoBranchesRepository: RepoBranchesRepository, prefix: string) {
+async function prepGithubPushPayload(
+  githubEvent: any,
+  repoBranchesRepository: RepoBranchesRepository,
+  prefix: string,
+  repoInfo: ReposBranchesDocument
+) {
   const branch_name = githubEvent.ref.split('/')[2];
   const branch_info = await repoBranchesRepository.getRepoBranchAliases(githubEvent.repository.name, branch_name);
   const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name;
-  const repo_info = await repoBranchesRepository.getRepo(githubEvent.repository.name);
-  const project = repo_info?.project ?? githubEvent.repository.name;
+  const project = repoInfo?.project ?? githubEvent.repository.name;
 
   return {
     title: githubEvent.repository.full_name,
@@ -57,6 +63,7 @@ export const TriggerBuild = async (event: any = {}, context: any = {}): Promise<
   const consoleLogger = new ConsoleLogger();
   const jobRepository = new JobRepository(db, c, consoleLogger);
   const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger);
+  const docsetsRepository = new DocsetsRepository(db, c, consoleLogger);
 
   if (!validateJsonWebhook(event, c.get<string>('githubSecret'))) {
     const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match";
@@ -77,10 +84,10 @@ export const TriggerBuild = async (event: any = {}, context: any = {}): Promise<
   }
 
   const env = c.get<string>('env');
-  const repoInfo = await repoBranchesRepository.getRepo(body.repository.name);
+  const repoInfo = await docsetsRepository.getRepo(body.repository.name);
   const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
   // TODO: Make job be of type Job
-  const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix);
+  const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo);
   try {
     consoleLogger.info(job.title, 'Creating Job');
     const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));

diff --git a/api/controllers/v1/jobs.ts b/api/controllers/v1/jobs.ts
@@ -2,7 +2,6 @@ import * as c from 'config';
 import * as mongodb from 'mongodb';
 import { IConfig } from 'config';
 import { RepoEntitlementsRepository } from '../../../src/repositories/repoEntitlementsRepository';
-import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository';
 import { ConsoleLogger } from '../../../src/services/logger';
 import { SlackConnector } from '../../../src/services/slack';
 import { JobRepository } from '../../../src/repositories/jobRepository';
@@ -12,6 +11,7 @@ import { ECSContainer } from '../../../src/services/containerServices';
 import { SQSConnector } from '../../../src/services/queue';
 import { Batch } from '../../../src/services/batch';
 import { notifyBuildSummary, snootyBuildComplete } from '../../handlers/jobs';
+import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
 
 export const TriggerLocalBuild = async (event: any = {}, context: any = {}): Promise<any> => {
   const client = new mongodb.MongoClient(c.get('dbUrl'));
@@ -258,7 +258,7 @@ async function SubmitArchiveJob(jobId: string) {
   const db = client.db(c.get('dbName'));
   const models = {
     jobs: new JobRepository(db, c, consoleLogger),
-    repoBranches: new RepoBranchesRepository(db, c, consoleLogger),
+    repoBranches: new DocsetsRepository(db, c, consoleLogger),
   };
   const job = await models.jobs.getJobById(jobId);
   const repo = await models.repoBranches.getRepo(job.payload.repoName);

diff --git a/api/controllers/v1/slack.ts b/api/controllers/v1/slack.ts
@@ -5,53 +5,14 @@ import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRe
 import { ConsoleLogger, ILogger } from '../../../src/services/logger';
 import { SlackConnector } from '../../../src/services/slack';
 import { JobRepository } from '../../../src/repositories/jobRepository';
-
-function isUserEntitled(entitlementsObject: any): boolean {
-  return (entitlementsObject?.repos?.length ?? 0) > 0;
-}
-
-function isRestrictedToDeploy(userId: string): boolean {
-  const { restrictedProdDeploy, entitledSlackUsers } = c.get<any>('prodDeploy');
-  return restrictedProdDeploy && !entitledSlackUsers.includes(userId);
-}
-
-function prepResponse(statusCode, contentType, body) {
-  return {
-    statusCode: statusCode,
-    headers: { 'Content-Type': contentType },
-    body: body,
-  };
-}
-
-async function buildEntitledBranchList(entitlement: any, repoBranchesRepository: RepoBranchesRepository) {
-  const entitledBranches: string[] = [];
-  for (const repo of entitlement.repos) {
-    const [repoOwner, repoName] = repo.split('/');
-    const branches = await repoBranchesRepository.getRepoBranches(repoName);
-    for (const branch of branches) {
-      let buildWithSnooty = true;
-      if ('buildsWithSnooty' in branch) {
-        buildWithSnooty = branch['buildsWithSnooty'];
-      }
-      if (buildWithSnooty) {
-        entitledBranches.push(`${repoOwner}/${repoName}/${branch['gitBranchName']}`);
-      }
-    }
-  }
-  return entitledBranches.sort();
-}
-
-function getQSString(qs: string) {
-  const key_val = {};
-  const arr = qs.split('&');
-  if (arr) {
-    arr.forEach((keyval) => {
-      const kvpair = keyval.split('=');
-      key_val[kvpair[0]] = kvpair[1];
-    });
-  }
-  return key_val;
-}
+import {
+  buildEntitledBranchList,
+  getQSString,
+  isRestrictedToDeploy,
+  isUserEntitled,
+  prepResponse,
+} from '../../handlers/slack';
+import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
 
 export const DisplayRepoOptions = async (event: any = {}, context: any = {}): Promise<any> => {
   const consoleLogger = new ConsoleLogger();
@@ -102,7 +63,12 @@ const deployHelper = (deployable, payload, jobTitle, jobUserName, jobUserEmail)
 
 // For every repo/branch selected to be deployed, return an array of jobs with the payload data
 // needed for a successful build.
-export const getDeployableJobs = async (values, entitlement, repoBranchesRepository: RepoBranchesRepository) => {
+export const getDeployableJobs = async (
+  values,
+  entitlement,
+  repoBranchesRepository: RepoBranchesRepository,
+  docsetsRepository: DocsetsRepository
+) => {
   const deployable = [];
 
   for (let i = 0; i < values.repo_option.length; i++) {
@@ -113,7 +79,7 @@ export const getDeployableJobs = async (values, entitlement, repoBranchesReposit
     const jobUserName = entitlement.github_username;
     const jobUserEmail = entitlement?.email ?? '';
 
-    const repoInfo = await repoBranchesRepository.getRepo(repoName);
+    const repoInfo = await docsetsRepository.getRepo(repoName);
     const non_versioned = repoInfo.branches.length === 1;
 
     const branchObject = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName);
@@ -196,6 +162,7 @@ export const DeployRepo = async (event: any = {}, context: any = {}): Promise<an
   const db = client.db(c.get('dbName'));
   const repoEntitlementRepository = new RepoEntitlementsRepository(db, c, consoleLogger);
   const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger);
+  const docsetsRepository = new DocsetsRepository(db, c, consoleLogger);
   const jobRepository = new JobRepository(db, c, consoleLogger);
 
   // This is coming in as urlencoded string, need to decode before parsing
@@ -210,7 +177,7 @@ export const DeployRepo = async (event: any = {}, context: any = {}): Promise<an
 
   const values = slackConnector.parseSelection(stateValues);
 
-  const deployable = await getDeployableJobs(values, entitlement, repoBranchesRepository);
+  const deployable = await getDeployableJobs(values, entitlement, repoBranchesRepository, docsetsRepository);
   if (deployable.length > 0) {
     await deployRepo(deployable, consoleLogger, jobRepository, c.get('jobsQueueUrl'));
   }

diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts
@@ -8,19 +8,21 @@ import { ConsoleLogger } from '../../../src/services/logger';
 import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository';
 import { EnhancedJob, JobStatus } from '../../../src/entities/job';
 import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github';
+import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
 import { getMonorepoPaths } from '../../../src/monorepo';
 import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils';
+import { ReposBranchesDocument } from '../../../modules/persistence/src/services/metadata/associated_products';
 
 async function prepGithubPushPayload(
   githubEvent: PushEvent,
   repoBranchesRepository: RepoBranchesRepository,
-  prefix: string
+  prefix: string,
+  repoInfo: ReposBranchesDocument
 ): Promise<Omit<EnhancedJob, '_id'>> {
   const branch_name = githubEvent.ref.split('/')[2];
   const branch_info = await repoBranchesRepository.getRepoBranchAliases(githubEvent.repository.name, branch_name);
   const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name;
-  const repo_info = await repoBranchesRepository.getRepo(githubEvent.repository.name);
-  const project = repo_info?.project ?? githubEvent.repository.name;
+  const project = repoInfo?.project ?? githubEvent.repository.name;
 
   return {
     title: githubEvent.repository.full_name,
@@ -58,6 +60,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
   const consoleLogger = new ConsoleLogger();
   const jobRepository = new JobRepository(db, c, consoleLogger);
   const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger);
+  const docsetsRepository = new DocsetsRepository(db, c, consoleLogger);
 
   if (!event.body) {
     const err = 'Trigger build does not have a body in event payload';
@@ -98,10 +101,10 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
   }
 
   const env = c.get<string>('env');
-  const repoInfo = await repoBranchesRepository.getRepo(body.repository.name);
+  const repoInfo = await docsetsRepository.getRepo(body.repository.name);
   const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
 
-  const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix);
+  const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo);
 
   if (process.env.MONOREPO_PATH_FEATURE === 'true') {
     try {

diff --git a/api/controllers/v2/slack.ts b/api/controllers/v2/slack.ts
@@ -7,53 +7,14 @@ import { SlackConnector } from '../../../src/services/slack';
 import { JobRepository } from '../../../src/repositories/jobRepository';
 import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda';
 import { JobStatus } from '../../../src/entities/job';
-
-function isUserEntitled(entitlementsObject: any): boolean {
-  return (entitlementsObject?.repos?.length ?? 0) > 0;
-}
-
-function isRestrictedToDeploy(userId: string): boolean {
-  const { restrictedProdDeploy, entitledSlackUsers } = c.get<any>('prodDeploy');
-  return restrictedProdDeploy && !entitledSlackUsers.includes(userId);
-}
-
-function prepResponse(statusCode, contentType, body) {
-  return {
-    statusCode: statusCode,
-    headers: { 'Content-Type': contentType },
-    body: body,
-  };
-}
-
-async function buildEntitledBranchList(entitlement: any, repoBranchesRepository: RepoBranchesRepository) {
-  const entitledBranches: string[] = [];
-  for (const repo of entitlement.repos) {
-    const [repoOwner, repoName] = repo.split('/');
-    const branches = await repoBranchesRepository.getRepoBranches(repoName);
-    for (const branch of branches) {
-      let buildWithSnooty = true;
-      if ('buildsWithSnooty' in branch) {
-        buildWithSnooty = branch['buildsWithSnooty'];
-      }
-      if (buildWithSnooty) {
-        entitledBranches.push(`${repoOwner}/${repoName}/${branch['gitBranchName']}`);
-      }
-    }
-  }
-  return entitledBranches.sort();
-}
-
-function getQSString(qs: string) {
-  const key_val = {};
-  const arr = qs.split('&');
-  if (arr) {
-    arr.forEach((keyval) => {
-      const kvpair = keyval.split('=');
-      key_val[kvpair[0]] = kvpair[1];
-    });
-  }
-  return key_val;
-}
+import {
+  buildEntitledBranchList,
+  getQSString,
+  isRestrictedToDeploy,
+  isUserEntitled,
+  prepResponse,
+} from '../../handlers/slack';
+import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
 
 export const DisplayRepoOptions = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult> => {
   const consoleLogger = new ConsoleLogger();
@@ -119,7 +80,12 @@ const deployHelper = (deployable, payload, jobTitle, jobUserName, jobUserEmail)
 
 // For every repo/branch selected to be deployed, return an array of jobs with the payload data
 // needed for a successful build.
-export const getDeployableJobs = async (values, entitlement, repoBranchesRepository: RepoBranchesRepository) => {
+export const getDeployableJobs = async (
+  values,
+  entitlement,
+  repoBranchesRepository: RepoBranchesRepository,
+  docsetsRepository: DocsetsRepository
+) => {
   const deployable = [];
 
   for (let i = 0; i < values.repo_option.length; i++) {
@@ -130,7 +96,7 @@ export const getDeployableJobs = async (values, entitlement, repoBranchesReposit
     const jobUserName = entitlement.github_username;
     const jobUserEmail = entitlement?.email ?? '';
 
-    const repoInfo = await repoBranchesRepository.getRepo(repoName);
+    const repoInfo = await docsetsRepository.getRepo(repoName);
     const non_versioned = repoInfo.branches.length === 1;
 
     const branchObject = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName);
@@ -220,6 +186,7 @@ export const DeployRepo = async (event: APIGatewayEvent): Promise<APIGatewayProx
   const db = client.db(c.get('dbName'));
   const repoEntitlementRepository = new RepoEntitlementsRepository(db, c, consoleLogger);
   const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger);
+  const docsetsRepository = new DocsetsRepository(db, c, consoleLogger);
   const jobRepository = new JobRepository(db, c, consoleLogger);
 
   // This is coming in as urlencoded string, need to decode before parsing
@@ -234,7 +201,7 @@ export const DeployRepo = async (event: APIGatewayEvent): Promise<APIGatewayProx
 
   const values = slackConnector.parseSelection(stateValues);
 
-  const deployable = await getDeployableJobs(values, entitlement, repoBranchesRepository);
+  const deployable = await getDeployableJobs(values, entitlement, repoBranchesRepository, docsetsRepository);
   if (deployable.length > 0) {
     await deployRepo(deployable, consoleLogger, jobRepository, c.get('jobsQueueUrl'));
   }

diff --git a/api/handlers/github.ts b/api/handlers/github.ts
@@ -4,7 +4,7 @@ import * as mongodb from 'mongodb';
 import { APIGatewayEvent } from 'aws-lambda';
 import { PullRequestEvent } from '@octokit/webhooks-types';
 import { ConsoleLogger } from '../../src/services/logger';
-import { RepoBranchesRepository } from '../../src/repositories/repoBranchesRepository';
+import { DocsetsRepository } from '../../src/repositories/docsetsRepository';
 import { UpdatedDocsRepository } from '../../src/repositories/updatedDocsRepository';
 import { MetadataRepository } from '../../src/repositories/metadataRepository';
 
@@ -94,8 +94,8 @@ export const markBuildArtifactsForDeletion = async (event: APIGatewayEvent) => {
   try {
     await client.connect();
     const poolDb = client.db(c.get('dbName'));
-    const repoBranchesRepository = new RepoBranchesRepository(poolDb, c, consoleLogger);
-    const project = (await repoBranchesRepository.getProjectByRepoName(repository.name)) as string;
+    const docsetsRepository = new DocsetsRepository(poolDb, c, consoleLogger);
+    const project = (await docsetsRepository.getProjectByRepoName(repository.name)) as string;
 
     // Start marking build artifacts for deletion
     const snootyDb = client.db(c.get('snootyDbName'));