Merge branch 'main' into DOP-4599

.github/workflows/deploy-prd-ecs.yml

@@ -14,7 +14,7 @@ jobs:
         with:
           node-version: '18.x'
       - name: Install Serverless Framework
-        run: npm install -g serverless
+        run: npm install -g serverless@3
       - name: Serverless AWS authentication
         run: sls config credentials --provider aws --key ${{ secrets.AWS_ACCESS_KEY_ID }} --secret ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Configure AWS credentials

.github/workflows/deploy-stg-ecs.yml

@@ -1,8 +1,8 @@
 on:
   push:
     branches:
-      - "main"
-      - "integration"
+      - 'main'
+      - 'integration'
 concurrency:
   group: environment-stg-${{ github.ref }}
   cancel-in-progress: true

Dockerfile

@@ -22,9 +22,9 @@ RUN cd ./modules/oas-page-builder \
 # where repo work will happen
 FROM ubuntu:20.04
 ARG WORK_DIRECTORY=/home/docsworker-xlarge
-ARG SNOOTY_PARSER_VERSION=0.16.7
-ARG SNOOTY_FRONTEND_VERSION=0.16.14
-ARG MUT_VERSION=0.11.2
+ARG SNOOTY_PARSER_VERSION=0.17.1
+ARG SNOOTY_FRONTEND_VERSION=0.17.5
+ARG MUT_VERSION=0.11.4
 ARG REDOC_CLI_VERSION=1.2.3
 ARG NPM_BASE_64_AUTH
 ARG NPM_EMAIL

Dockerfile.local

@@ -1,9 +1,9 @@
 FROM arm64v8/ubuntu:20.04 as initial
 ARG NPM_BASE_64_AUTH
 ARG NPM_EMAIL
-ARG SNOOTY_PARSER_VERSION=0.16.7
-ARG SNOOTY_FRONTEND_VERSION=0.16.14
-ARG MUT_VERSION=0.11.2
+ARG SNOOTY_PARSER_VERSION=0.17.1
+ARG SNOOTY_FRONTEND_VERSION=0.17.5
+ARG MUT_VERSION=0.11.4
 ARG REDOC_CLI_VERSION=1.2.3
 ARG NPM_BASE_64_AUTH
 ARG NPM_EMAIL

README.md

@@ -169,7 +169,7 @@ docs-worker-pool contains various triggers for release to higher environments. C
 #### Installation
 
 ```shell
-npm install -g serverless
+npm install -g serverless@3
 ```
 
 #### AWS Config

api/controllers/v1/github.ts

@@ -7,6 +7,7 @@ import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handle
 import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
 import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/services/metadata/repos_branches';
 import { PushEvent } from '@octokit/webhooks-types';
+import { APIGatewayProxyResult } from 'aws-lambda';
 
 async function prepGithubPushPayload(
   githubEvent: any,
@@ -61,75 +62,11 @@ async function prepGithubPushPayload(
   };
 }
 
-export const TriggerBuild = async (event: any = {}, context: any = {}): Promise<any> => {
-  const client = new mongodb.MongoClient(c.get('dbUrl'));
-  await client.connect();
-  const db = client.db(c.get('dbName'));
-  const consoleLogger = new ConsoleLogger();
-  const jobRepository = new JobRepository(db, c, consoleLogger);
-  const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger);
-  const docsetsRepository = new DocsetsRepository(db, c, consoleLogger);
-
-  if (!validateJsonWebhook(event, c.get<string>('githubSecret'))) {
-    const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match";
-    return {
-      statusCode: 401,
-      headers: { 'Content-Type': 'text/plain' },
-      body: errMsg,
-    };
-  }
-  if (!event.body) {
-    const err = 'Trigger build does not have a body in event payload';
-    consoleLogger.error('TriggerBuildError', err);
-    return {
-      statusCode: 400,
-      headers: { 'Content-Type': 'text/plain' },
-      body: err,
-    };
-  }
-
-  let body: PushEvent;
-  try {
-    body = JSON.parse(event.body) as PushEvent;
-  } catch (e) {
-    consoleLogger.error('[TriggerBuild]', `ERROR! Could not parse event.body ${e}`);
-    console.log(`event: ${event} and event body: ${event.body}`);
-    return {
-      statusCode: 502,
-      headers: { 'Content-Type': 'text/plain' },
-      body: ' ERROR! Could not parse event.body',
-    };
-  }
-
-  if (body.deleted) {
-    return {
-      statusCode: 200,
-      headers: { 'Content-Type': 'text/plain' },
-      body: 'Job Ignored (Deletion)',
-    };
-  }
-
-  const env = c.get<string>('env');
-  const repoInfo = await docsetsRepository.getRepo(body.repository.name);
-  const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
-  // TODO: Make job be of type Job
-  const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo);
-  try {
-    consoleLogger.info(job.title, 'Creating Job');
-    const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));
-    consoleLogger.info(job.title, `Created Job ${jobId}`);
-  } catch (err) {
-    return {
-      statusCode: 500,
-      headers: { 'Content-Type': 'text/plain' },
-      body: err,
-    };
-  }
+export const TriggerBuild = async (): Promise<APIGatewayProxyResult> => {
   return {
-    statusCode: 202,
+    statusCode: 404,
     headers: { 'Content-Type': 'text/plain' },
-    body: 'Job Queued',
+    body: 'The Autobuilder is currently disabled for staging. Please use Netlify instead.',
   };
 };
-
 export const MarkBuildArtifactsForDeletion = markBuildArtifactsForDeletion;

api/controllers/v2/github.ts

@@ -10,10 +10,7 @@ import { ProjectsRepository } from '../../../src/repositories/projectsRepository
 import { EnhancedJob, EnhancedPayload, JobStatus } from '../../../src/entities/job';
 import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github';
 import { DocsetsRepository } from '../../../src/repositories/docsetsRepository';
-import { getMonorepoPaths } from '../../../src/monorepo';
-import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils';
 import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/services/metadata/repos_branches';
-import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants';
 
 const SMOKETEST_SITES = [
   'docs-landing',
@@ -241,129 +238,11 @@ export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Pr
   }
 };
 
-export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayProxyResult> => {
-  const client = new mongodb.MongoClient(c.get('dbUrl'));
-  await client.connect();
-  const db = client.db(c.get('dbName'));
-  const consoleLogger = new ConsoleLogger();
-  const jobRepository = new JobRepository(db, c, consoleLogger);
-  const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger);
-  const docsetsRepository = new DocsetsRepository(db, c, consoleLogger);
-
-  if (!event.body) {
-    const err = 'Trigger build does not have a body in event payload';
-    return {
-      statusCode: 400,
-      headers: { 'Content-Type': 'text/plain' },
-      body: err,
-    };
-  }
-
-  if (!validateJsonWebhook(event, c.get<string>('githubSecret'))) {
-    const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match";
-    return {
-      statusCode: 401,
-      headers: { 'Content-Type': 'text/plain' },
-      body: errMsg,
-    };
-  }
-  let body: PushEvent;
-  try {
-    body = JSON.parse(event.body) as PushEvent;
-  } catch (e) {
-    console.log('[TriggerBuild]: ERROR! Could not parse event.body', e);
-    return {
-      statusCode: 502,
-      headers: { 'Content-Type': 'text/plain' },
-      body: ' ERROR! Could not parse event.body',
-    };
-  }
-
-  if (body.deleted) {
-    return {
-      statusCode: 200,
-      headers: { 'Content-Type': 'text/plain' },
-      body: 'Job Ignored (Deletion)',
-    };
-  }
-
-  const env = c.get<string>('env');
-
-  async function createAndInsertJob(path?: string) {
-    const repo = body.repository;
-    const repoInfo = await docsetsRepository.getRepo(repo.name, path);
-    const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : '';
-    const jobTitle = repo.full_name;
-    const payload = await createPayload({
-      repoName: repo.name,
-      prefix: jobPrefix,
-      repoBranchesRepository,
-      repoInfo,
-      githubEvent: body,
-    });
-
-    const job = await prepGithubPushPayload(body, payload, jobTitle);
-
-    consoleLogger.info(job.title, 'Creating Job');
-    const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl'));
-    jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0);
-    consoleLogger.info(job.title, `Created Job ${jobId}`);
-  }
-
-  if (process.env.FEATURE_FLAG_MONOREPO_PATH === 'true' && body.repository.name === MONOREPO_NAME) {
-    let monorepoPaths: string[] = [];
-    try {
-      if (body.head_commit && body.repository.owner.name) {
-        monorepoPaths = await getMonorepoPaths({
-          commitSha: body.head_commit.id,
-          repoName: body.repository.name,
-          ownerName: body.repository.owner.name,
-          updatedFilePaths: getUpdatedFilePaths(body.head_commit),
-        });
-        consoleLogger.info(body.repository.full_name, `Monorepo Paths with new changes: ${monorepoPaths}`);
-      }
-    } catch (error) {
-      consoleLogger.warn('Warning, attempting to get monorepo paths caused an error', error);
-    }
-
-    /* Create and insert Job for each monorepo project that has changes */
-    for (const path of monorepoPaths) {
-      consoleLogger.info(body.repository.full_name, `Create Job for Monorepo directory: /${path}`);
-      // TODO: Deal with nested monorepo projects
-      /* For now, we will ignore nested monorepo projects until necessary */
-      if (path.split('/').length > 1) continue;
-
-      try {
-        await createAndInsertJob(path);
-      } catch (err) {
-        return {
-          statusCode: 500,
-          headers: { 'Content-Type': 'text/plain' },
-          body: err,
-        };
-      }
-    }
-
-    return {
-      statusCode: 202,
-      headers: { 'Content-Type': 'text/plain' },
-      body: 'Jobs Queued',
-    };
-  }
-
-  try {
-    await createAndInsertJob();
-  } catch (err) {
-    return {
-      statusCode: 500,
-      headers: { 'Content-Type': 'text/plain' },
-      body: err,
-    };
-  }
+export const TriggerBuild = async (): Promise<APIGatewayProxyResult> => {
   return {
-    statusCode: 202,
+    statusCode: 404,
     headers: { 'Content-Type': 'text/plain' },
-    body: 'Job Queued',
+    body: 'The Autobuilder is currently disabled for staging. Please use Netlify instead.',
   };
 };
 

api/controllers/v2/jobs.ts

@@ -110,11 +110,15 @@ function prepProgressMessage(
   status: string,
   errorReason: string
 ): string {
-  const msg = `Your Job (<${jobUrl}${jobId}|${jobTitle}>) `;
   const env = c.get<string>('env');
+  const deprecationWarning =
+    env === 'prd'
+      ? 'The Autobuilder for staging will be deprecated by Sept. 30th. Please reach out to #ask-docs-platform on questions on how to migrate to Netlify for staging builds.'
+      : '';
+  const msg = `Your Job (<${jobUrl}${jobId}|${jobTitle}>) `;
   switch (status) {
     case 'inQueue':
-      return msg + 'has successfully been added to the ' + env + ' queue.';
+      return msg + `has successfully been added to the ${env} queue. ${deprecationWarning}`;
     case 'inProgress':
       return msg + 'is now being processed.';
     case 'completed':

cdk-infra/lib/constructs/worker/worker-env-construct.ts

@@ -64,6 +64,13 @@ export class WorkerEnvConstruct extends Construct {
     const docsetsCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/docsets`);
     const jobCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/job/queue`);
     const gatsbyMarianUrl = StringParameter.valueFromLookup(this, `${ssmPrefix}/frontend/marian_url`);
+    const gatsbyEnableDarkMode = StringParameter.valueFromLookup(this, `${ssmPrefix}/frontend/enable_dark_mode`);
+    // Feature flag to render locales in the process of being translated. Might want to keep this while we continue to translate
+    // to new languages to avoid needing to keep moving this flag around
+    const gatsbyFeatureShowHiddenLocales = StringParameter.valueFromLookup(
+      this,
+      `${ssmPrefix}/flag/show_hidden_locales`
+    );
 
     const dbPassword = secureStrings['MONGO_ATLAS_PASSWORD'];
     this.environment = {
@@ -101,6 +108,8 @@ export class WorkerEnvConstruct extends Construct {
       GATSBY_HIDE_UNIFIED_FOOTER_LOCALE: gatsbyHideUnifiedFooterLocale,
       GATSBY_MARIAN_URL: gatsbyMarianUrl,
       IS_FEATURE_BRANCH: getIsFeatureBranch(),
+      GATSBY_ENABLE_DARK_MODE: gatsbyEnableDarkMode,
+      GATSBY_FEATURE_SHOW_HIDDEN_LOCALES: gatsbyFeatureShowHiddenLocales,
     };
   }
 }

config/custom-environment-variables.json

@@ -24,6 +24,7 @@
   "featureFlagUpdatePages": "FEATURE_FLAG_UPDATE_PAGES",
   "featureFlagSearchUI": "GATSBY_TEST_SEARCH_UI",
   "gatsbyHideUnifiedFooterLocale": "GATSBY_HIDE_UNIFIED_FOOTER_LOCALE",
+  "gatsbyFeatureShowHiddenLocales": "GATSBY_FEATURE_SHOW_HIDDEN_LOCALES",
   "gatsbyMarianURL": "GATSBY_MARIAN_URL",
   "repoBranchesCollection": "REPO_BRANCHES_COL_NAME",
   "docsetsCollection": "DOCSETS_COL_NAME",
@@ -34,5 +35,6 @@
   "cdnClientID": "CDN_CLIENT_ID",
   "cdnClientSecret": "CDN_CLIENT_SECRET",
   "cdnInvalidatorServiceURL": "CDN_INVALIDATOR_SERVICE_URL",
-  "searchIndexBucket": "SEARCH_INDEX_BUCKET"
+  "searchIndexBucket": "SEARCH_INDEX_BUCKET",
+  "gatsbyEnableDarkMode": "GATSBY_ENABLE_DARK_MODE"
 }