Fix_Validation_issues (#523)

* Fix_Validation_issues

* Update jobManager.ts

* typo

* Update jobHandler.ts

* Update serverless.yml

.github/workflows/deploy-prd-ecs.yml

@@ -50,4 +50,4 @@ jobs:
       - name: Deploy Lambdas	
         run: |
           npm ci
-          sls deploy --stage dev
+          sls deploy --stage prd

.github/workflows/deploy-stg-ecs.yml

@@ -53,3 +53,7 @@ jobs:
           sls deploy --stage stg
           aws ecs update-service --force-new-deployment --service docs-worker-pool-stg --cluster docs-worker-pool-stg
 
+      - name: Deploy Lambdas
+        run: |
+          npm ci
+          sls deploy --stage stg

serverless.yml

@@ -1,4 +1,4 @@
-service: docs-worker-pool
+service: docs-worker-pool-api
 variablesResolutionMode: 20210326
 
 plugins:

src/job/jobHandler.ts

@@ -7,6 +7,7 @@ import { IRepoConnector } from "../services/repo";
 import { IFileSystemServices } from "../services/fileServices";
 import { AutoBuilderError, InvalidJobError, JobStoppedError, PublishError } from "../errors/errors";
 import { IConfig } from "config";
+import { IJobValidator } from "./jobValidator";
 var fs = require('fs');
 
 export abstract class JobHandler {
@@ -19,7 +20,7 @@ export abstract class JobHandler {
     protected get commandExecutor(): IJobCommandExecutor {
         return this._commandExecutor;
     }
-   
+
     protected _cdnConnector: ICDNConnector;
 
     private _repoConnector: IRepoConnector;
@@ -34,9 +35,9 @@ export abstract class JobHandler {
         return this._jobRepository;
     }
     private _fileSystemServices: IFileSystemServices;
-  
+
     private _shouldStop: boolean;
-  
+
     private _stopped: boolean;
     public get stopped(): boolean {
         return this._stopped;
@@ -45,12 +46,14 @@ export abstract class JobHandler {
         this._stopped = value;
     }
 
+    private _validator: IJobValidator;
+
     protected _config: IConfig;
 
-    protected name:string;
+    protected name: string;
 
     constructor(job: IJob, config: IConfig, jobRepository: JobRepository, fileSystemServices: IFileSystemServices, commandExecutor: IJobCommandExecutor,
-        cdnConnector: ICDNConnector, repoConnector: IRepoConnector, logger: IJobRepoLogger) {
+        cdnConnector: ICDNConnector, repoConnector: IRepoConnector, logger: IJobRepoLogger, validator: IJobValidator) {
         this._commandExecutor = commandExecutor;
         this._cdnConnector = cdnConnector;
         this._repoConnector = repoConnector;
@@ -60,11 +63,12 @@ export abstract class JobHandler {
         this._fileSystemServices = fileSystemServices;
         this._shouldStop = false;
         this._config = config;
+        this._validator = validator;
     }
 
     abstract prepStageSpecificNextGenCommands(): void;
 
-    private async update(publishResult:CommandExecutorResponse): Promise<void> {
+    private async update(publishResult: CommandExecutorResponse): Promise<void> {
         if (publishResult && publishResult.status === 'success') {
             let files = this._fileSystemServices.getFilesInDirectory(`./${this.currJob.payload.repoName}/build/public`, '', null, null);
             await this.jobRepository.updateWithCompletionStatus(this.currJob._id, files);
@@ -92,7 +96,7 @@ export abstract class JobHandler {
     @throwIfJobInterupted()
     private async logError(error): Promise<void> {
         await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}failed with code: ${error.code}. `);
-        await  this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}stdErr: ${error.stderr}`);
+        await this._logger.save(this.currJob._id, `${'(BUILD)'.padEnd(15)}stdErr: ${error.stderr}`);
     }
 
     @throwIfJobInterupted()
@@ -164,13 +168,17 @@ export abstract class JobHandler {
     @throwIfJobInterupted()
     private async prepNextGenBuild(): Promise<void> {
         if (this.isbuildNextGen()) {
+            await this._validator.throwIfBranchNotConfigured(this.currJob);
             await this.constructPrefix();
             if (!this.currJob.payload.aliased || (this.currJob.payload.aliased && this.currJob.payload.primaryAlias)) {
                 await this.constructManifestIndexPath();
             }
             this.prepStageSpecificNextGenCommands();
             this.constructEnvVars();
             this.currJob.payload.isNextGen = true;
+            if (this._currJob.payload.jobType === 'productionDeploy') {
+                this._validator.throwIfItIsNotPublishable(this._currJob);
+            }
         } else {
             this.currJob.payload.isNextGen = false;
         }
@@ -187,35 +195,35 @@ export abstract class JobHandler {
             if (resp.status != 'success') {
                 const error = new AutoBuilderError(resp.error, "BuildError")
                 await this.logError(error);
-                throw error 
+                throw error
             }
         } else {
             const error = new AutoBuilderError("No commands to execute", "BuildError")
             await this.logError(error);
-            throw error 
+            throw error
         }
         return true;
     }
 
     private constructEnvVars(): void {
-        let envVars = `GATSBY_PARSER_USER=${ this._config.get<string>("GATSBY_PARSER_USER")}\nGATSBY_PARSER_BRANCH=${this.currJob.payload.branchName}\n`;
+        let envVars = `GATSBY_PARSER_USER=${this._config.get<string>("GATSBY_PARSER_USER")}\nGATSBY_PARSER_BRANCH=${this.currJob.payload.branchName}\n`;
         const pathPrefix = this.currJob.payload.pathPrefix;
-        if(typeof pathPrefix !== 'undefined' && pathPrefix !== null){
-          envVars += `PATH_PREFIX=${pathPrefix}\n`
+        if (typeof pathPrefix !== 'undefined' && pathPrefix !== null) {
+            envVars += `PATH_PREFIX=${pathPrefix}\n`
         }
         // const snootyFrontEndVars = {
         //   'GATSBY_FEATURE_FLAG_CONSISTENT_NAVIGATION': this._config.get<boolean>("gatsbyConsitentNavFlag"),
         //   'GATSBY_FEATURE_FLAG_SDK_VERSION_DROPDOWN': this._config.get<boolean>("gatsbySDKVersionDropdownFlag"),
 
         // };
-    
+
         // for (const[envName, envValue] of Object.entries(snootyFrontEndVars)) {
         //   if (envValue) envVars += `${envName}=TRUE\n`;
         // }
-        this._fileSystemServices.writeToFile(`repos/${this.currJob.payload.repoName}/.env.production`, envVars,  { encoding: 'utf8', flag: 'w' });
+        this._fileSystemServices.writeToFile(`repos/${this.currJob.payload.repoName}/.env.production`, envVars, { encoding: 'utf8', flag: 'w' });
     }
 
-    protected getPathPrefix(): Promise<string|undefined> {
+    protected getPathPrefix(): Promise<string | undefined> {
         return Promise.resolve(undefined);
     }
 
@@ -230,45 +238,46 @@ export abstract class JobHandler {
     protected prepBuildCommands(): void {
         this.currJob.buildCommands = [
             `. /venv/bin/activate`,
-            `cd repos/${ this.currJob.payload.repoName}`,
+            `cd repos/${this.currJob.payload.repoName}`,
             `rm -f makefile`,
             `make html`
         ];
     }
 
     @throwIfJobInterupted()
     protected async build(): Promise<boolean> {
-            this.cleanup();
-            await this.cloneRepo(this._config.get<string>("repo_dir"));
-            this._logger.info(this._currJob._id,"Cloned Repo");
-            await this.commitCheck();
-            this._logger.info(this._currJob._id,"Checked Commit");
-            await this.pullRepo();
-            this._logger.info(this._currJob._id,"Pulled Repo");
-            await this._repoConnector.applyPatch(this.currJob);
-            this._logger.info(this._currJob._id,"Patch Applied");
-            await this.downloadMakeFile();
-            this._logger.info(this._currJob._id,"Downloaded Makefile");
-            this.prepBuildCommands();
-            this._logger.info(this._currJob._id,"Prepared Build commands");
-            await this.prepNextGenBuild();
-            this._logger.info(this._currJob._id,"Prepared Next Gen build");
-            return await this.executeBuild();
+        this.cleanup();
+        await this.cloneRepo(this._config.get<string>("repo_dir"));
+        this._logger.info(this._currJob._id, "Cloned Repo");
+        await this.commitCheck();
+        this._logger.info(this._currJob._id, "Checked Commit");
+        await this.pullRepo();
+        this.prepBuildCommands();
+        this._logger.info(this._currJob._id, "Prepared Build commands");
+        await this.prepNextGenBuild();
+        this._logger.info(this._currJob._id, "Pulled Repo");
+        await this._repoConnector.applyPatch(this.currJob);
+        this._logger.info(this._currJob._id, "Patch Applied");
+        await this.downloadMakeFile();
+        this._logger.info(this._currJob._id, "Downloaded Makefile");
+
+        this._logger.info(this._currJob._id, "Prepared Next Gen build");
+        return await this.executeBuild();
     }
 
     @throwIfJobInterupted()
     protected async deployGeneric(): Promise<CommandExecutorResponse> {
         this.prepDeployCommands();
         await this._logger.save(this.currJob._id, `${this._config.get<string>('stage').padEnd(15)}Pushing to ${this.name}`);
-        
-        if (this.currJob.deployCommands && this.currJob.deployCommands.length > 0 ) {
+
+        if (this.currJob.deployCommands && this.currJob.deployCommands.length > 0) {
             const resp = await this._commandExecutor.execute(this.currJob.deployCommands)
-            if (resp && resp.error && typeof(resp.error) === 'string' && resp.error.indexOf('ERROR') !== -1) {
+            if (resp && resp.error && typeof (resp.error) === 'string' && resp.error.indexOf('ERROR') !== -1) {
                 await this._logger.save(this.currJob._id, `${this._config.get<string>('stage').padEnd(15)}Failed to push to ${this.name}`);
                 throw new PublishError(`Failed pushing to ${this.name}: ${resp.error}`)
-              }
-            await this._logger.save(this.currJob._id,`${this._config.get<string>('stage').padEnd(15)}Finished pushing to ${this.name}`);
-            await this._logger.save(this.currJob._id,`${this._config.get<string>('stage').padEnd(15)}Staging push details:\n\n${resp.output}`);
+            }
+            await this._logger.save(this.currJob._id, `${this._config.get<string>('stage').padEnd(15)}Finished pushing to ${this.name}`);
+            await this._logger.save(this.currJob._id, `${this._config.get<string>('stage').padEnd(15)}Staging push details:\n\n${resp.output}`);
             return resp;
 
         } else {

src/job/jobManager.ts

@@ -15,15 +15,16 @@ import { IFileSystemServices } from "../services/fileServices";
 import { IConfig } from "config";
 
 export class JobHandlerFactory {
-    public createJobHandler(job: IJob,  config: IConfig, jobRepository:JobRepository, fileSystemServices:IFileSystemServices, commandExecutor: IJobCommandExecutor, cdnConnector:ICDNConnector, repoConnector:IRepoConnector, logger: IJobRepoLogger) : JobHandler {
+    public createJobHandler(job: IJob,  config: IConfig, jobRepository:JobRepository, fileSystemServices:IFileSystemServices, 
+        commandExecutor: IJobCommandExecutor, cdnConnector:ICDNConnector, repoConnector:IRepoConnector, logger: IJobRepoLogger, validator: IJobValidator) : JobHandler {
         if (job.payload.jobType === "regression") {
-            return new RegressionJobHandler(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger);
+            return new RegressionJobHandler(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger,validator);
         } else if (job.payload.jobType === "githubPush") {
-            return new StagingJobHandler(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger);
+            return new StagingJobHandler(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger,validator);
         } else if (job.payload.jobType === "productionDeploy") {
-            return new ProductionJobHandler(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger);
+            return new ProductionJobHandler(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger,validator);
         } else if (job.payload.jobType === "publishDochub") {
-            return new ProductionJobHandler(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger);
+            return new ProductionJobHandler(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger,validator);
         }
         throw new InvalidJobError("Job type not supported");
     }
@@ -99,13 +100,8 @@ export class JobManager {
 
     async createHandlerAndExecute(job: IJob): Promise<void> {
         this._jobHandler = this._jobHandlerFactory.createJobHandler(job, this._config, this._jobRepository,
-            this._fileSystemServices, this._jobCommandExecutor, this._cdnConnector, this._repoConnector, this._logger);
+            this._fileSystemServices, this._jobCommandExecutor, this._cdnConnector, this._repoConnector, this._logger, this._jobValidator);
 
-        if ( this._jobHandler.isbuildNextGen()) {
-            job.payload.isNextGen = true
-        } else {
-            job.payload.isNextGen = false
-        }
         await this._jobValidator.throwIfJobInvalid(job);
         await this._jobHandler?.execute();
         await this._logger.save(job._id, `${'    (DONE)'.padEnd(this._config.get("LOG_PADDING"))}Finished Job with ID: ${job._id}`);

src/job/jobValidator.ts

@@ -51,13 +51,6 @@ export class JobValidator implements IJobValidator {
         if (this.isProd(job.payload.jobType)) {
             await this.throwIfUserNotEntitled(job);
         }
-        if (job.payload.isNextGen) {
-            await this.throwIfBranchNotConfigured(job);
-        }
-        if (this.isProd(job.payload.jobType)) {
-            this.throwIfItIsNotPublishable(job);
-        }
-
     }
 
     private isProd(jobType: string): boolean {

src/job/productionJobHandler.ts

@@ -9,12 +9,13 @@ import { IFileSystemServices } from "../services/fileServices";
 import { IJobRepoLogger } from "../services/logger";
 import { IRepoConnector } from "../services/repo";
 import { JobHandler } from "./jobHandler";
+import { IJobValidator } from "./jobValidator";
 
 export class ProductionJobHandler extends JobHandler {
 
     constructor(job: IJob, config: IConfig, jobRepository: JobRepository, fileSystemServices: IFileSystemServices, commandExecutor: IJobCommandExecutor,
-        cdnConnector: ICDNConnector, repoConnector: IRepoConnector, logger: IJobRepoLogger) {
-        super(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger);
+        cdnConnector: ICDNConnector, repoConnector: IRepoConnector, logger: IJobRepoLogger, validator:IJobValidator) {
+        super(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger, validator);
         this.name = "Production";
     }
     prepDeployCommands(): void {

src/job/regressionJobHandler.ts

@@ -6,13 +6,14 @@ import { IJobCommandExecutor } from "../services/commandExecutor";
 import { IFileSystemServices } from "../services/fileServices";
 import { IJobRepoLogger } from "../services/logger";
 import { IRepoConnector } from "../services/repo";
+import { IJobValidator } from "./jobValidator";
 import { ProductionJobHandler } from "./productionJobHandler";
 
 export class RegressionJobHandler extends ProductionJobHandler {
 
     constructor(job: IJob, config: IConfig, jobRepository: JobRepository, fileSystemServices: IFileSystemServices, commandExecutor: IJobCommandExecutor,
-        cdnConnector: ICDNConnector, repoConnector: IRepoConnector, logger: IJobRepoLogger) {
-            super(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger);
+        cdnConnector: ICDNConnector, repoConnector: IRepoConnector, logger: IJobRepoLogger, validator:IJobValidator) {
+            super(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger,validator);
             this.name = "Regression";
         }
 }

src/job/stagingJobHandler.ts

@@ -7,11 +7,12 @@ import { CommandExecutorResponse, IJobCommandExecutor } from "../services/comman
 import { IFileSystemServices } from "../services/fileServices";
 import { IJobRepoLogger } from "../services/logger";
 import { IRepoConnector } from "../services/repo";
+import { IJobValidator } from "./jobValidator";
 
 export class StagingJobHandler extends JobHandler {
     constructor(job: IJob, config: IConfig, jobRepository: JobRepository, fileSystemServices: IFileSystemServices, commandExecutor: IJobCommandExecutor,
-        cdnConnector: ICDNConnector, repoConnector: IRepoConnector, logger: IJobRepoLogger) {
-        super(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger);
+        cdnConnector: ICDNConnector, repoConnector: IRepoConnector, logger: IJobRepoLogger, validator:IJobValidator) {
+        super(job, config, jobRepository, fileSystemServices, commandExecutor, cdnConnector, repoConnector, logger, validator);
         this.name = "Staging";
     }
 

tests/unit/job/jobValidator.test.ts

@@ -97,7 +97,6 @@ describe('JobValidator Tests', () => {
         job.payload.isNextGen=true;
         fileSystemServices.downloadYaml.calledWith(`https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/publishedbranches/${job.payload.repoName}.yaml`).mockReturnValue( { status: 'success', content:pubBranchRetVal});
         await jobValidator.throwIfJobInvalid(job);
-        expect(fileSystemServices.downloadYaml).toHaveBeenCalledTimes(1);
         expect(repoEntitlementRepository.getRepoEntitlementsByGithubUsername).toHaveBeenCalledTimes(0);
 
     })

tests/unit/job/stagingJobHandler.test.ts

@@ -2,7 +2,7 @@ import { TestDataProvider } from '../../data/data';
 import { JobHandlerTestHelper } from '../../utils/jobHandlerTestHelper';
 
 
-describe('JobHandlerFactory Tests', () => {
+describe('StagingJobHandler Tests', () => {
 
     let jobHandlerTestHelper: JobHandlerTestHelper;
 

tests/utils/jobHandlerTestHelper.ts

@@ -1,6 +1,7 @@
 import { IConfig } from "config";
 import { mockDeep } from "jest-mock-extended";
 import { IJob } from "../../src/entities/job";
+import { IJobValidator } from "../../src/job/jobValidator";
 import { ProductionJobHandler } from "../../src/job/productionJobHandler";
 import { StagingJobHandler } from "../../src/job/stagingJobHandler";
 import { JobRepository } from "../../src/repositories/jobRepository";
@@ -23,6 +24,7 @@ export class JobHandlerTestHelper {
     repoConnector: IRepoConnector;
     logger: IJobRepoLogger;
     jobHandler: ProductionJobHandler | StagingJobHandler;
+    jobValidator:IJobValidator;
     lengthPrototype;
     handlerMapper = {
         "prod": ProductionJobHandler,
@@ -38,7 +40,8 @@ export class JobHandlerTestHelper {
         this.cdnConnector = mockDeep<ICDNConnector>();
         this.repoConnector = mockDeep<IRepoConnector>();
         this.logger = mockDeep<IJobRepoLogger>();
-        this.jobHandler = new this.handlerMapper[handlerName](this.job, this.config, this.jobRepo, this.fileSystemServices, this.jobCommandExecutor, this.cdnConnector, this.repoConnector, this.logger);
+        this.jobValidator = mockDeep<IJobValidator>();
+        this.jobHandler = new this.handlerMapper[handlerName](this.job, this.config, this.jobRepo, this.fileSystemServices, this.jobCommandExecutor, this.cdnConnector, this.repoConnector, this.logger, this.jobValidator);
         return this.jobHandler;
     }
     setStageForDeploySuccess(isNextGen:boolean = true, prodDeploy:boolean = true): string[] {