1564 Add logging for failed subscriber creation

server/.nvmrc

@@ -1 +1 @@
-18.16.0
+18.19.0

server/package.json

@@ -11,7 +11,7 @@
     "build": "rimraf dist && tsc -p tsconfig.build.json",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\" \"libs/**/*.ts\"",
     "start": "ts-node -r tsconfig-paths/register src/main.ts",
-    "start:dev": "tsc-watch -p tsconfig.build.json --onSuccess \"node dist/main.js\"",
+    "start:dev": "nest start --watch",
     "start:dev:skip-auth": "SKIP_AUTH=true yarn run start:dev",
     "start:debug": "tsc-watch -p tsconfig.build.json --onSuccess \"node --inspect-brk dist/main.js\"",
     "start:prod": "node dist/main.js",
@@ -29,13 +29,15 @@
   },
   "dependencies": {
     "@azure/msal-node": "^2.6.5",
-    "@nestjs/common": "^7.6.15",
-    "@nestjs/core": "^7.6.15",
-    "@nestjs/platform-express": "^7.6.15",
-    "@nestjs/testing": "^7.6.15",
-    "@nestjs/typeorm": "^7.1.5",
+    "@nestjs/common": "^8.4.7",
+    "@nestjs/core": "^8.4.7",
+    "@nestjs/platform-express": "^8.4.7",
+    "@nestjs/testing": "^8.4.7",
     "@octokit/rest": "^16.43.1",
     "@sendgrid/client": "^8.1.3",
+    "@sentry/nestjs": "^8.35.0",
+    "@sentry/profiling-node": "^8.35.0",
+    "@sentry/wizard": "^3.34.2",
     "@turf/bbox": "^6.0.1",
     "@turf/buffer": "^5.1.5",
     "@turf/helpers": "^6.1.4",
@@ -71,7 +73,7 @@
     "reflect-metadata": "^0.1.12",
     "request": "^2.88.2",
     "rimraf": "^2.6.2",
-    "rxjs": "^6.3.3",
+    "rxjs": "^7.1.0",
     "shortid": "^2.2.15",
     "sphericalmercator": "^1.0.5",
     "superagent": "^6.1.0",
@@ -84,8 +86,8 @@
     "zlib": "^1.0.5"
   },
   "devDependencies": {
-    "@nestjs/cli": "^7.6.0",
-    "@nestjs/schematics": "^7.3.1",
+    "@nestjs/cli": "^8.2.8",
+    "@nestjs/schematics": "^8.0.11",
     "@types/compression": "^1.7.5",
     "@types/express": "^4.17.1",
     "@types/inflected": "^2.1.3",
@@ -105,7 +107,7 @@
     "tsc-watch": "2.2.1",
     "tsconfig-paths": "3.8.0",
     "tslint": "5.16.0",
-    "typescript": "^4.2.3"
+    "typescript": "^4.9.5"
   },
   "jest": {
     "moduleFileExtensions": [

server/src/app.module.ts

@@ -1,4 +1,7 @@
 import { Module, NestModule, MiddlewareConsumer } from "@nestjs/common";
+import { SentryModule } from "@sentry/nestjs/setup";
+import { APP_FILTER } from "@nestjs/core";
+import { SentryGlobalFilter } from "@sentry/nestjs/setup";
 import bodyParser from "body-parser";
 import cookieparser from "cookie-parser";
 import compression from "compression";
@@ -16,7 +19,14 @@ import { ZoningResolutionsModule } from "./zoning-resolutions/zoning-resolutions
 import { SubscriberModule } from "./subscriber/subscriber.module";
 
 @Module({
+  providers: [
+    {
+      provide: APP_FILTER,
+      useClass: SentryGlobalFilter,
+    },
+  ],
   imports: [
+    SentryModule.forRoot(),
     ProjectModule,
     ContactModule,
     ConfigModule,

server/src/main.ts

@@ -1,3 +1,6 @@
+// Sentry needs to be imported first
+import "./sentry/instrument";
+
 import { NestFactory } from "@nestjs/core";
 import * as fs from "fs";
 import { AppModule } from "./app.module";

server/src/sentry/instrument.ts

@@ -0,0 +1,44 @@
+import * as Sentry from "@sentry/nestjs";
+import { nodeProfilingIntegration } from '@sentry/profiling-node';
+import * as dotenv from "dotenv";
+import * as fs from "fs";
+
+// Load Sentry environment variables from process.env or development.env file
+let envConfig: { [key: string]: string } = {};
+if (
+  process.env.NODE_ENV === "production" ||
+  process.env.NODE_ENV === "test"
+) {
+  envConfig = process.env;
+} else {
+  try {
+    const filePath = "development.env";
+    const envValuesFromFile: { [key: string]: string } = {} = dotenv.parse(fs.readFileSync(filePath)) || {};
+
+    envConfig = { ...process.env, ...envValuesFromFile };
+  } catch (e) {
+    console.log(`Something went wrong loading the environment file: ${e}`);
+
+    // fallback to whatever the environment is
+    envConfig = process.env;
+  }
+}
+
+// Ensure to call this before importing any other modules!
+Sentry.init({
+  dsn: process.env.SENTRY_DSN,
+  environment: process.env.SENTRY_ENVIRONMENT === undefined ? "local" : process.env.SENTRY_ENVIRONMENT,
+  enabled: process.env.SENTRY_ENVIRONMENT === undefined ? false : true,
+  integrations: [
+    // Add our Profiling integration
+    nodeProfilingIntegration(),
+  ],
+
+  // Add Tracing by setting tracesSampleRate
+  // We recommend adjusting this value in production
+  tracesSampleRate: 1.0,
+
+  // Set sampling rate for profiling
+  // This is relative to tracesSampleRate
+  profilesSampleRate: 1.0,
+});

server/src/subscriber/subscriber.controller.ts

@@ -3,6 +3,10 @@ import { ConfigService } from "../config/config.service";
 import { SubscriberService } from "./subscriber.service";
 import { Request } from "express";
 import validateEmail from "../_utils/validate-email";
+import * as Sentry from "@sentry/nestjs";
+
+const PAUSE_BETWEEN_CHECKS = 30000;
+const CHECKS_BEFORE_FAIL = 10;
 
 @Controller()
 export class SubscriberController {
@@ -38,6 +42,7 @@ export class SubscriberController {
     const existingUser = await this.subscriberService.findByEmail(request.body.email)
     if(![200, 404].includes(existingUser.code)) {
       console.error(existingUser.code, existingUser.message);
+      Sentry.captureException(existingUser)
       response.status(existingUser.code).send({ error: existingUser.message })
       return;
     }
@@ -48,6 +53,8 @@ export class SubscriberController {
         status: "error",
         error: "A user with that email address already exists, and they are already in the desired list.",
       })
+      console.error({...existingUser, message: "A user with that email address already exists, and they are already in the desired list."})
+      Sentry.captureException({...existingUser, message: "A user with that email address already exists, and they are already in the desired list."})
       return;
     }
 
@@ -62,6 +69,16 @@ export class SubscriberController {
     response.status(200).send({
       status: "success",
     })
+
+    const errorInfo = {
+      email: request.body.email,
+      anonymous_id: addToQueue.anonymous_id,
+      lists: this.list
+    }
+
+    // Now we keep checking to make sure the import was successful
+    const importConfirmation = await this.subscriberService.checkCreate(addToQueue.result[1]["job_id"], response, 0, CHECKS_BEFORE_FAIL, PAUSE_BETWEEN_CHECKS, errorInfo);
+
     return;
 
   }

server/src/subscriber/subscriber.service.ts

@@ -2,6 +2,8 @@ import { Injectable, Res } from "@nestjs/common";
 import { ConfigService } from "../config/config.service";
 import { Client } from "@sendgrid/client";
 import crypto from 'crypto';
+import * as Sentry from "@sentry/nestjs";
+
 const validCustomFieldNames = ["K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17", "K18", "X01", "X02", "X03", "X04", "X05", "X06", "X07", "X08", "X09", "X10", "X11", "X12", "M01", "M02", "M03", "M04", "M05", "M06", "M07", "M08", "M09", "M10", "M11", "M12", "Q01", "Q02", "Q03", "Q04", "Q05", "Q06", "Q07", "Q08", "Q09", "Q10", "Q11", "Q12", "Q13", "Q14", "R01", "R02", "R03", "CW"] as const;
 export type CustomFieldNameTuple = typeof validCustomFieldNames;
 type CustomFieldName = CustomFieldNameTuple[number];
@@ -82,9 +84,64 @@ export class SubscriberService {
     // https://www.twilio.com/docs/sendgrid/api-reference/contacts/add-or-update-a-contact
     try {
       const result = await this.client.request(addRequest);
-      return {isError: false, result: result};
+      return {isError: false, result: result, anonymous_id: id};
     } catch(error) {
-      return {isError: true, ...error};
+      console.error(error, addRequest.body)
+      Sentry.captureException({error, email, id, list})
+      return {isError: true, ...error, request_body: addRequest.body};
+    }
+  }
+
+
+  /**
+   * Checks that a job has imported correctly.
+   * @param {string} importId - The job id returned by the initial request
+   * @param {number} counter - Tracks the number of times we have checked
+   * @param {number} checksBeforeFail - Max # times to check
+   * @param {number} pauseBetweenChecks - How long to wait between checks, in milliseconds
+   * @param {object} errorInfo - Additional info to log in case of error
+   * @returns {object}
+   */
+  async checkCreate(importId: string, @Res() response, counter: number = 0, checksBeforeFail: number, pauseBetweenChecks: number, errorInfo: any) {
+    if(counter >= checksBeforeFail) {
+      console.error({
+        code: 408,
+        message: `Polling limit of ${checksBeforeFail} checks with a ${pauseBetweenChecks/1000} second delay between each has been reached.`,
+        job_id: importId,
+        errorInfo
+      })
+      Sentry.captureException({
+        code: 408,
+        message: `Polling limit of ${checksBeforeFail} checks with a ${pauseBetweenChecks/1000} second delay between each has been reached.`,
+        job_id: importId,
+        errorInfo
+      })
+      return { isError: true, code: 408, errorInfo }
+    }
+
+    await delay(pauseBetweenChecks);
+
+    const confirmationRequest = {
+      url: `/v3/marketing/contacts/imports/${importId}`,
+      // method:<HttpMethod> 'GET',
+      method:<HttpMethod> 'GET',
+    }
+
+    // https://www.twilio.com/docs/sendgrid/api-reference/contacts/import-contacts-status
+    try {
+      const job = await this.client.request(confirmationRequest);
+      if(job[1].status === "pending") {
+        return await this.checkCreate(importId, response, counter + 1, checksBeforeFail, pauseBetweenChecks, errorInfo);
+      } else if (["errored", "failed"].includes(job[1].status)) {
+        console.error(job, errorInfo);
+        Sentry.captureException(job, errorInfo);
+        return {isError: true, job, errorInfo};
+      }
+      return {isError: false, status: job[1].status, ...job};
+    } catch(error) {
+      console.error(error, errorInfo);
+      Sentry.captureException(error, errorInfo);
+      return {isError: true, ...error, errorInfo};
     }
   }
 
@@ -125,4 +182,6 @@ export class SubscriberService {
   private validateSubscriptionValue(value: number): value is CustomFieldValue {
     return validCustomFieldValues.includes(value as CustomFieldValue);
   }
+
+
 }