Creating /articles endpoint

manifest.json

@@ -1,7 +1,7 @@
 {
   "name": "docs-in-tickets",
   "vendor": "vtexhelp",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "title": "Docs in Tickets",
   "description": "Process information about documentation links sent in support tickets.",
   "categories": [],
@@ -41,6 +41,13 @@
         "host": "vtexhelp.zendesk.com",
         "path": "*"
       }
+    },
+    {
+      "name": "outbound-access",
+      "attrs": {
+        "host": "data-consumption.vtex.com",
+        "path": "*"
+      }
     }
   ],
   "$schema": "https://raw.githubusercontent.com/vtex/node-vtex-api/master/gen/manifest.schema"

node/clients/dataQuery.ts

@@ -0,0 +1,59 @@
+// getCommentData fetches Zendesk comment data from the data lake via Data Query API.
+// https://internal-docs.vtex.com/Analytics/VTEX-Data-Platform/data-query-API/
+// DQAPI Metric:
+// https://github.com/vtex/analytics-platform/blob/main/lambdas/functions/data_consumption_entrypoint_v2/metrics/docs-in-tickets.yaml
+
+import { ExternalClient } from '@vtex/api'
+import type { IOContext, InstanceOptions } from '@vtex/api'
+
+// These types must be `string | string[]` because technically querystrings can be arrays of strings. But we plan to deal only with simple strings.
+export interface fetchCommentsParams {
+  startDate: string | string[],
+  endDate: string | string[],
+  containsHelpArticle?: string | string[],
+  containsDevArticle?: string | string[],
+  articleUrl?: string | string[]
+}
+
+export default class DataQueryClient extends ExternalClient {
+  constructor(ctx: IOContext, options?: InstanceOptions) {
+    super(`https://data-consumption.vtex.com`, ctx, {
+      ...options,
+      retries: 2,
+      timeout: 6000,
+      headers: {
+        ...options?.headers
+      }
+    })
+  }
+
+  public async fetchCommentData (params: fetchCommentsParams, token: string | undefined | string[]) {
+
+    // Assembling URL considering optional query params
+    var queryParams = `an=vtexhelp&startDate=${params.startDate}&endDate=${params.endDate}`
+
+    if (params.containsHelpArticle) {
+      queryParams = `${queryParams}&contains_help_article=${params.containsHelpArticle}`
+    }
+    if (params.containsDevArticle) {
+      queryParams = `${queryParams}&contains_dev_article=${params.containsDevArticle}`
+    }
+    if (params.articleUrl) {
+      queryParams = `${queryParams}&article_url=${params.articleUrl}`
+    }
+
+    const url = `https://data-consumption.vtex.com/api/analytics/consumption/docs-in-tickets?${queryParams}`
+
+    console.log(url)
+
+    // Making the request
+    const dataQuery = this.http.get(url,
+      {
+        headers: {
+          cookie: `VtexIdclientAutCookie=${token}`
+        }
+      }
+    )
+    return dataQuery
+  }
+}

node/clients/index.ts

@@ -3,6 +3,7 @@ import { IOClients } from '@vtex/api'
 import RedshiftClient from './redshift'
 import ZendeskClient from './zendesk'
 import SlackClient from './slack'
+import DataQueryClient from './dataQuery'
 
 // Extend the default IOClients implementation with our own custom clients.
 export class Clients extends IOClients {
@@ -17,4 +18,8 @@ export class Clients extends IOClients {
   public get slack() {
     return this.getOrSet('slack', SlackClient)
   }
+
+  public get dataQuery() {
+    return this.getOrSet('dataQuery', DataQueryClient)
+  }
 }

node/clients/slack.ts

@@ -4,7 +4,7 @@
 import { ExternalClient } from '@vtex/api'
 import type { IOContext, InstanceOptions } from '@vtex/api'
 
-const url = 'https://hooks.slack.com/services/xxxxxxxxxx'
+const url = 'https://hooks.slack.com/services/{secret}'
 const requestHeaders = {
   'Content-Type': 'application/json',
 }

node/clients/zendesk.ts

@@ -3,13 +3,13 @@
 import { ExternalClient, IOContext, InstanceOptions } from '@vtex/api'
 
 // These are the real credentials
-const username = 'xxxxxxxxxx'
-const token = 'xxxxxxxxxx'
+const username = '{secret}'
+const token = '{secret}'
 const authValue = btoa(`${username}/token:${token}`)
 
 // Below are sandbox auth variables, just to facilitate debugging. Note that the authvalue format is different.
-// const username = 'xxxxxxxxxx'
-// const token = 'xxxxxxxxxx'
+// const username = '{secret}'
+// const token = '{secret}'
 // const authValue = btoa(`${username}:${token}`)
 
 const requestHeaders = {

node/index.ts

@@ -4,6 +4,9 @@ import { LRUCache, method, Service } from '@vtex/api'
 import { Clients } from './clients'
 import { processTicket } from './middlewares/processTicket'
 import { verifyZendeskSignature } from './middlewares/verifyZendeskSignature'
+import { getCommentData } from './middlewares/getCommentData'
+import { processCommentData } from './middlewares/processCommentData'
+import { validateParams } from './middlewares/validateParams'
 
 const TIMEOUT_MS = 800
 
@@ -48,9 +51,11 @@ declare global {
 export default new Service({
   clients,
   routes: {
-    // `status` is the route ID from service.json. It maps to an array of middlewares (or a single handler).
     tickets: method({
       POST: [verifyZendeskSignature, processTicket],
     }),
+    articles: method({
+      GET: [validateParams, getCommentData, processCommentData],
+    }),
   },
 })

node/middlewares/errorLogs.ts

@@ -1,6 +1,6 @@
 // Returning and logging errors
 
-export async function returnError(
+export async function returnErrorTicket(
   zendeskTicket: string,
   status: number,
   errorMessage: string,
@@ -16,3 +16,26 @@ export async function returnError(
     message: errorMessage
   }
 }
+
+export async function returnErrorUrl(
+  articleUrl: string | undefined,
+  status: number,
+  errorMessage: string,
+  ctx: Context
+) {
+  // Making sure the returned url is not between %%
+  var urlToReturn: string | undefined
+  if (articleUrl) {
+    urlToReturn = articleUrl.slice(1, -1)
+  } else {
+    urlToReturn = articleUrl
+  }
+  const slack = ctx.clients.slack
+  slack.sendLog(`Article: ${urlToReturn}\n${errorMessage}`, 'error')
+
+  ctx.status = status
+  ctx.response.body = {
+    article: urlToReturn,
+    message: errorMessage
+  }
+}

node/middlewares/getCommentData.ts

@@ -0,0 +1,30 @@
+// Getting comment data from the Data Lake
+
+import { returnErrorUrl } from '../middlewares/errorLogs'
+
+export async function getCommentData(
+  ctx: Context,
+  next: () => Promise<Record<string, unknown>>
+) {
+  console.log('Running getCommentData')
+
+  const dataQuery = ctx.clients.dataQuery
+
+  const params = ctx.state.body
+  try{
+    const dataQueryResponse = await dataQuery.fetchCommentData(params, ctx.request.headers.vtexidclientautcookie)
+
+    // Passing the data to the body to get it in other middlewares
+    ctx.state.body = {
+      articleData: dataQueryResponse,
+      queryParams: params
+    }
+
+  } catch (error) {
+    returnErrorUrl(params.articleUrl, 500, `Error trying to get article data from the DQAPI >>> ${error}`, ctx)
+    console.log('error: '+error)
+    return
+  }
+
+  await next()
+}

node/middlewares/processCommentData.ts

@@ -0,0 +1,80 @@
+// Getting comment data from the Data Lake
+
+interface Comment {
+  ticketid: string
+  commentid: number
+  authorid: number
+  createdat: string
+  containshelparticle: boolean
+  containsdevarticle: boolean
+  numberofdocsportalsurls: number
+  docsportalsurls: string[]
+  numberofarticleurls: number
+  articleurls: string[]
+}
+
+export async function processCommentData(
+  ctx: Context,
+  next: () => Promise<Record<string, unknown>>
+) {
+  console.log('Running processUniqueTicketIds')
+
+  const articleData = ctx.state.body.articleData
+
+  // Get unique tickets
+  const uniqueTicketIds = [...new Set(articleData.map((item: Comment) => item.ticketid))];
+
+  // Calculate number of tickets comments given that "empty" means an empty object in the array and still counts as 1.
+  var numberOfComments = 0
+  var numberOfTickets
+  if (articleData[0].ticketid) {
+    numberOfComments = articleData.length
+    numberOfTickets = uniqueTicketIds.length
+  }
+
+  // Set a max quantity and return a sample of the comments. The goal is to set up pagination to return all results, but first we must set up pagination in the DQAPI metric.
+  const sampleSize = 20
+  var sampleQuantity = 0
+  var sampleData: Comment[] = []
+  if (numberOfComments > sampleSize) {
+    sampleQuantity = sampleSize,
+    sampleData = articleData.slice(0,sampleSize)
+  } else {
+    sampleQuantity = numberOfComments,
+    sampleData = articleData
+  }
+
+  // Assembling response
+
+  // Making sure the returned url is not between %%
+  const queryParams = ctx.state.body.queryParams
+  var urlToReturn: string | undefined
+  if (queryParams.articleUrl) {
+    urlToReturn = queryParams.articleUrl.slice(1, -1)
+  } else {
+    urlToReturn = queryParams.articleUrl
+  }
+
+  ctx.status = 200
+  ctx.response.body = {
+    query: {
+      startDate: queryParams.startDate,
+      endDate: queryParams.endDate,
+      containsHelpArticle: queryParams.containsHelpArticle,
+      containsDevArticle: queryParams.containsDevArticle,
+      articleUrl: urlToReturn
+    },
+    tickets : {
+      quantity: numberOfTickets,
+      ids: uniqueTicketIds,
+    },
+    comments: {
+      quantity: numberOfComments,
+      sample: {
+        quantity: sampleQuantity,
+        data: sampleData
+      }
+    }
+  }
+  await next()
+}

node/middlewares/processTicket.ts

@@ -3,7 +3,7 @@
 import { or } from 'ramda'
 
 import type { MessageData } from '../clients/redshift'
-import { returnError } from './errorLogs'
+import { returnErrorTicket } from './errorLogs'
 
 // Defining URLs that will be used to parse and process the comment data
 // Substrings to look for and to exclude
@@ -20,6 +20,22 @@ const helpUrlSlashHttp = 'http://help.vtex.com/'
 const devUrlSlashHttp = 'http://developers.vtex.com/'
 const helpUrlShortSlash = 'help.vtex.com/'
 const devUrlShortSlash = 'developers.vtex.com/'
+const helpAnnouncements = 'https://help.vtex.com/announcements'
+const helpAnnouncementsSlash = 'https://help.vtex.com/announcements'
+const helpAnnouncementsShort = 'help.vtex.com/announcements'
+const helpAnnouncementsShortSlash = 'https://help.vtex.com/announcements'
+const helpAnnouncementsPt = 'https://help.vtex.com/pt/announcements'
+const helpAnnouncementsSlashPt = 'https://help.vtex.com/pt/announcements'
+const helpAnnouncementsShortPt = 'help.vtex.com/pt/announcements'
+const helpAnnouncementsShortSlashPt = 'https://help.vtex.com/pt/announcements'
+const helpAnnouncementsEn = 'https://help.vtex.com/en/announcements'
+const helpAnnouncementsSlashEn = 'https://help.vtex.com/en/announcements'
+const helpAnnouncementsShortEn = 'help.vtex.com/en/announcements'
+const helpAnnouncementsShortSlashEn = 'https://help.vtex.com/en/announcements'
+const helpAnnouncementsEs = 'https://help.vtex.com/es/announcements'
+const helpAnnouncementsSlashEs = 'https://help.vtex.com/es/announcements'
+const helpAnnouncementsShortEs = 'help.vtex.com/es/announcements'
+const helpAnnouncementsShortSlashEs = 'https://help.vtex.com/es/announcements'
 
 export async function processTicket(
   ctx: Context,
@@ -51,7 +67,7 @@ export async function processTicket(
       ticketComments = zendeskData.comments
       nextPage = zendeskData.next_page
     } catch (error) {
-      returnError(zendeskTicket, 500, `Error trying to get ticket data from Zendesk >>> ${error}`, ctx)
+      returnErrorTicket(zendeskTicket, 500, `Error trying to get ticket data from Zendesk >>> ${error}`, ctx)
       return
     }
 
@@ -86,6 +102,22 @@ export async function processTicket(
         .filter((url: string) => url !== devUrlHttp)
         .filter((url: string) => url !== helpUrlSlashHttp)
         .filter((url: string) => url !== devUrlSlashHttp)
+        .filter((url: string) => url !== helpAnnouncements)
+        .filter((url: string) => url !== helpAnnouncementsSlash)
+        .filter((url: string) => url !== helpAnnouncementsShort)
+        .filter((url: string) => url !== helpAnnouncementsShortSlash)
+        .filter((url: string) => url !== helpAnnouncementsPt)
+        .filter((url: string) => url !== helpAnnouncementsSlashPt)
+        .filter((url: string) => url !== helpAnnouncementsShortPt)
+        .filter((url: string) => url !== helpAnnouncementsShortSlashPt)
+        .filter((url: string) => url !== helpAnnouncementsEn)
+        .filter((url: string) => url !== helpAnnouncementsSlashEn)
+        .filter((url: string) => url !== helpAnnouncementsShortEn)
+        .filter((url: string) => url !== helpAnnouncementsShortSlashEn)
+        .filter((url: string) => url !== helpAnnouncementsEs)
+        .filter((url: string) => url !== helpAnnouncementsSlashEs)
+        .filter((url: string) => url !== helpAnnouncementsShortEs)
+        .filter((url: string) => url !== helpAnnouncementsShortSlashEs)
 
       // Checking to see which portals the articles pertain to
       const hasHelpArticle = docUrls.some((url: string) =>
@@ -114,11 +146,11 @@ export async function processTicket(
         allCommentsWithUrls.push(messageData)
 
         try {
-          console.log('try redshift')
+          // console.log('try redshift')
           await redshift.saveMessage(messageData)
           console.log(messageData.numberOfArticleUrls)
         } catch (error) {
-          returnError(zendeskTicket, 500, `Error trying to save comment data to RedShift >>> ${error}`, ctx)
+          returnErrorTicket(zendeskTicket, 500, `Error trying to save comment data to RedShift >>> ${error}`, ctx)
         }
       }
     }