From e6d6681b830f96d01319e910274b17df30e2fa19 Mon Sep 17 00:00:00 2001 From: PedroAntunesCosta <47991446+PedroAntunesCosta@users.noreply.github.com> Date: Fri, 13 Sep 2024 15:26:56 -0300 Subject: [PATCH] Creating /articles endpoint --- manifest.json | 9 ++- node/clients/dataQuery.ts | 59 ++++++++++++++ node/clients/index.ts | 5 ++ node/clients/slack.ts | 2 +- node/clients/zendesk.ts | 8 +- node/index.ts | 7 +- node/middlewares/errorLogs.ts | 25 +++++- node/middlewares/getCommentData.ts | 30 +++++++ node/middlewares/processCommentData.ts | 80 +++++++++++++++++++ node/middlewares/processTicket.ts | 40 +++++++++- node/middlewares/validateParams.ts | 91 ++++++++++++++++++++++ node/middlewares/verifyZendeskSignature.ts | 12 +-- node/service.json | 4 + 13 files changed, 354 insertions(+), 18 deletions(-) create mode 100644 node/clients/dataQuery.ts create mode 100644 node/middlewares/getCommentData.ts create mode 100644 node/middlewares/processCommentData.ts create mode 100644 node/middlewares/validateParams.ts diff --git a/manifest.json b/manifest.json index 4906835..7104d07 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "name": "docs-in-tickets", "vendor": "vtexhelp", - "version": "1.0.4", + "version": "1.0.5", "title": "Docs in Tickets", "description": "Process information about documentation links sent in support tickets.", "categories": [], @@ -41,6 +41,13 @@ "host": "vtexhelp.zendesk.com", "path": "*" } + }, + { + "name": "outbound-access", + "attrs": { + "host": "data-consumption.vtex.com", + "path": "*" + } } ], "$schema": "https://raw.githubusercontent.com/vtex/node-vtex-api/master/gen/manifest.schema" diff --git a/node/clients/dataQuery.ts b/node/clients/dataQuery.ts new file mode 100644 index 0000000..34d0a9b --- /dev/null +++ b/node/clients/dataQuery.ts @@ -0,0 +1,59 @@ +// getCommentData fetches Zendesk comment data from the data lake via Data Query API. +// https://internal-docs.vtex.com/Analytics/VTEX-Data-Platform/data-query-API/ +// DQAPI Metric: +// https://github.com/vtex/analytics-platform/blob/main/lambdas/functions/data_consumption_entrypoint_v2/metrics/docs-in-tickets.yaml + +import { ExternalClient } from '@vtex/api' +import type { IOContext, InstanceOptions } from '@vtex/api' + +// These types must be `string | string[]` because technically querystrings can be arrays of strings. But we plan to deal only with simple strings. +export interface fetchCommentsParams { + startDate: string | string[], + endDate: string | string[], + containsHelpArticle?: string | string[], + containsDevArticle?: string | string[], + articleUrl?: string | string[] +} + +export default class DataQueryClient extends ExternalClient { + constructor(ctx: IOContext, options?: InstanceOptions) { + super(`https://data-consumption.vtex.com`, ctx, { + ...options, + retries: 2, + timeout: 6000, + headers: { + ...options?.headers + } + }) + } + + public async fetchCommentData (params: fetchCommentsParams, token: string | undefined | string[]) { + + // Assembling URL considering optional query params + var queryParams = `an=vtexhelp&startDate=${params.startDate}&endDate=${params.endDate}` + + if (params.containsHelpArticle) { + queryParams = `${queryParams}&contains_help_article=${params.containsHelpArticle}` + } + if (params.containsDevArticle) { + queryParams = `${queryParams}&contains_dev_article=${params.containsDevArticle}` + } + if (params.articleUrl) { + queryParams = `${queryParams}&article_url=${params.articleUrl}` + } + + const url = `https://data-consumption.vtex.com/api/analytics/consumption/docs-in-tickets?${queryParams}` + + console.log(url) + + // Making the request + const dataQuery = this.http.get(url, + { + headers: { + cookie: `VtexIdclientAutCookie=${token}` + } + } + ) + return dataQuery + } +} diff --git a/node/clients/index.ts b/node/clients/index.ts index c8a06f6..e09864d 100644 --- a/node/clients/index.ts +++ b/node/clients/index.ts @@ -3,6 +3,7 @@ import { IOClients } from '@vtex/api' import RedshiftClient from './redshift' import ZendeskClient from './zendesk' import SlackClient from './slack' +import DataQueryClient from './dataQuery' // Extend the default IOClients implementation with our own custom clients. export class Clients extends IOClients { @@ -17,4 +18,8 @@ export class Clients extends IOClients { public get slack() { return this.getOrSet('slack', SlackClient) } + + public get dataQuery() { + return this.getOrSet('dataQuery', DataQueryClient) + } } diff --git a/node/clients/slack.ts b/node/clients/slack.ts index b3c015b..76f0673 100644 --- a/node/clients/slack.ts +++ b/node/clients/slack.ts @@ -4,7 +4,7 @@ import { ExternalClient } from '@vtex/api' import type { IOContext, InstanceOptions } from '@vtex/api' -const url = 'https://hooks.slack.com/services/xxxxxxxxxx' +const url = 'https://hooks.slack.com/services/{secret}' const requestHeaders = { 'Content-Type': 'application/json', } diff --git a/node/clients/zendesk.ts b/node/clients/zendesk.ts index 6975083..1b26b7d 100644 --- a/node/clients/zendesk.ts +++ b/node/clients/zendesk.ts @@ -3,13 +3,13 @@ import { ExternalClient, IOContext, InstanceOptions } from '@vtex/api' // These are the real credentials -const username = 'xxxxxxxxxx' -const token = 'xxxxxxxxxx' +const username = '{secret}' +const token = '{secret}' const authValue = btoa(`${username}/token:${token}`) // Below are sandbox auth variables, just to facilitate debugging. Note that the authvalue format is different. -// const username = 'xxxxxxxxxx' -// const token = 'xxxxxxxxxx' +// const username = '{secret}' +// const token = '{secret}' // const authValue = btoa(`${username}:${token}`) const requestHeaders = { diff --git a/node/index.ts b/node/index.ts index 70a9585..684868a 100644 --- a/node/index.ts +++ b/node/index.ts @@ -4,6 +4,9 @@ import { LRUCache, method, Service } from '@vtex/api' import { Clients } from './clients' import { processTicket } from './middlewares/processTicket' import { verifyZendeskSignature } from './middlewares/verifyZendeskSignature' +import { getCommentData } from './middlewares/getCommentData' +import { processCommentData } from './middlewares/processCommentData' +import { validateParams } from './middlewares/validateParams' const TIMEOUT_MS = 800 @@ -48,9 +51,11 @@ declare global { export default new Service({ clients, routes: { - // `status` is the route ID from service.json. It maps to an array of middlewares (or a single handler). tickets: method({ POST: [verifyZendeskSignature, processTicket], }), + articles: method({ + GET: [validateParams, getCommentData, processCommentData], + }), }, }) diff --git a/node/middlewares/errorLogs.ts b/node/middlewares/errorLogs.ts index d5b5dd3..43103b1 100644 --- a/node/middlewares/errorLogs.ts +++ b/node/middlewares/errorLogs.ts @@ -1,6 +1,6 @@ // Returning and logging errors -export async function returnError( +export async function returnErrorTicket( zendeskTicket: string, status: number, errorMessage: string, @@ -16,3 +16,26 @@ export async function returnError( message: errorMessage } } + +export async function returnErrorUrl( + articleUrl: string | undefined, + status: number, + errorMessage: string, + ctx: Context +) { + // Making sure the returned url is not between %% + var urlToReturn: string | undefined + if (articleUrl) { + urlToReturn = articleUrl.slice(1, -1) + } else { + urlToReturn = articleUrl + } + const slack = ctx.clients.slack + slack.sendLog(`Article: ${urlToReturn}\n${errorMessage}`, 'error') + + ctx.status = status + ctx.response.body = { + article: urlToReturn, + message: errorMessage + } +} diff --git a/node/middlewares/getCommentData.ts b/node/middlewares/getCommentData.ts new file mode 100644 index 0000000..73a421a --- /dev/null +++ b/node/middlewares/getCommentData.ts @@ -0,0 +1,30 @@ +// Getting comment data from the Data Lake + +import { returnErrorUrl } from '../middlewares/errorLogs' + +export async function getCommentData( + ctx: Context, + next: () => Promise> +) { + console.log('Running getCommentData') + + const dataQuery = ctx.clients.dataQuery + + const params = ctx.state.body + try{ + const dataQueryResponse = await dataQuery.fetchCommentData(params, ctx.request.headers.vtexidclientautcookie) + + // Passing the data to the body to get it in other middlewares + ctx.state.body = { + articleData: dataQueryResponse, + queryParams: params + } + + } catch (error) { + returnErrorUrl(params.articleUrl, 500, `Error trying to get article data from the DQAPI >>> ${error}`, ctx) + console.log('error: '+error) + return + } + + await next() +} diff --git a/node/middlewares/processCommentData.ts b/node/middlewares/processCommentData.ts new file mode 100644 index 0000000..282f3c2 --- /dev/null +++ b/node/middlewares/processCommentData.ts @@ -0,0 +1,80 @@ +// Getting comment data from the Data Lake + +interface Comment { + ticketid: string + commentid: number + authorid: number + createdat: string + containshelparticle: boolean + containsdevarticle: boolean + numberofdocsportalsurls: number + docsportalsurls: string[] + numberofarticleurls: number + articleurls: string[] +} + +export async function processCommentData( + ctx: Context, + next: () => Promise> +) { + console.log('Running processUniqueTicketIds') + + const articleData = ctx.state.body.articleData + + // Get unique tickets + const uniqueTicketIds = [...new Set(articleData.map((item: Comment) => item.ticketid))]; + + // Calculate number of tickets comments given that "empty" means an empty object in the array and still counts as 1. + var numberOfComments = 0 + var numberOfTickets + if (articleData[0].ticketid) { + numberOfComments = articleData.length + numberOfTickets = uniqueTicketIds.length + } + + // Set a max quantity and return a sample of the comments. The goal is to set up pagination to return all results, but first we must set up pagination in the DQAPI metric. + const sampleSize = 20 + var sampleQuantity = 0 + var sampleData: Comment[] = [] + if (numberOfComments > sampleSize) { + sampleQuantity = sampleSize, + sampleData = articleData.slice(0,sampleSize) + } else { + sampleQuantity = numberOfComments, + sampleData = articleData + } + + // Assembling response + + // Making sure the returned url is not between %% + const queryParams = ctx.state.body.queryParams + var urlToReturn: string | undefined + if (queryParams.articleUrl) { + urlToReturn = queryParams.articleUrl.slice(1, -1) + } else { + urlToReturn = queryParams.articleUrl + } + + ctx.status = 200 + ctx.response.body = { + query: { + startDate: queryParams.startDate, + endDate: queryParams.endDate, + containsHelpArticle: queryParams.containsHelpArticle, + containsDevArticle: queryParams.containsDevArticle, + articleUrl: urlToReturn + }, + tickets : { + quantity: numberOfTickets, + ids: uniqueTicketIds, + }, + comments: { + quantity: numberOfComments, + sample: { + quantity: sampleQuantity, + data: sampleData + } + } + } + await next() +} diff --git a/node/middlewares/processTicket.ts b/node/middlewares/processTicket.ts index 74c097f..3c35d1f 100644 --- a/node/middlewares/processTicket.ts +++ b/node/middlewares/processTicket.ts @@ -3,7 +3,7 @@ import { or } from 'ramda' import type { MessageData } from '../clients/redshift' -import { returnError } from './errorLogs' +import { returnErrorTicket } from './errorLogs' // Defining URLs that will be used to parse and process the comment data // Substrings to look for and to exclude @@ -20,6 +20,22 @@ const helpUrlSlashHttp = 'http://help.vtex.com/' const devUrlSlashHttp = 'http://developers.vtex.com/' const helpUrlShortSlash = 'help.vtex.com/' const devUrlShortSlash = 'developers.vtex.com/' +const helpAnnouncements = 'https://help.vtex.com/announcements' +const helpAnnouncementsSlash = 'https://help.vtex.com/announcements' +const helpAnnouncementsShort = 'help.vtex.com/announcements' +const helpAnnouncementsShortSlash = 'https://help.vtex.com/announcements' +const helpAnnouncementsPt = 'https://help.vtex.com/pt/announcements' +const helpAnnouncementsSlashPt = 'https://help.vtex.com/pt/announcements' +const helpAnnouncementsShortPt = 'help.vtex.com/pt/announcements' +const helpAnnouncementsShortSlashPt = 'https://help.vtex.com/pt/announcements' +const helpAnnouncementsEn = 'https://help.vtex.com/en/announcements' +const helpAnnouncementsSlashEn = 'https://help.vtex.com/en/announcements' +const helpAnnouncementsShortEn = 'help.vtex.com/en/announcements' +const helpAnnouncementsShortSlashEn = 'https://help.vtex.com/en/announcements' +const helpAnnouncementsEs = 'https://help.vtex.com/es/announcements' +const helpAnnouncementsSlashEs = 'https://help.vtex.com/es/announcements' +const helpAnnouncementsShortEs = 'help.vtex.com/es/announcements' +const helpAnnouncementsShortSlashEs = 'https://help.vtex.com/es/announcements' export async function processTicket( ctx: Context, @@ -51,7 +67,7 @@ export async function processTicket( ticketComments = zendeskData.comments nextPage = zendeskData.next_page } catch (error) { - returnError(zendeskTicket, 500, `Error trying to get ticket data from Zendesk >>> ${error}`, ctx) + returnErrorTicket(zendeskTicket, 500, `Error trying to get ticket data from Zendesk >>> ${error}`, ctx) return } @@ -86,6 +102,22 @@ export async function processTicket( .filter((url: string) => url !== devUrlHttp) .filter((url: string) => url !== helpUrlSlashHttp) .filter((url: string) => url !== devUrlSlashHttp) + .filter((url: string) => url !== helpAnnouncements) + .filter((url: string) => url !== helpAnnouncementsSlash) + .filter((url: string) => url !== helpAnnouncementsShort) + .filter((url: string) => url !== helpAnnouncementsShortSlash) + .filter((url: string) => url !== helpAnnouncementsPt) + .filter((url: string) => url !== helpAnnouncementsSlashPt) + .filter((url: string) => url !== helpAnnouncementsShortPt) + .filter((url: string) => url !== helpAnnouncementsShortSlashPt) + .filter((url: string) => url !== helpAnnouncementsEn) + .filter((url: string) => url !== helpAnnouncementsSlashEn) + .filter((url: string) => url !== helpAnnouncementsShortEn) + .filter((url: string) => url !== helpAnnouncementsShortSlashEn) + .filter((url: string) => url !== helpAnnouncementsEs) + .filter((url: string) => url !== helpAnnouncementsSlashEs) + .filter((url: string) => url !== helpAnnouncementsShortEs) + .filter((url: string) => url !== helpAnnouncementsShortSlashEs) // Checking to see which portals the articles pertain to const hasHelpArticle = docUrls.some((url: string) => @@ -114,11 +146,11 @@ export async function processTicket( allCommentsWithUrls.push(messageData) try { - console.log('try redshift') + // console.log('try redshift') await redshift.saveMessage(messageData) console.log(messageData.numberOfArticleUrls) } catch (error) { - returnError(zendeskTicket, 500, `Error trying to save comment data to RedShift >>> ${error}`, ctx) + returnErrorTicket(zendeskTicket, 500, `Error trying to save comment data to RedShift >>> ${error}`, ctx) } } } diff --git a/node/middlewares/validateParams.ts b/node/middlewares/validateParams.ts new file mode 100644 index 0000000..425e3e6 --- /dev/null +++ b/node/middlewares/validateParams.ts @@ -0,0 +1,91 @@ +// Getting params and validating them + +import { fetchCommentsParams } from '../clients/dataQuery' +import { returnErrorUrl } from './errorLogs' +// import { returnErrorUrl } from '../middlewares/errorLogs' + +// Date validation function +function isValidDate(date: string): boolean { + + // Regular expression to match yyyy-mm-dd format + const regex = /^\d{4}-\d{2}-\d{2}$/ + + // Check if the string matches the format + if (!regex.test(date)) { + return false + } + + // Check date basic attributes + const year = parseInt(date.split('-')[0]) + if (year<2010) { + return false + } + + const month = parseInt(date.split('-')[1]) + if (month>12) { + return false + } + + const day = parseInt(date.split('-')[2]) + if (day>31) { + return false + } + + return true +} + +export async function validateParams( + ctx: Context, + next: () => Promise> +) { + console.log('Running validateParams') + + // Getting params from querystring + const receivedQueryStrings = ctx.request.query + + // Making sure the article url is either a string between %% or undefined + const receivedArticleUrl = receivedQueryStrings.articleUrl !== undefined + ? `%${receivedQueryStrings.articleUrl}%` + : undefined + + // Validate dates + const receivedStartDate = JSON.stringify(receivedQueryStrings.startDate).replace(/"/g, '') + if (!isValidDate(receivedStartDate)) { + returnErrorUrl(receivedArticleUrl, 500, `${receivedStartDate} is not a valid startDate value. Use the format yyyy-mm-dd.`, ctx) + return + } + + const receivedEndDate = JSON.stringify(receivedQueryStrings.endDate).replace(/"/g, '') + if (!isValidDate(receivedEndDate)) { + returnErrorUrl(receivedArticleUrl, 500, `${receivedEndDate} is not a valid endDate value. Use the format yyyy-mm-dd.`, ctx) + return + } + + // Validate boolean params + const receivedContainsHelpArticle = receivedQueryStrings.containsHelpArticle + if (receivedContainsHelpArticle && !(receivedContainsHelpArticle == 'true' || receivedContainsHelpArticle == 'false')) { + returnErrorUrl(receivedArticleUrl, 500, `${receivedContainsHelpArticle} is not a valid containsHelpArticle value. Use true or false.`, ctx) + return + } + + + const receivedContainsDevArticle = receivedQueryStrings.containsDevArticle + if (receivedContainsDevArticle && !(receivedContainsDevArticle == 'true' || receivedContainsDevArticle == 'false')) { + returnErrorUrl(receivedArticleUrl, 500, `${receivedContainsDevArticle} is not a valid containsDevArticle value. Use true or false.`, ctx) + return + } + + // Assembling params + const params: fetchCommentsParams = { + startDate: receivedStartDate, + endDate: receivedEndDate, + containsHelpArticle: receivedContainsHelpArticle, + containsDevArticle: receivedContainsDevArticle, + articleUrl: receivedArticleUrl + } + + // Passing the params to the state body to get it in other middlewares + ctx.state.body = params + + await next() +} diff --git a/node/middlewares/verifyZendeskSignature.ts b/node/middlewares/verifyZendeskSignature.ts index 4f8db2a..475f516 100644 --- a/node/middlewares/verifyZendeskSignature.ts +++ b/node/middlewares/verifyZendeskSignature.ts @@ -2,10 +2,10 @@ // https://developer.zendesk.com/documentation/webhooks/verifying/#verifying-the-signature import bodyParser from 'co-body' import * as crypto from 'crypto' -import { returnError } from './errorLogs' +import { returnErrorTicket } from './errorLogs' -const ZENDESK_SECRET_KEY_PRODUCTION = "xxxxxxxxxx" -const ZENDESK_SECRET_KEY_SANDBOX = "xxxxxxxxxx" +const ZENDESK_SECRET_KEY_PRODUCTION = "{secret}" +const ZENDESK_SECRET_KEY_SANDBOX = "{secret}" const SIGNING_SECRET_ALGORITHM = "sha256" export async function verifyZendeskSignature ( @@ -33,7 +33,7 @@ export async function verifyZendeskSignature ( console.log(requestBody.parsed.ticketId) if (requestBody.parsed.ticketId == undefined) { - returnError('undefined', 400, 'ticketId not found.', ctx) + returnErrorTicket('undefined', 400, 'ticketId not found.', ctx) } const zendeskTicket = requestBody.parsed.ticketId @@ -56,13 +56,13 @@ export async function verifyZendeskSignature ( await next() } else { - returnError(zendeskTicket, 400, 'Zendesk signature not valid.', ctx) + returnErrorTicket(zendeskTicket, 400, 'Zendesk signature not valid.', ctx) return } } } else { - returnError(zendeskTicket, 400, `Zendesk signature not found.`, ctx) + returnErrorTicket(zendeskTicket, 400, `Zendesk signature not found.`, ctx) return } } diff --git a/node/service.json b/node/service.json index db843ca..04f933f 100644 --- a/node/service.json +++ b/node/service.json @@ -9,6 +9,10 @@ "tickets": { "path": "/tickets", "public": true + }, + "articles": { + "path": "/articles", + "public": true } } }