diff --git a/src/internal/processScans.ts b/src/internal/processScans.ts index ea790d3..05d5d32 100644 --- a/src/internal/processScans.ts +++ b/src/internal/processScans.ts @@ -13,7 +13,7 @@ export const processScans = async (event) => { const pollScans = (givenJobIds) => new Promise(async (finalRes) => { await sleep(1000); const remainingScans = []; - const batchesOfJobIds = chunk(givenJobIds, 100); + const batchesOfJobIds = chunk(givenJobIds, 25); for (const [index, batchOfJobIds] of batchesOfJobIds.entries()) { console.log(`Start ${index} of ${batchesOfJobIds.length} batches`); await Promise.allSettled(batchOfJobIds.map(jobId => new Promise(async (res) => { diff --git a/src/routes/getResultsAll.ts b/src/routes/getResultsAll.ts index 25efeed..f0bb8ab 100644 --- a/src/routes/getResultsAll.ts +++ b/src/routes/getResultsAll.ts @@ -1,4 +1,5 @@ import { db, getMode, graphql } from '#src/utils'; +import { gzipSync } from 'zlib'; export const getResultsAll = async ({ request, reply }) => { /* @@ -60,14 +61,16 @@ export const getResultsAll = async ({ request, reply }) => { id message type - messageTags: message_tags(where:{ - ${filters.tags.length > 0 ? `tag:{id: {_in: $tagIds}},` : ``} - }) { - tag { - id - tag + ${urls.length < 100 ? ` + messageTags: message_tags(where:{ + ${filters.tags.length > 0 ? `tag:{id: {_in: $tagIds}},` : ``} + }) { + tag { + id + tag + } } - } + ` : ''} } } } @@ -99,7 +102,7 @@ export const getResultsAll = async ({ request, reply }) => { const formattedTags = {}; for (const tag of filteredNodes .map(obj => obj.messageNodes).flat() - .map(obj => obj.message.messageTags).flat() + .map(obj => obj.message?.messageTags ?? []).flat() .map(obj => obj.tag) ) { if (!formattedTags[tag.tag]) { @@ -136,10 +139,11 @@ export const getResultsAll = async ({ request, reply }) => { values: [stats, request.query.reportId], }); - return { + const arrayLimit = 10000; + const body = { reportName: report.name, - urls: urls.filter(url => nodeUrlIds.includes(url.id)), - nodes: filteredNodes.map(obj => ({ + urls: urls.filter(url => nodeUrlIds.slice(0, arrayLimit).includes(url.id)), + nodes: filteredNodes.slice(0, arrayLimit).map(obj => ({ nodeId: obj.nodeId, html: obj.html, targets: obj.targets, @@ -147,13 +151,16 @@ export const getResultsAll = async ({ request, reply }) => { equalified: obj.equalified, })), messages: Object.values(formattedMessages) - .sort((a, b) => a.activeCount > b.activeCount ? -1 : 1) + .sort((a, b) => a.activeCount > b.activeCount ? -1 : 1).slice(0, arrayLimit) .map(obj => ({ ...obj, totalCount: obj.equalifiedCount + obj.activeCount, })), tags: Object.values(formattedTags), - chart: Object.values(formattedChart) + chart: Object.values(formattedChart).slice(0, arrayLimit) .sort((a, b) => a.date > b.date ? -1 : 1), }; + const compressedBody = gzipSync(JSON.stringify(body)); + reply.headers({ 'content-encoding': 'gzip' }); + return reply.send(compressedBody) } \ No newline at end of file diff --git a/src/scheduled/runEveryFifteenMinutes.ts b/src/scheduled/runEveryFifteenMinutes.ts index 44ba702..b21f004 100644 --- a/src/scheduled/runEveryFifteenMinutes.ts +++ b/src/scheduled/runEveryFifteenMinutes.ts @@ -5,10 +5,23 @@ const lambda = new LambdaClient(); export const runEveryFifteenMinutes = async () => { await db.connect(); const pendingScans = (await db.query({ - text: `SELECT DISTINCT "user_id", "property_id" FROM "scans" WHERE "processing" = true`, + text: `SELECT DISTINCT s.user_id, s.property_id, CONCAT(u.first_name,' ',u.last_name) AS user_name, u.email AS user_email, p.name AS property_name, COUNT(s.id) AS "pending_scans" FROM scans AS s + INNER JOIN users AS u ON u.id=s.user_id + INNER JOIN properties AS p ON p.id=s.property_id + WHERE s.processing = true + GROUP BY s.user_id, s.property_id, user_name, user_email, property_name + ORDER BY pending_scans DESC`, })).rows; await db.clean(); + console.log(JSON.stringify({ pendingScans })); + await fetch(process.env.SLACK_WEBHOOK, { + method: 'POST', + body: JSON.stringify({ + text: pendingScans.map(obj => `*${obj.user_name}* (${obj.user_email}) scan for *${obj.property_name}*: ${obj.pending_scans} scans left`).join('\n') + }) + }); + for (const { user_id, property_id } of pendingScans) { lambda.send(new InvokeCommand({ FunctionName: `equalify-api${isStaging ? '-staging' : ''}`,