Compressing getResults, ommitting messageTags join for expensive queries

EqualifyEverything · Aug 26, 2024 · b850a23 · b850a23
1 parent a9516cd
commit b850a23
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 15 deletions.
diff --git a/src/internal/processScans.ts b/src/internal/processScans.ts
@@ -13,7 +13,7 @@ export const processScans = async (event) => {
     const pollScans = (givenJobIds) => new Promise(async (finalRes) => {
         await sleep(1000);
         const remainingScans = [];
-        const batchesOfJobIds = chunk(givenJobIds, 100);
+        const batchesOfJobIds = chunk(givenJobIds, 25);
         for (const [index, batchOfJobIds] of batchesOfJobIds.entries()) {
             console.log(`Start ${index} of ${batchesOfJobIds.length} batches`);
             await Promise.allSettled(batchOfJobIds.map(jobId => new Promise(async (res) => {

diff --git a/src/routes/getResultsAll.ts b/src/routes/getResultsAll.ts
@@ -1,4 +1,5 @@
 import { db, getMode, graphql } from '#src/utils';
+import { gzipSync } from 'zlib';
 
 export const getResultsAll = async ({ request, reply }) => {
     /*
@@ -60,14 +61,16 @@ export const getResultsAll = async ({ request, reply }) => {
                         id
                         message
                         type
-                        messageTags: message_tags(where:{
-                            ${filters.tags.length > 0 ? `tag:{id: {_in: $tagIds}},` : ``}
-                        }) {
-                            tag {
-                                id
-                                tag
+                        ${urls.length < 100 ? `
+                            messageTags: message_tags(where:{
+                                ${filters.tags.length > 0 ? `tag:{id: {_in: $tagIds}},` : ``}
+                            }) {
+                                tag {
+                                    id
+                                    tag
+                                }
                             }
-                        }
+                        ` : ''}
                     }
                 }
             }
@@ -99,7 +102,7 @@ export const getResultsAll = async ({ request, reply }) => {
     const formattedTags = {};
     for (const tag of filteredNodes
         .map(obj => obj.messageNodes).flat()
-        .map(obj => obj.message.messageTags).flat()
+        .map(obj => obj.message?.messageTags ?? []).flat()
         .map(obj => obj.tag)
     ) {
         if (!formattedTags[tag.tag]) {
@@ -136,24 +139,28 @@ export const getResultsAll = async ({ request, reply }) => {
         values: [stats, request.query.reportId],
     });
 
-    return {
+    const arrayLimit = 10000;
+    const body = {
         reportName: report.name,
-        urls: urls.filter(url => nodeUrlIds.includes(url.id)),
-        nodes: filteredNodes.map(obj => ({
+        urls: urls.filter(url => nodeUrlIds.slice(0, arrayLimit).includes(url.id)),
+        nodes: filteredNodes.slice(0, arrayLimit).map(obj => ({
             nodeId: obj.nodeId,
             html: obj.html,
             targets: obj.targets,
             relatedUrlId: obj.relatedUrlId,
             equalified: obj.equalified,
         })),
         messages: Object.values(formattedMessages)
-            .sort((a, b) => a.activeCount > b.activeCount ? -1 : 1)
+            .sort((a, b) => a.activeCount > b.activeCount ? -1 : 1).slice(0, arrayLimit)
             .map(obj => ({
                 ...obj,
                 totalCount: obj.equalifiedCount + obj.activeCount,
             })),
         tags: Object.values(formattedTags),
-        chart: Object.values(formattedChart)
+        chart: Object.values(formattedChart).slice(0, arrayLimit)
             .sort((a, b) => a.date > b.date ? -1 : 1),
     };
+    const compressedBody = gzipSync(JSON.stringify(body));
+    reply.headers({ 'content-encoding': 'gzip' });
+    return reply.send(compressedBody)
 }
diff --git a/src/scheduled/runEveryFifteenMinutes.ts b/src/scheduled/runEveryFifteenMinutes.ts
@@ -5,10 +5,23 @@ const lambda = new LambdaClient();
 export const runEveryFifteenMinutes = async () => {
     await db.connect();
     const pendingScans = (await db.query({
-        text: `SELECT DISTINCT "user_id", "property_id" FROM "scans" WHERE "processing" = true`,
+        text: `SELECT DISTINCT s.user_id, s.property_id, CONCAT(u.first_name,' ',u.last_name) AS user_name, u.email AS user_email, p.name AS property_name, COUNT(s.id) AS "pending_scans" FROM scans AS s 
+            INNER JOIN users AS u ON u.id=s.user_id 
+            INNER JOIN properties AS p ON p.id=s.property_id 
+            WHERE s.processing = true 
+            GROUP BY s.user_id, s.property_id, user_name, user_email, property_name
+            ORDER BY pending_scans DESC`,
     })).rows;
     await db.clean();
+
     console.log(JSON.stringify({ pendingScans }));
+    await fetch(process.env.SLACK_WEBHOOK, {
+        method: 'POST',
+        body: JSON.stringify({
+            text: pendingScans.map(obj => `*${obj.user_name}* (${obj.user_email}) scan for *${obj.property_name}*: ${obj.pending_scans} scans left`).join('\n')
+        })
+    });
+
     for (const { user_id, property_id } of pendingScans) {
         lambda.send(new InvokeCommand({
             FunctionName: `equalify-api${isStaging ? '-staging' : ''}`,