Sorting scans, reorganized processScans, adding messageNodes/messageTags

EqualifyEverything · Jul 2, 2024 · 8f76138 · 8f76138
1 parent f55f44a
commit 8f76138
Show file tree

Hide file tree

Showing 2 changed files with 62 additions and 63 deletions.
diff --git a/src/routes/getScans.ts b/src/routes/getScans.ts
@@ -3,7 +3,7 @@ import { graphqlQuery } from '#src/utils';
 export const getScans = async ({ request, reply }) => {
     const response = (await graphqlQuery({
         query: `query($first: Int, $offset: Int){
-            scans: scansConnection(first: $first, offset: $offset, ${(request.query.scanIds) ? `filter: { id: {in: [
+            scans: scansConnection(first: $first, offset: $offset, orderBy: CREATED_AT_DESC, ${(request.query.scanIds) ? `filter: { id: {in: [
                         ${request.query.scanIds.split(',').map(obj => `"${obj}"`).join()}
                     ]}}` : ''}
                 ) { 
@@ -12,6 +12,7 @@ export const getScans = async ({ request, reply }) => {
                     createdAt
                     processing
                     jobId
+                    results
                     property {
                         id
                         name

diff --git a/src/scheduled/processScans.ts b/src/scheduled/processScans.ts
@@ -34,69 +34,67 @@ const scanProcessor = async ({ result, scan }) => {
     await db.query(`UPDATE "scans" SET "processing"=FALSE, "results"=$1`, [result]);
 
     // 2. Find existing IDs for urls, messages, tags, & nodes (or create them)
-    await Promise.allSettled([
-        new Promise(async (res) => {
-            for (const row of result.urls) {
-                row.id =
-                    (await db.query({
-                        text: `SELECT "id" FROM "urls" WHERE "user_id"=$1 AND "url"=$2 AND "property_id"=$3`,
-                        values: [jwtClaims.sub, row.url, scan.property_id],
-                    })).rows?.[0]?.id
-                    ??
-                    (await db.query({
-                        text: `INSERT INTO "urls" ("user_id", "url", "property_id") VALUES ($1, $2, $3) RETURNING "id"`,
-                        values: [jwtClaims.sub, row.url, scan.property_id]
-                    })).rows?.[0]?.id;
-            }
-            res(1);
-        }),
-        new Promise(async (res) => {
-            for (const row of result.messages) {
-                row.id =
-                    (await db.query({
-                        text: `SELECT "id" FROM "messages" WHERE "user_id"=$1 AND "message"=$2 AND "type"=$3`,
-                        values: [jwtClaims.sub, row.message, row.type],
-                    })).rows?.[0]?.id
-                    ??
-                    (await db.query({
-                        text: `INSERT INTO "messages" ("user_id", "message", "type") VALUES ($1, $2, $3) RETURNING "id"`,
-                        values: [jwtClaims.sub, row.message, row.type],
-                    })).rows?.[0]?.id;
-            }
-            res(1);
-        }),
-        new Promise(async (res) => {
-            for (const row of result.tags) {
-                row.id =
-                    (await db.query({
-                        text: `SELECT "id" FROM "tags" WHERE "user_id"=$1 AND "tag"=$2`,
-                        values: [jwtClaims.sub, row.tag],
-                    })).rows?.[0]?.id
-                    ??
-                    (await db.query({
-                        text: `INSERT INTO "tags" ("user_id", "tag") VALUES ($1, $2) RETURNING "id"`,
-                        values: [jwtClaims.sub, row.tag],
-                    })).rows?.[0]?.id;
-            }
-            res(1);
-        }),
-        new Promise(async (res) => {
-            for (const row of result.nodes) {
-                row.id =
-                    (await db.query({
-                        text: `SELECT "id" FROM "enodes" WHERE "user_id"=$1 AND "html"=$2 AND "targets"=$3 AND "url_id"=$4`,
-                        values: [jwtClaims.sub, row.html, JSON.stringify(row.targets), row.url_id],
-                    })).rows?.[0]?.id
-                    ??
-                    (await db.query({
-                        text: `INSERT INTO "enodes" ("user_id", "html", "targets", "url_id") VALUES ($1, $2, $3, $4) RETURNING "id"`,
-                        values: [jwtClaims.sub, row.html, JSON.stringify(row.targets), row.url_id],
-                    })).rows?.[0]?.id;
-            }
-            res(1);
-        }),
-    ])
+    for (const row of result.urls) {
+        row.id =
+            (await db.query({
+                text: `SELECT "id" FROM "urls" WHERE "user_id"=$1 AND "url"=$2 AND "property_id"=$3`,
+                values: [jwtClaims.sub, row.url, scan.property_id],
+            })).rows?.[0]?.id
+            ??
+            (await db.query({
+                text: `INSERT INTO "urls" ("user_id", "url", "property_id") VALUES ($1, $2, $3) RETURNING "id"`,
+                values: [jwtClaims.sub, row.url, scan.property_id]
+            })).rows?.[0]?.id;
+    }
+    for (const row of result.nodes) {
+        row.id =
+            (await db.query({
+                text: `SELECT "id" FROM "enodes" WHERE "user_id"=$1 AND "html"=$2 AND "targets"=$3 AND "url_id"=$4`,
+                values: [jwtClaims.sub, row.html, JSON.stringify(row.targets), row.url_id],
+            })).rows?.[0]?.id
+            ??
+            (await db.query({
+                text: `INSERT INTO "enodes" ("user_id", "html", "targets", "url_id") VALUES ($1, $2, $3, $4) RETURNING "id"`,
+                values: [jwtClaims.sub, row.html, JSON.stringify(row.targets), result.urls.find(obj => obj.urlId === row.relatedUrlId).id],
+            })).rows?.[0]?.id;
+    }
+    for (const row of result.tags) {
+        row.id =
+            (await db.query({
+                text: `SELECT "id" FROM "tags" WHERE "user_id"=$1 AND "tag"=$2`,
+                values: [jwtClaims.sub, row.tag],
+            })).rows?.[0]?.id
+            ??
+            (await db.query({
+                text: `INSERT INTO "tags" ("user_id", "tag") VALUES ($1, $2) RETURNING "id"`,
+                values: [jwtClaims.sub, row.tag],
+            })).rows?.[0]?.id;
+    }
+    for (const row of result.messages) {
+        row.id =
+            (await db.query({
+                text: `SELECT "id" FROM "messages" WHERE "user_id"=$1 AND "message"=$2 AND "type"=$3`,
+                values: [jwtClaims.sub, row.message, row.type],
+            })).rows?.[0]?.id
+            ??
+            (await db.query({
+                text: `INSERT INTO "messages" ("user_id", "message", "type") VALUES ($1, $2, $3) RETURNING "id"`,
+                values: [jwtClaims.sub, row.message, row.type],
+            })).rows?.[0]?.id;
 
+        for (const relatedNodeId of row.relatedNodeIds) {
+            await db.query({
+                text: `INSERT INTO "message_nodes" ("user_id", "message_id", "enode_id") VALUES ($1, $2, $3)`,
+                values: [jwtClaims.sub, row.id, result.nodes.find(obj => obj.nodeId === relatedNodeId).id]
+            })
+        }
+        for (const relatedTagId of row.relatedTagIds) {
+            await db.query({
+                text: `INSERT INTO "message_tags" ("user_id", "message_id", "tag_id") VALUES ($1, $2, $3)`,
+                values: [jwtClaims.sub, row.id, result.tags.find(obj => obj.tagId === relatedTagId).id]
+            })
+        }
+    }
     // 3. Compare & update nodes
 
     // 4. Delete the scan and move on!