diff --git a/src/routes/getResultsAll.ts b/src/routes/getResultsAll.ts index ba4dad5..658ef59 100644 --- a/src/routes/getResultsAll.ts +++ b/src/routes/getResultsAll.ts @@ -89,9 +89,11 @@ export const getResultsAll = async ({ request, reply }) => { formattedChart[date][node.equalified ? 'equalified' : 'active'] += 1; } + const nodeUrlIds = [...new Set(response.data.nodes.map(obj => obj.relatedUrlId))]; + return { reportName: report.name, - urls: urls, + urls: urls.filter(url => nodeUrlIds.includes(url.id)), nodes: response.data.nodes.map(obj => ({ nodeId: obj.nodeId, html: obj.html, diff --git a/src/scheduled/processScans.ts b/src/scheduled/processScans.ts index b79b7a7..304cee4 100644 --- a/src/scheduled/processScans.ts +++ b/src/scheduled/processScans.ts @@ -7,7 +7,11 @@ export const processScans = async () => { console.log(scans); await Promise.allSettled(scans.map(scan => new Promise(async (res) => { try { - const { result, status } = await (await fetch(`https://scan.equalify.app/results/${scan.job_id}`)).json(); + const scanResultsText = await fetch(`https://scan.equalify.app/results/${scan.job_id}`); + const scanResultsTextProcessed = await scanResultsText.text(); + console.log(JSON.stringify({ scanResultsTextProcessed })); + const scanResults = await fetch(`https://scan.equalify.app/results/${scan.job_id}`); + const { result, status } = await scanResults.json(); if (['delayed', 'active', 'waiting'].includes(status)) { console.log(`Scan ${scan.id} is ${status}- scan skipped.`); } @@ -34,68 +38,69 @@ const scanProcessor = async ({ result, scan }) => { await db.query(`UPDATE "scans" SET "processing"=FALSE, "results"=$1`, [result]); // 2. Find existing IDs for urls, messages, tags, & nodes (or create them) - for (const row of result.urls) { - row.id = - (await db.query({ - text: `SELECT "id" FROM "urls" WHERE "user_id"=$1 AND "url"=$2 AND "property_id"=$3`, - values: [scan.user_id, row.url, scan.property_id], - })).rows?.[0]?.id - ?? - (await db.query({ - text: `INSERT INTO "urls" ("user_id", "url", "property_id") VALUES ($1, $2, $3) RETURNING "id"`, - values: [scan.user_id, row.url, scan.property_id] - })).rows?.[0]?.id; - } - for (const row of result.nodes) { - row.id = - (await db.query({ - text: `SELECT "id" FROM "enodes" WHERE "user_id"=$1 AND "html"=$2 AND "targets"=$3 AND "url_id"=$4`, - values: [scan.user_id, row.html, JSON.stringify(row.targets), row.url_id], - })).rows?.[0]?.id - ?? - (await db.query({ - text: `INSERT INTO "enodes" ("user_id", "html", "targets", "url_id") VALUES ($1, $2, $3, $4) RETURNING "id"`, - values: [scan.user_id, row.html, JSON.stringify(row.targets), result.urls.find(obj => obj.urlId === row.relatedUrlId).id], - })).rows?.[0]?.id; - } - for (const row of result.tags) { - row.id = - (await db.query({ - text: `SELECT "id" FROM "tags" WHERE "user_id"=$1 AND "tag"=$2`, - values: [scan.user_id, row.tag], - })).rows?.[0]?.id - ?? - (await db.query({ - text: `INSERT INTO "tags" ("user_id", "tag") VALUES ($1, $2) RETURNING "id"`, - values: [scan.user_id, row.tag], - })).rows?.[0]?.id; - } - for (const row of result.messages) { - row.id = - (await db.query({ - text: `SELECT "id" FROM "messages" WHERE "user_id"=$1 AND "message"=$2 AND "type"=$3`, - values: [scan.user_id, row.message, row.type], - })).rows?.[0]?.id - ?? - (await db.query({ - text: `INSERT INTO "messages" ("user_id", "message", "type") VALUES ($1, $2, $3) RETURNING "id"`, - values: [scan.user_id, row.message, row.type], - })).rows?.[0]?.id; - - for (const relatedNodeId of row.relatedNodeIds) { - await db.query({ - text: `INSERT INTO "message_nodes" ("user_id", "message_id", "enode_id") VALUES ($1, $2, $3)`, - values: [scan.user_id, row.id, result.nodes.find(obj => obj.nodeId === relatedNodeId).id] - }) + if (result.nodes.length > 0) { + for (const row of result.urls) { + row.id = + (await db.query({ + text: `SELECT "id" FROM "urls" WHERE "user_id"=$1 AND "url"=$2 AND "property_id"=$3`, + values: [scan.user_id, row.url, scan.property_id], + })).rows?.[0]?.id + ?? + (await db.query({ + text: `INSERT INTO "urls" ("user_id", "url", "property_id") VALUES ($1, $2, $3) RETURNING "id"`, + values: [scan.user_id, row.url, scan.property_id] + })).rows?.[0]?.id; + } + for (const row of result.nodes) { + row.id = + (await db.query({ + text: `SELECT "id" FROM "enodes" WHERE "user_id"=$1 AND "html"=$2 AND "targets"=$3 AND "url_id"=$4`, + values: [scan.user_id, row.html, JSON.stringify(row.targets), row.url_id], + })).rows?.[0]?.id + ?? + (await db.query({ + text: `INSERT INTO "enodes" ("user_id", "html", "targets", "url_id") VALUES ($1, $2, $3, $4) RETURNING "id"`, + values: [scan.user_id, row.html, JSON.stringify(row.targets), result.urls.find(obj => obj.urlId === row.relatedUrlId).id], + })).rows?.[0]?.id; } - for (const relatedTagId of row.relatedTagIds) { - await db.query({ - text: `INSERT INTO "message_tags" ("user_id", "message_id", "tag_id") VALUES ($1, $2, $3)`, - values: [scan.user_id, row.id, result.tags.find(obj => obj.tagId === relatedTagId).id] - }) + for (const row of result.tags) { + row.id = + (await db.query({ + text: `SELECT "id" FROM "tags" WHERE "user_id"=$1 AND "tag"=$2`, + values: [scan.user_id, row.tag], + })).rows?.[0]?.id + ?? + (await db.query({ + text: `INSERT INTO "tags" ("user_id", "tag") VALUES ($1, $2) RETURNING "id"`, + values: [scan.user_id, row.tag], + })).rows?.[0]?.id; + } + for (const row of result.messages) { + row.id = + (await db.query({ + text: `SELECT "id" FROM "messages" WHERE "user_id"=$1 AND "message"=$2 AND "type"=$3`, + values: [scan.user_id, row.message, row.type], + })).rows?.[0]?.id + ?? + (await db.query({ + text: `INSERT INTO "messages" ("user_id", "message", "type") VALUES ($1, $2, $3) RETURNING "id"`, + values: [scan.user_id, row.message, row.type], + })).rows?.[0]?.id; + + for (const relatedNodeId of row.relatedNodeIds) { + await db.query({ + text: `INSERT INTO "message_nodes" ("user_id", "message_id", "enode_id") VALUES ($1, $2, $3)`, + values: [scan.user_id, row.id, result.nodes.find(obj => obj.nodeId === relatedNodeId).id] + }) + } + for (const relatedTagId of row.relatedTagIds) { + await db.query({ + text: `INSERT INTO "message_tags" ("user_id", "message_id", "tag_id") VALUES ($1, $2, $3)`, + values: [scan.user_id, row.id, result.tags.find(obj => obj.tagId === relatedTagId).id] + }) + } } } - // 3. Compare & update nodes // 4. Delete the scan and move on! // await db.query(`DELETE FROM "scans" WHERE "id"=$1`, [scan.id]);