Skip to content

Commit

Permalink
Only returning URLs that have any nodes to prevent empty URL filters
Browse files Browse the repository at this point in the history
  • Loading branch information
@heythisischris
heythisischris committed Jul 31, 2024
1 parent 38f23da commit 91aae0e
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 61 deletions.
4 changes: 3 additions & 1 deletion src/routes/getResultsAll.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,11 @@ export const getResultsAll = async ({ request, reply }) => {
formattedChart[date][node.equalified ? 'equalified' : 'active'] += 1;
}

const nodeUrlIds = [...new Set(response.data.nodes.map(obj => obj.relatedUrlId))];

return {
reportName: report.name,
urls: urls,
urls: urls.filter(url => nodeUrlIds.includes(url.id)),
nodes: response.data.nodes.map(obj => ({
nodeId: obj.nodeId,
html: obj.html,
Expand Down
125 changes: 65 additions & 60 deletions src/scheduled/processScans.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@ export const processScans = async () => {
console.log(scans);
await Promise.allSettled(scans.map(scan => new Promise(async (res) => {
try {
const { result, status } = await (await fetch(`https://scan.equalify.app/results/${scan.job_id}`)).json();
const scanResultsText = await fetch(`https://scan.equalify.app/results/${scan.job_id}`);
const scanResultsTextProcessed = await scanResultsText.text();
console.log(JSON.stringify({ scanResultsTextProcessed }));
const scanResults = await fetch(`https://scan.equalify.app/results/${scan.job_id}`);
const { result, status } = await scanResults.json();
if (['delayed', 'active', 'waiting'].includes(status)) {
console.log(`Scan ${scan.id} is ${status}- scan skipped.`);
}
Expand All @@ -34,68 +38,69 @@ const scanProcessor = async ({ result, scan }) => {
await db.query(`UPDATE "scans" SET "processing"=FALSE, "results"=$1`, [result]);

// 2. Find existing IDs for urls, messages, tags, & nodes (or create them)
for (const row of result.urls) {
row.id =
(await db.query({
text: `SELECT "id" FROM "urls" WHERE "user_id"=$1 AND "url"=$2 AND "property_id"=$3`,
values: [scan.user_id, row.url, scan.property_id],
})).rows?.[0]?.id
??
(await db.query({
text: `INSERT INTO "urls" ("user_id", "url", "property_id") VALUES ($1, $2, $3) RETURNING "id"`,
values: [scan.user_id, row.url, scan.property_id]
})).rows?.[0]?.id;
}
for (const row of result.nodes) {
row.id =
(await db.query({
text: `SELECT "id" FROM "enodes" WHERE "user_id"=$1 AND "html"=$2 AND "targets"=$3 AND "url_id"=$4`,
values: [scan.user_id, row.html, JSON.stringify(row.targets), row.url_id],
})).rows?.[0]?.id
??
(await db.query({
text: `INSERT INTO "enodes" ("user_id", "html", "targets", "url_id") VALUES ($1, $2, $3, $4) RETURNING "id"`,
values: [scan.user_id, row.html, JSON.stringify(row.targets), result.urls.find(obj => obj.urlId === row.relatedUrlId).id],
})).rows?.[0]?.id;
}
for (const row of result.tags) {
row.id =
(await db.query({
text: `SELECT "id" FROM "tags" WHERE "user_id"=$1 AND "tag"=$2`,
values: [scan.user_id, row.tag],
})).rows?.[0]?.id
??
(await db.query({
text: `INSERT INTO "tags" ("user_id", "tag") VALUES ($1, $2) RETURNING "id"`,
values: [scan.user_id, row.tag],
})).rows?.[0]?.id;
}
for (const row of result.messages) {
row.id =
(await db.query({
text: `SELECT "id" FROM "messages" WHERE "user_id"=$1 AND "message"=$2 AND "type"=$3`,
values: [scan.user_id, row.message, row.type],
})).rows?.[0]?.id
??
(await db.query({
text: `INSERT INTO "messages" ("user_id", "message", "type") VALUES ($1, $2, $3) RETURNING "id"`,
values: [scan.user_id, row.message, row.type],
})).rows?.[0]?.id;

for (const relatedNodeId of row.relatedNodeIds) {
await db.query({
text: `INSERT INTO "message_nodes" ("user_id", "message_id", "enode_id") VALUES ($1, $2, $3)`,
values: [scan.user_id, row.id, result.nodes.find(obj => obj.nodeId === relatedNodeId).id]
})
if (result.nodes.length > 0) {
for (const row of result.urls) {
row.id =
(await db.query({
text: `SELECT "id" FROM "urls" WHERE "user_id"=$1 AND "url"=$2 AND "property_id"=$3`,
values: [scan.user_id, row.url, scan.property_id],
})).rows?.[0]?.id
??
(await db.query({
text: `INSERT INTO "urls" ("user_id", "url", "property_id") VALUES ($1, $2, $3) RETURNING "id"`,
values: [scan.user_id, row.url, scan.property_id]
})).rows?.[0]?.id;
}
for (const row of result.nodes) {
row.id =
(await db.query({
text: `SELECT "id" FROM "enodes" WHERE "user_id"=$1 AND "html"=$2 AND "targets"=$3 AND "url_id"=$4`,
values: [scan.user_id, row.html, JSON.stringify(row.targets), row.url_id],
})).rows?.[0]?.id
??
(await db.query({
text: `INSERT INTO "enodes" ("user_id", "html", "targets", "url_id") VALUES ($1, $2, $3, $4) RETURNING "id"`,
values: [scan.user_id, row.html, JSON.stringify(row.targets), result.urls.find(obj => obj.urlId === row.relatedUrlId).id],
})).rows?.[0]?.id;
}
for (const relatedTagId of row.relatedTagIds) {
await db.query({
text: `INSERT INTO "message_tags" ("user_id", "message_id", "tag_id") VALUES ($1, $2, $3)`,
values: [scan.user_id, row.id, result.tags.find(obj => obj.tagId === relatedTagId).id]
})
for (const row of result.tags) {
row.id =
(await db.query({
text: `SELECT "id" FROM "tags" WHERE "user_id"=$1 AND "tag"=$2`,
values: [scan.user_id, row.tag],
})).rows?.[0]?.id
??
(await db.query({
text: `INSERT INTO "tags" ("user_id", "tag") VALUES ($1, $2) RETURNING "id"`,
values: [scan.user_id, row.tag],
})).rows?.[0]?.id;
}
for (const row of result.messages) {
row.id =
(await db.query({
text: `SELECT "id" FROM "messages" WHERE "user_id"=$1 AND "message"=$2 AND "type"=$3`,
values: [scan.user_id, row.message, row.type],
})).rows?.[0]?.id
??
(await db.query({
text: `INSERT INTO "messages" ("user_id", "message", "type") VALUES ($1, $2, $3) RETURNING "id"`,
values: [scan.user_id, row.message, row.type],
})).rows?.[0]?.id;

for (const relatedNodeId of row.relatedNodeIds) {
await db.query({
text: `INSERT INTO "message_nodes" ("user_id", "message_id", "enode_id") VALUES ($1, $2, $3)`,
values: [scan.user_id, row.id, result.nodes.find(obj => obj.nodeId === relatedNodeId).id]
})
}
for (const relatedTagId of row.relatedTagIds) {
await db.query({
text: `INSERT INTO "message_tags" ("user_id", "message_id", "tag_id") VALUES ($1, $2, $3)`,
values: [scan.user_id, row.id, result.tags.find(obj => obj.tagId === relatedTagId).id]
})
}
}
}
// 3. Compare & update nodes

// 4. Delete the scan and move on!
// await db.query(`DELETE FROM "scans" WHERE "id"=$1`, [scan.id]);
Expand Down

0 comments on commit 91aae0e

Please sign in to comment.