Now using vdom to normalize HTML & compare hashes, add CSV/admin routes

heythisischris · heythisischris · commit 3dba2e6e13ec · 2025-04-03T20:23:04.000-05:00
diff --git a/package.json b/package.json
@@ -21,6 +21,7 @@
     "@fastify/aws-lambda": "^4.1.0",
     "@types/node": "^20.12.7",
     "aws-jwt-verify": "^4.0.1",
+    "cheerio": "^1.0.0",
     "esbuild": "^0.21.5",
     "fastify": "^4.26.2",
     "nodemon": "^3.1.0",
diff --git a/src/app.ts b/src/app.ts
@@ -1,5 +1,5 @@
 import Fastify from 'fastify';
-import { addProperties, addReports, addResults, addScans, deleteProperties, deleteReports, deleteUser, getApikey, getCharts, getFilters, getProperties, getReports, getResultsAll, getResultsMessages, getResultsSchema, getResultsTags, getResultsUrls, getScans, getUpdates, help, trackUser, updateProperties, updateReports } from '#src/routes';
+import { addProperties, addReports, addResults, addScans, adminClearCache, adminProcessScans, deleteProperties, deleteReports, deleteUser, getApikey, getCharts, getFilters, getProperties, getReports, getResultsAll, getResultsCsv, getResultsMessages, getResultsSchema, getResultsTags, getResultsUrls, getScans, getUpdates, help, trackUser, updateProperties, updateReports } from '#src/routes';
 import { CognitoJwtVerifier } from 'aws-jwt-verify';
 import { db } from './utils';
 import { getScan } from './routes/getScan';
@@ -13,9 +13,10 @@ export const jwtClaims = { sub: null };
 
 fastify.addHook('preHandler', async (request, reply) => {
     try {
-        if (request.headers.apikey) {
+        const apikey = request.headers.apikey ?? request.query.apikey;
+        if (apikey) {
             await db.connect();
-            const userId = (await db.query(`SELECT "id" FROM "users" WHERE "apikey"=$1`, [request.headers.apikey])).rows[0].id;
+            const userId = (await db.query(`SELECT "id" FROM "users" WHERE "apikey"=$1`, [apikey])).rows[0].id;
             await db.clean();
             request.headers['x-hasura-user-id'] = userId;
             request.headers['x-hasura-role'] = 'user';
@@ -35,6 +36,7 @@ fastify.addHook('preHandler', async (request, reply) => {
 fastify.get('/get/results', {}, async (request, reply) => getResultsAll({ request, reply }));
 fastify.get('/get/results/schema', {}, async (request, reply) => getResultsSchema({ request, reply }));
 fastify.get('/get/results/all', {}, async (request, reply) => getResultsAll({ request, reply }));
+fastify.get('/get/results/csv', {}, async (request, reply) => getResultsCsv({ request, reply }));
 fastify.get('/get/results/messages', {}, async (request, reply) => getResultsMessages({ request, reply }));
 fastify.get('/get/results/tags', {}, async (request, reply) => getResultsTags({ request, reply }));
 fastify.get('/get/results/urls', {}, async (request, reply) => getResultsUrls({ request, reply }));
@@ -65,6 +67,8 @@ fastify.delete('/delete/user', {}, async (request, reply) => deleteUser({ reques
 // MISC requests
 fastify.post('/help', {}, async (request, reply) => help({ request, reply }));
 fastify.post('/track/user', {}, async (request, reply) => trackUser({ request, reply }));
+fastify.get('/admin/clear-cache', {}, async (request, reply) => adminClearCache({ request, reply }));
+fastify.get('/admin/process-scans', {}, async (request, reply) => adminProcessScans({ request, reply }));
 
 fastify.listen({ port: 3000 }, (err) => {
     console.log(`Server listening on ${fastify.server.address().port}`)
diff --git a/src/internal/processScans.ts b/src/internal/processScans.ts
@@ -1,4 +1,5 @@
 import { chunk, db, isStaging, sleep, hashStringToUuid } from '#src/utils';
+import * as cheerio from 'cheerio';
 
 export const processScans = async (event) => {
     console.log(`START PROCESS SCANS`);
@@ -95,13 +96,16 @@ export const processScans = async (event) => {
     const equalifiedNodeIds = (await db.query({
         text: `SELECT "id" FROM "enodes" WHERE "equalified"=$1 AND "user_id"=$2 AND "url_id"=ANY($3)`,
         values: [false, userId, allPropertyUrlIds],
-    })).rows.filter(obj => ![...allNodeIds, ...failedNodeIds].map(obj => obj.id).includes(obj.id)).map(obj => obj.id);
+    })).rows.map(obj => obj.id).filter(obj => ![...allNodeIds, ...failedNodeIds].map(obj => obj).includes(obj));
+
+    console.log(JSON.stringify({ allPropertyUrlIds, equalifiedNodeIds, allNodeIds, failedNodeIds }));
 
     for (const equalifiedNodeId of equalifiedNodeIds) {
         const existingNodeUpdateId = (await db.query({
             text: `SELECT "id" FROM "enode_updates" WHERE "user_id"=$1 AND "enode_id"=$2 AND "created_at"::text LIKE $3`,
             values: [userId, equalifiedNodeId, `${new Date().toISOString().split('T')[0]}%`],
         })).rows[0]?.id;
+        console.log(JSON.stringify({ equalifiedNodeId }));
         if (existingNodeUpdateId) {
             // We found an existing node update for today, let's simply update it
             await db.query({
@@ -115,12 +119,14 @@ export const processScans = async (event) => {
                 text: `INSERT INTO "enode_updates" ("user_id", "enode_id", "equalified") VALUES ($1, $2, $3)`,
                 values: [userId, equalifiedNodeId, true],
             });
+            console.log(JSON.stringify({ message: 'Inserted equalified node update' }));
         }
         // Now that we've inserted an "equalified" node update, let's set the parent node to "equalified" too!
         await db.query({
             text: `UPDATE "enodes" SET "equalified"=$1 WHERE "id"=$2`,
             values: [true, equalifiedNodeId],
         });
+        console.log(JSON.stringify({ message: 'Update equalified node' }));
     }
 
     // For our failed nodes, we need to "copy" the last node update that exists (if there even is one!)
@@ -129,6 +135,7 @@ export const processScans = async (event) => {
             text: `SELECT "id" FROM "enode_updates" WHERE "user_id"=$1 AND "enode_id"=$2 AND "created_at"::text LIKE $3`,
             values: [userId, failedNodeId, `${new Date().toISOString().split('T')[0]}%`],
         })).rows[0]?.id;
+        console.log(JSON.stringify({ existingNodeUpdateId }))
         if (existingNodeUpdateId) {
             await db.query({
                 text: `UPDATE "enode_updates" SET "equalified"=$1 WHERE "id"=$2`,
@@ -141,12 +148,14 @@ export const processScans = async (event) => {
                 text: `INSERT INTO "enode_updates" ("user_id", "enode_id", "equalified") VALUES ($1, $2, $3)`,
                 values: [userId, failedNodeId, false],
             });
+            console.log(JSON.stringify({ message: 'no node update found!' }))
         }
         // Now that we've inserted an "unequalified" node update, let's set the parent node to "unequalified" too!
         await db.query({
             text: `UPDATE "enodes" SET "equalified"=$1 WHERE "id"=$2`,
             values: [false, failedNodeId],
         });
+        console.log(JSON.stringify({ message: 'updating parent node!' }))
     }
 
     await db.clean();
@@ -170,17 +179,21 @@ const scanProcessor = async ({ result, jobId, userId, propertyId }) => {
                 })).rows?.[0]?.id;
         }
         for (const row of result.nodes) {
+            const normalizedHtml = normalizeHtmlWithVdom(row.html);
+            const htmlHashId = hashStringToUuid(normalizedHtml);
             const existingId = (await db.query({
-                text: `SELECT "id" FROM "enodes" WHERE "user_id"=$1 AND "html"=$2 AND "targets"=$3 AND "url_id"=$4`,
-                values: [userId, row.html, JSON.stringify(row.targets), result.urls.find(obj => obj.urlId === row.relatedUrlId)?.id],
+                text: `SELECT "id" FROM "enodes" WHERE "user_id"=$1 AND "html_hash_id"=$2 AND "url_id"=$3`,
+                values: [userId, htmlHashId, result.urls.find(obj => obj.urlId === row.relatedUrlId)?.id],
             })).rows?.[0]?.id;
 
             row.id = existingId ??
                 (await db.query({
-                    text: `INSERT INTO "enodes" ("user_id", "html", "targets", "url_id", "equalified") VALUES ($1, $2, $3, $4, $5) RETURNING "id"`,
-                    values: [userId, row.html, JSON.stringify(row.targets), result.urls.find(obj => obj.urlId === row.relatedUrlId)?.id, false],
+                    text: `INSERT INTO "enodes" ("user_id", "targets", "html", "html_normalized", "html_hash_id", "url_id", "equalified") VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING "id"`,
+                    values: [userId, JSON.stringify(row.targets), row.html, normalizedHtml, htmlHashId, result.urls.find(obj => obj.urlId === row.relatedUrlId)?.id, false],
                 })).rows?.[0]?.id;
 
+            // We used to compare by targets as well, by something in the scan ocassionally returns different targets!!
+
             const existingNodeUpdateId = (await db.query({
                 text: `SELECT "id" FROM "enode_updates" WHERE "user_id"=$1 AND "enode_id"=$2 AND "created_at"::text LIKE $3`,
                 values: [userId, row.id, `${new Date().toISOString().split('T')[0]}%`],
@@ -269,4 +282,75 @@ const scanProcessor = async ({ result, jobId, userId, propertyId }) => {
     });
 
     return result.nodes.map(obj => obj.id);
-}
+}
+
+const normalizeHtmlWithRegex = (html) => {
+    if (!html) return html;
+
+    // Remove query parameters from src and href attributes
+    html = html.replace(/(src|href)=(["'])([^?"']*)\?[^"']*?\2/g, '$1=$2$3$2');
+
+    // Normalize data-version paths with version hashes
+    html = html.replace(/data-version=(["'])(\/s\/player\/)[a-zA-Z0-9]{8,}(\/.*?\2)/g, 'data-version=$1$2NORMALIZED$3');
+
+    // Normalize tabindex attributes
+    html = html.replace(/tabindex=["']-?\d+["']/g, 'tabindex="NORMALIZED"');
+
+    return html;
+};
+
+const normalizeHtmlWithVdom = (html) => {
+    if (!html) return '';
+
+    const $ = cheerio.load(`<div id="wrapper">${html}</div>`);
+    const root = $('#wrapper');
+
+    // Process all elements
+    root.find('*').each(function () {
+        const el = $(this);
+
+        // Normalize IDs with numbers
+        if (el.attr('id') && /\d{4,}/.test(el.attr('id'))) {
+            el.attr('id', 'NORMALIZED');
+        }
+
+        // Always normalize tabindex
+        if (el.attr('tabindex')) {
+            el.attr('tabindex', 'NORMALIZED');
+        }
+
+        // Remove query params from URLs
+        ['src', 'href'].forEach(attr => {
+            if (el.attr(attr) && el.attr(attr).includes('?')) {
+                el.attr(attr, el.attr(attr).split('?')[0]);
+            }
+        });
+
+        // Handle h5p quiz elements
+        if (el.hasClass('h5p-sc-alternative')) {
+            if (el.hasClass('h5p-sc-is-correct') || el.hasClass('h5p-sc-is-wrong')) {
+                el.removeClass('h5p-sc-is-correct h5p-sc-is-wrong')
+                    .addClass('h5p-sc-is-NORMALIZED');
+            }
+        }
+
+        // Normalize data-version attributes
+        if (el.attr('data-version') && el.attr('data-version').includes('/s/player/')) {
+            el.attr('data-version', el.attr('data-version')
+                .replace(/\/s\/player\/[a-zA-Z0-9]{8,}\//, '/s/player/NORMALIZED/'));
+        }
+
+        // Add more element-specific normalizations based on your data patterns
+    });
+
+    // Remove all whitespace between tags for more reliable comparison
+    let result = root.html();
+
+    // Remove excess whitespace for more consistent matching
+    result = result.replace(/>\s+</g, '><');
+
+    // Remove all text node whitespace variations
+    result = result.replace(/\s{2,}/g, ' ');
+
+    return result;
+}
diff --git a/src/routes/adminClearCache.ts b/src/routes/adminClearCache.ts
@@ -0,0 +1,18 @@
+import { jwtClaims } from '#src/app';
+import { db } from '#src/utils';
+
+export const adminClearCache = async ({ request, reply }) => {
+    const adminIds = JSON.parse(process.env.ADMIN_IDS);
+    if (adminIds.includes(jwtClaims.sub)) {
+        await db.connect();
+        await db.query({
+            text: `UPDATE "reports" SET "cache_date"=$1 WHERE "user_id"=$2`,
+            values: ['2025-01-01', jwtClaims.sub],
+        })
+        await db.clean();
+        return { success: true };
+    }
+    else {
+        return { success: false };
+    }
+}
diff --git a/src/routes/adminProcessScans.ts b/src/routes/adminProcessScans.ts
@@ -0,0 +1,13 @@
+import { jwtClaims } from '#src/app';
+import { runEveryFifteenMinutes } from '#src/scheduled/runEveryFifteenMinutes';
+
+export const adminProcessScans = async ({ request, reply }) => {
+    const adminIds = JSON.parse(process.env.ADMIN_IDS);
+    if (adminIds.includes(jwtClaims.sub)) {
+        await runEveryFifteenMinutes();
+        return { success: true };
+    }
+    else {
+        return { success: false };
+    }
+}
diff --git a/src/routes/getApikey.ts b/src/routes/getApikey.ts
@@ -3,7 +3,8 @@ import { db, formatId } from '#src/utils';
 
 export const getApikey = async ({ request, reply }) => {
     await db.connect();
-    const apikey = (await db.query(`SELECT "apikey" FROM "users" WHERE "id" = $1`, [jwtClaims.sub])).rows?.[0]?.apikey;
+    const { id, apikey } = (await db.query(`SELECT "id", "apikey" FROM "users" WHERE "id" = $1`, [jwtClaims.sub])).rows?.[0];
     await db.clean();
-    return { apikey: formatId(apikey) };
+    const adminIds = JSON.parse(process.env.ADMIN_IDS);
+    return { apikey: formatId(apikey), isAdmin: adminIds.includes(id) };
 }
diff --git a/src/routes/getResultsCsv.ts b/src/routes/getResultsCsv.ts
diff --git a/src/routes/index.ts b/src/routes/index.ts
diff --git a/yarn.lock b/yarn.lock
