From 2be613df7383fc2a701006c46fc5849b112c2010 Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Tue, 12 Sep 2023 11:39:40 -0700 Subject: [PATCH] Usage metrics for v2023.4 (#964) * Stubs for 2023.4 analytics * Updating analytics queries and tests * Change part of unprocessed query * Minor changes to queries * Check sso for user metrics --- config/default.json | 2 +- lib/data/analytics.js | 9 +- lib/model/query/analytics.js | 61 ++++- test/integration/other/analytics-queries.js | 246 ++++++++++++++++++-- 4 files changed, 297 insertions(+), 21 deletions(-) diff --git a/config/default.json b/config/default.json index cc0b73bf6..aa315ed36 100644 --- a/config/default.json +++ b/config/default.json @@ -30,7 +30,7 @@ "analytics": { "url": "https://data.getodk.cloud/v1/key/eOZ7S4bzyUW!g1PF6dIXsnSqktRuewzLTpmc6ipBtRq$LDfIMTUKswCexvE0UwJ9/projects/1/submission", "formId": "odk-analytics", - "version": "2023.05.22.01" + "version": "v2023.4.0_1" } } }, diff --git a/lib/data/analytics.js b/lib/data/analytics.js index bd94a6496..4612ec320 100644 --- a/lib/data/analytics.js +++ b/lib/data/analytics.js @@ -30,7 +30,14 @@ const metricsTemplate = { "num_unique_viewers": {}, "num_unique_collectors": {}, "database_size": {}, - "uses_external_db": 0 + "uses_external_db": 0, + "sso_enabled": 0, + "num_client_audit_attachments": 0, + "num_client_audit_attachments_failures": 0, + "num_client_audit_rows": 0, + "num_audits_failed": 0, + "num_audits_failed5": 0, + "num_audits_unprocessed": 0 }, "projects": [ { diff --git a/lib/model/query/analytics.js b/lib/model/query/analytics.js index 38e90390f..202ee31b4 100644 --- a/lib/model/query/analytics.js +++ b/lib/model/query/analytics.js @@ -11,6 +11,7 @@ const config = require('config'); const { sql } = require('slonik'); const { clone } = require('ramda'); const { metricsTemplate } = require('../../data/analytics'); +const oidc = require('../../util/oidc'); const DAY_RANGE = 45; const _cutoffDate = sql`current_date - cast(${DAY_RANGE} as int)`; @@ -38,12 +39,21 @@ const databaseExternal = (dbHost) => () => // GENERAL const auditLogs = () => ({ one }) => one(sql` -select count(*) as total, +SELECT + -- Total count of audit logs + count(*) AS total, + -- Count of recent audit logs only count(CASE WHEN "loggedAt" >= ${_cutoffDate} THEN 1 ELSE null - END) AS "recent" -from audits`); + END) AS recent, + -- Any failure, even if ultimately processed + count(*) FILTER (WHERE failures > 0) AS any_failure, + -- Completely failed + count(*) FILTER (WHERE processed IS NULL AND failures >= 5) AS failed5, + -- Unexpectedly unprocessed + count(*) FILTER (WHERE processed IS NULL AND failures < 5 AND "loggedAt" < now() - INTERVAL '1 day') AS unprocessed +FROM audits`); const countAdmins = () => ({ one }) => one(sql` select count(u."actorId") as total, count(activeUsers."actorId") as recent @@ -106,6 +116,28 @@ where r."system" = 'formfill'`); const databaseSize = () => ({ one }) => one(sql` select pg_database_size(current_database()) as database_size`); +const countClientAuditAttachments = () => ({ oneFirst }) => oneFirst(sql` +SELECT count(*) +FROM submission_attachments +WHERE "isClientAudit" AND "blobId" IS NOT NULL`); + +const countClientAuditProcessingFailed = () => ({ oneFirst }) => oneFirst(sql` +SELECT count(*) +FROM audits +JOIN submission_attachments ON + submission_attachments."submissionDefId" = (audits.details->'submissionDefId')::INTEGER AND + submission_attachments.name = audits.details->>'name' +WHERE + audits.action = 'submission.attachment.update' AND + audits.processed IS NULL AND + audits.failures >= 5 AND + submission_attachments."isClientAudit" + -- Intentionally not filtering on submission_attachments."blobId" + -- on the off-chance that a failing attachment was cleared.`); + +const countClientAuditRows = () => ({ oneFirst }) => oneFirst(sql` +SELECT count(*) FROM client_audits`); + // PER PROJECT // Users const countUsersPerRole = () => ({ all }) => all(sql` @@ -277,7 +309,7 @@ inner join ( group by (xml_form_id, proj_id) ) as deleted_form_ids on forms."xmlFormId" = deleted_form_ids."xml_form_id" and - forms."projectId" = deleted_form_ids."proj_id" + forms."projectId" = deleted_form_ids."proj_id" where forms."deletedAt" is null group by forms."projectId"`); @@ -383,7 +415,7 @@ FROM datasets ds JOIN forms f ON f.id = fa."formId" ) ON fa."datasetId" = ds.id LEFT JOIN ( - SELECT + SELECT COUNT (a.details -> 'submissionId'::TEXT) total, SUM (CASE WHEN a."loggedAt" >= current_date - cast(${DAY_RANGE} as int) THEN 1 ELSE 0 END) recent, dfd."datasetId" @@ -575,9 +607,14 @@ const previewMetrics = () => (({ Analytics }) => Promise.all([ Analytics.countUniqueManagers(), Analytics.countUniqueViewers(), Analytics.countUniqueDataCollectors(), + Analytics.countClientAuditAttachments(), + Analytics.countClientAuditProcessingFailed(), + Analytics.countClientAuditRows(), Analytics.projectMetrics() ]).then(([db, encrypt, bigForm, admins, audits, - archived, managers, viewers, collectors, projMetrics]) => { + archived, managers, viewers, collectors, + caAttachments, caFailures, caRows, + projMetrics]) => { const metrics = clone(metricsTemplate); // system for (const [key, value] of Object.entries(db)) @@ -600,6 +637,15 @@ const previewMetrics = () => (({ Analytics }) => Promise.all([ metrics.system.uses_external_db = Analytics.databaseExternal(config.get('default.database.host')); + // 2023.4.0 metrics + metrics.system.num_client_audit_attachments = caAttachments; + metrics.system.num_client_audit_attachments_failures = caFailures; + metrics.system.num_client_audit_rows = caRows; + metrics.system.num_audits_failed = audits.any_failure; + metrics.system.num_audits_failed5 = audits.failed5; + metrics.system.num_audits_unprocessed = audits.unprocessed; + metrics.system.sso_enabled = oidc.isEnabled() ? 1 : 0; + return metrics; })); @@ -626,6 +672,9 @@ module.exports = { countAdmins, countAppUsers, countDeviceIds, + countClientAuditAttachments, + countClientAuditProcessingFailed, + countClientAuditRows, countForms, countFormsEncrypted, countFormFieldTypes, diff --git a/test/integration/other/analytics-queries.js b/test/integration/other/analytics-queries.js index 5c3ea9a85..344be5f1f 100644 --- a/test/integration/other/analytics-queries.js +++ b/test/integration/other/analytics-queries.js @@ -1,9 +1,11 @@ const appRoot = require('app-root-path'); const { sql } = require('slonik'); const { testService, testContainer } = require('../setup'); -const { createReadStream } = require('fs'); +const { createReadStream, readFileSync } = require('fs'); + +const { promisify } = require('util'); const testData = require('../../data/xml'); -const { exhaust } = require(appRoot + '/lib/worker/worker'); +const { runner, exhaust } = require(appRoot + '/lib/worker/worker'); const geoForm = ` @@ -216,6 +218,189 @@ describe('analytics task queries', function () { Analytics.databaseExternal(undefined).should.equal(1); Analytics.databaseExternal(null).should.equal(1); })); + + describe('counting client audits', () => { + it('should count the total number of client audit submission attachments', testService(async (service, { Analytics }) => { + const asAlice = await service.login('alice'); + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.clientAudits) + .expect(200); + + // the one sub with good client audit attachment + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('audit.csv', createReadStream(appRoot + '/test/data/audit.csv'), { filename: 'audit.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.one), { filename: 'data.xml' }) + .expect(201); + + // client audit attachment is missing + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' }) + .expect(201); + + // another attachment that is not a client audit + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.binaryType) + .expect(200) + .then(() => asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('xml_submission_file', Buffer.from(testData.instances.binaryType.one), { filename: 'data.xml' }) + .attach('my_file1.mp4', createReadStream(appRoot + '/test/data/audit.csv'), { filename: 'my_file1.mp4' }) + .expect(201)); + + const res = await Analytics.countClientAuditAttachments(); + res.should.equal(1); + })); + + it('should count client audit attachments that failed processing', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + // encrypt default project and send one encrypted client audit attachment + const { extractPubkey, extractVersion, sendEncrypted } = require(appRoot + '/test/util/crypto-odk'); + await asAlice.post('/v1/projects/1/key') + .send({ passphrase: 'supersecret', hint: 'it is a secret' }) + .expect(200) + .then(() => asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.clientAudits) + .expect(200)) + .then(() => asAlice.get('/v1/projects/1/forms/audits.xml') + .expect(200) + .then(({ text }) => sendEncrypted(asAlice, extractVersion(text), extractPubkey(text))) + .then((send) => send(testData.instances.clientAudits.one, { 'audit.csv.enc': readFileSync(appRoot + '/test/data/audit.csv') }))); + + await exhaust(container); + + // at this point there should be 0 failed + await container.Analytics.countClientAuditProcessingFailed() + .then((res) => res.should.equal(0)); + + // but there will be 0 rows in the clients audits table bc encrypted ones dont get extracted + await container.Analytics.countClientAuditRows() + .then((res) => res.should.equal(0)); + + // make a new project to not encrypt + const newProjectId = await asAlice.post('/v1/projects') + .set('Content-Type', 'application/json') + .send({ name: 'Test Project' }) + .expect(200) + .then(({ body }) => body.id); + + await asAlice.post(`/v1/projects/${newProjectId}/forms?publish=true`) + .set('Content-Type', 'application/xml') + .send(testData.forms.clientAudits) + .expect(200); + + // submit a new submission with client audit attachment + await asAlice.post(`/v1/projects/${newProjectId}/submission`) + .set('X-OpenRosa-Version', '1.0') + .attach('audit.csv', createReadStream(appRoot + '/test/data/audit.csv'), { filename: 'audit.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.one), { filename: 'data.xml' }) + .expect(201); + + // fail the processing of this latest event + let event = (await container.Audits.getLatestByAction('submission.attachment.update')).get(); + // eslint-disable-next-line prefer-promise-reject-errors + const jobMap = { 'submission.attachment.update': [ () => Promise.reject({ uh: 'oh' }) ] }; + await promisify(runner(container, jobMap))(event); + + // should still be 0 because the failure count is only at 1, needs to be at 5 to count + event = (await container.Audits.getLatestByAction('submission.attachment.update')).get(); + event.failures.should.equal(1); + await container.Analytics.countClientAuditProcessingFailed() + .then((res) => res.should.equal(0)); + + // there should still be 0 extracted client audit rows + await container.Analytics.countClientAuditRows() + .then((res) => res.should.equal(0)); + + // manually upping failure count to 5 + await container.run(sql`update audits set failures = 5, "loggedAt" = '1999-01-01' where id = ${event.id}`); + + await container.Analytics.countClientAuditProcessingFailed() + .then((res) => res.should.equal(1)); + + await container.Analytics.auditLogs() + .then((res) => { + res.any_failure.should.equal(1); + res.failed5.should.equal(1); + res.unprocessed.should.equal(0); + }); + })); + + it('should count the number of rows extracted from client audit attachments', testService(async (service, container) => { + const asAlice = await service.login('alice'); + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.clientAudits) + .expect(200); + + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('audit.csv', createReadStream(appRoot + '/test/data/audit.csv'), { filename: 'audit.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.one), { filename: 'data.xml' }) + .expect(201); + + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('log.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'log.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' }) + .expect(201); + + await container.Analytics.countClientAuditRows() + .then((res) => res.should.equal(0)); + + await exhaust(container); + + await container.Analytics.countClientAuditRows() + .then((res) => res.should.equal(8)); + })); + }); + + it('should count failed audits', testService(async (service, container) => { + const asAlice = await service.login('alice'); + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.clientAudits) + .expect(200); + + // making the processing of this attachment fail once + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('audit.csv', createReadStream(appRoot + '/test/data/audit.csv'), { filename: 'audit.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.one), { filename: 'data.xml' }) + .expect(201); + + // eslint-disable-next-line prefer-promise-reject-errors + const jobMap = { 'submission.attachment.update': [ () => Promise.reject({ uh: 'oh' }) ] }; + const eventOne = (await container.Audits.getLatestByAction('submission.attachment.update')).get(); + await promisify(runner(container, jobMap))(eventOne); + + // making this look like it failed 5 times + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('log.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'log.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' }) + .expect(201); + + + // we haven't run exhaust(container so there are some unprocessed events) + const eventTwo = (await container.Audits.getLatestByAction('submission.attachment.update')).get(); + await container.run(sql`update audits set failures = 5 where id = ${eventTwo.id}`); + + const eventSubCreate = (await container.Audits.getLatestByAction('submission.create')).get(); + await container.run(sql`update audits set "loggedAt" = '2000-01-01T00:00Z' where id = ${eventSubCreate.id}`); + + await container.Analytics.auditLogs() + .then((res) => { + res.any_failure.should.equal(2); + res.failed5.should.equal(1); + res.unprocessed.should.equal(1); + }); + })); }); describe('user metrics', () => { @@ -892,30 +1077,65 @@ describe('analytics task queries', function () { describe('combined analytics', () => { it('should combine system level queries', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + // creating client audits (before encrypting the project) + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.clientAudits) + .expect(200); + + // the one sub with good client audit attachment + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('audit.csv', createReadStream(appRoot + '/test/data/audit.csv'), { filename: 'audit.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.one), { filename: 'data.xml' }) + .expect(201); + + await exhaust(container); + + // add another client audit attachment to fail + await asAlice.post('/v1/projects/1/submission') + .set('X-OpenRosa-Version', '1.0') + .attach('log.csv', createReadStream(appRoot + '/test/data/audit2.csv'), { filename: 'log.csv' }) + .attach('xml_submission_file', Buffer.from(testData.instances.clientAudits.two), { filename: 'data.xml' }) + .expect(201); + + // alter the second unprocessed client audit event to count as a failure + const event = (await container.Audits.getLatestByAction('submission.attachment.update')).get(); + await container.run(sql`update audits set failures = 5 where id = ${event.id}`); + // encrypting a project - await service.login('alice', (asAlice) => - asAlice.post('/v1/projects/1/key') - .send({ passphrase: 'supersecret', hint: 'it is a secret' })); + await asAlice.post('/v1/projects/1/key') + .send({ passphrase: 'supersecret', hint: 'it is a secret' }); // creating and archiving a project - await service.login('alice', (asAlice) => - asAlice.post('/v1/projects') + await asAlice.post('/v1/projects') + .set('Content-Type', 'application/json') + .send({ name: 'New Project' }) + .expect(200) + .then(({ body }) => asAlice.patch(`/v1/projects/${body.id}`) .set('Content-Type', 'application/json') - .send({ name: 'New Project' }) - .expect(200) - .then(({ body }) => asAlice.patch(`/v1/projects/${body.id}`) - .set('Content-Type', 'application/json') - .send({ archived: true }) - .expect(200))); + .send({ archived: true }) + .expect(200)); // creating more roles await createTestUser(service, container, 'Viewer1', 'viewer', 1); await createTestUser(service, container, 'Collector1', 'formfill', 1); + // creating audit events in various states + await container.run(sql`insert into audits ("actorId", action, "acteeId", details, "loggedAt", "failures") + values + (null, 'dummy.action', null, null, '1999-1-1', 1), + (null, 'dummy.action', null, null, '1999-1-1', 5), + (null, 'dummy.action', null, null, '1999-1-1', 0)`); + + const res = await container.Analytics.previewMetrics(); // can't easily test this metric delete res.system.uses_external_db; + delete res.system.sso_enabled; // everything in system filled in Object.values(res.system).forEach((metric) =>