From 075967a89c5501e82bd0296947e59f3085c5af9b Mon Sep 17 00:00:00 2001 From: lemu Date: Tue, 10 Oct 2023 11:04:08 -0300 Subject: [PATCH] fix: snapshot status banner (#1323) * chore: error rate measuring WIP * chore: show banner if error rate > 0.25, log snapshot status if a recent request failed, refactors * refactor: extract to snapshot status service * refactor: extract snapshot status config to env vars --- .env.example | 5 ++ gatsby-browser.js | 3 +- src/back/jobs/PingSnapshot.ts | 7 +- src/back/routes/snapshot.ts | 3 +- src/clients/SnapshotApi.ts | 4 +- src/clients/SnapshotTypes.ts | 5 +- src/components/Debug/SnapshotStatus.tsx | 6 +- src/constants.ts | 5 +- src/server.ts | 2 +- src/services/SnapshotService.ts | 58 +------------ src/services/SnapshotStatusService.ts | 107 ++++++++++++++++++++++++ 11 files changed, 130 insertions(+), 75 deletions(-) create mode 100644 src/services/SnapshotStatusService.ts diff --git a/.env.example b/.env.example index 1e58a356c..e946b25f3 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,11 @@ GATSBY_SNAPSHOT_DELEGATE_CONTRACT_ADDRESS=0x469788fE6E9E9681C6ebF3bF78e7Fd26Fc01 GATSBY_SNAPSHOT_ADDRESS= SNAPSHOT_PRIVATE_KEY= +# Snapshot Status +SNAPSHOT_STATUS_ENABLED='false' +SNAPSHOT_STATUS_MAX_ERROR_BUFFER_SIZE=30 +SNAPSHOT_STATUS_ERROR_RATE_THRESHOLD=0.33 + # Discourse integration GATSBY_DISCOURSE_CATEGORY=14 GATSBY_DISCOURSE_USER=dao diff --git a/gatsby-browser.js b/gatsby-browser.js index fe7f2fa07..017d247dd 100644 --- a/gatsby-browser.js +++ b/gatsby-browser.js @@ -32,7 +32,6 @@ import { SEGMENT_KEY, SSO_URL } from "./src/constants" import { flattenMessages } from "./src/utils/intl" import en from "./src/intl/en.json" import SnapshotStatus from "./src/components/Debug/SnapshotStatus" -import SNAPSHOT_STATUS_ENABLED from "./src/constants" const queryClient = new QueryClient() @@ -51,7 +50,7 @@ export const wrapPageElement = ({ element, props }) => { - {SNAPSHOT_STATUS_ENABLED && } + }> {element} diff --git a/src/back/jobs/PingSnapshot.ts b/src/back/jobs/PingSnapshot.ts index dccc065ea..bf2820285 100644 --- a/src/back/jobs/PingSnapshot.ts +++ b/src/back/jobs/PingSnapshot.ts @@ -1,8 +1,5 @@ -import { SNAPSHOT_STATUS_ENABLED } from '../../constants' -import { SnapshotService } from '../../services/SnapshotService' +import { SnapshotStatusService } from '../../services/SnapshotStatusService' export async function pingSnapshot() { - if (SNAPSHOT_STATUS_ENABLED) { - await SnapshotService.ping() - } + await SnapshotStatusService.ping() } diff --git a/src/back/routes/snapshot.ts b/src/back/routes/snapshot.ts index 635f9fab0..e2df759dc 100644 --- a/src/back/routes/snapshot.ts +++ b/src/back/routes/snapshot.ts @@ -5,6 +5,7 @@ import { Request } from 'express' import { SnapshotVote } from '../../clients/SnapshotTypes' import { SnapshotService } from '../../services/SnapshotService' +import { SnapshotStatusService } from '../../services/SnapshotStatusService' import { validateAddress, validateDates, @@ -26,7 +27,7 @@ export default routes((router) => { }) async function getStatus(req: Request) { - return await SnapshotService.getStatus() + return await SnapshotStatusService.getStatus() } async function getConfig(req: Request<{ spaceName?: string }>) { diff --git a/src/clients/SnapshotApi.ts b/src/clients/SnapshotApi.ts index 400e82abd..50f7a1653 100644 --- a/src/clients/SnapshotApi.ts +++ b/src/clients/SnapshotApi.ts @@ -188,8 +188,8 @@ export class SnapshotApi { return Number(time.toString().slice(0, -3)) } - async ping(addressesSample: string[]) { - const addresses = addressesSample.length === 0 ? DEBUG_ADDRESSES : addressesSample + async ping(addressesSample?: string[]) { + const addresses = !addressesSample || addressesSample.length === 0 ? DEBUG_ADDRESSES : addressesSample try { const { formattedAddresses, spaceName, network, strategies, scoreApiUrl } = await this.prepareScoresQueryArgs( addresses diff --git a/src/clients/SnapshotTypes.ts b/src/clients/SnapshotTypes.ts index dd37c7817..c16e6b998 100644 --- a/src/clients/SnapshotTypes.ts +++ b/src/clients/SnapshotTypes.ts @@ -134,10 +134,9 @@ export enum StrategyOrder { export enum ServiceHealth { Normal = 'normal', - Slow = 'slow', Failing = 'failing', Unknown = 'unknown', } -export type ServiceStatus = { health: ServiceHealth; responseTime: number } +export type ServiceStatus = { health: ServiceHealth; responseTime: number; errorRate: number } export type SnapshotStatus = { scoresStatus: ServiceStatus; graphQlStatus: ServiceStatus } -export const UNKNOWN_STATUS = { health: ServiceHealth.Unknown, responseTime: 0 } +export const UNKNOWN_STATUS = { health: ServiceHealth.Unknown, responseTime: 0, errorRate: 0 } diff --git a/src/components/Debug/SnapshotStatus.tsx b/src/components/Debug/SnapshotStatus.tsx index b9e1311a3..b99589419 100644 --- a/src/components/Debug/SnapshotStatus.tsx +++ b/src/components/Debug/SnapshotStatus.tsx @@ -4,6 +4,7 @@ import classNames from 'classnames' import { Governance } from '../../clients/Governance' import { ServiceHealth, SnapshotStatus as SnapshotServiceStatus } from '../../clients/SnapshotTypes' +import { SNAPSHOT_STATUS_ENABLED } from '../../constants' import { useBurgerMenu } from '../../hooks/useBurgerMenu' import useFormatMessage from '../../hooks/useFormatMessage' import Markdown from '../Common/Typography/Markdown' @@ -11,7 +12,7 @@ import WarningTriangle from '../Icon/WarningTriangle' import './SnapshotStatus.css' -const PING_INTERVAL_IN_MS = 10000 // 10 seconds +const PING_INTERVAL_IN_MS = 30000 // 30 seconds function logIfNotNormal(status: SnapshotServiceStatus) { if (status.scoresStatus.health !== ServiceHealth.Normal || status.graphQlStatus.health !== ServiceHealth.Normal) { @@ -27,8 +28,7 @@ export default function SnapshotStatus() { const updateServiceStatus = async () => { const status = await Governance.get().getSnapshotStatus() logIfNotNormal(status) - const show = - status.scoresStatus.health === ServiceHealth.Slow || status.scoresStatus.health === ServiceHealth.Failing + const show = status.scoresStatus.health === ServiceHealth.Failing && SNAPSHOT_STATUS_ENABLED setShowTopBar(show) setStatus((prev) => ({ ...prev, snapshotStatusBarOpen: show })) } diff --git a/src/constants.ts b/src/constants.ts index 8c094061f..fc641ca17 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -38,4 +38,7 @@ export const DEBUG_ADDRESSES = (process.env.DEBUG_ADDRESSES || '') .split(',') .filter(isEthereumAddress) .map((address) => address.toLowerCase()) -export const SNAPSHOT_STATUS_ENABLED = false + +export const SNAPSHOT_STATUS_ENABLED = process.env.SNAPSHOT_STATUS_ENABLED === 'true' +export const SNAPSHOT_STATUS_MAX_ERROR_BUFFER_SIZE = Number(process.env.SNAPSHOT_STATUS_MAX_ERROR_BUFFER_SIZE || 30) +export const SNAPSHOT_STATUS_ERROR_RATE_THRESHOLD = Number(process.env.SNAPSHOT_STATUS_ERROR_RATE_THRESHOLD || 0.33) diff --git a/src/server.ts b/src/server.ts index 0dd09715d..7549588ec 100644 --- a/src/server.ts +++ b/src/server.ts @@ -45,7 +45,7 @@ const jobs = manager() jobs.cron('@eachMinute', finishProposal) jobs.cron('@eachMinute', activateProposals) jobs.cron('@eachMinute', publishBids) -jobs.cron('@eachMinute', pingSnapshot) +jobs.cron('@each10Second', pingSnapshot) jobs.cron('@daily', updateGovernanceBudgets) jobs.cron('@daily', runAirdropJobs) jobs.cron('@monthly', giveTopVoterBadges) diff --git a/src/services/SnapshotService.ts b/src/services/SnapshotService.ts index 341151045..e78c316e0 100644 --- a/src/services/SnapshotService.ts +++ b/src/services/SnapshotService.ts @@ -3,76 +3,20 @@ import isNumber from 'lodash/isNumber' import { SnapshotApi, SnapshotReceipt } from '../clients/SnapshotApi' import { SnapshotGraphql } from '../clients/SnapshotGraphql' -import { - DetailedScores, - ServiceHealth, - ServiceStatus, - SnapshotProposal, - SnapshotStatus, - SnapshotVote, - UNKNOWN_STATUS, - VpDistribution, -} from '../clients/SnapshotTypes' +import { DetailedScores, SnapshotProposal, SnapshotVote, VpDistribution } from '../clients/SnapshotTypes' import * as templates from '../entities/Proposal/templates' import { proposalUrl, snapshotProposalUrl } from '../entities/Proposal/utils' import { SNAPSHOT_SPACE } from '../entities/Snapshot/constants' import { isSameAddress } from '../entities/Snapshot/utils' import { inBackground } from '../helpers' import { Avatar } from '../utils/Catalyst/types' -import { ErrorCategory } from '../utils/errorCategories' -import CacheService from './CacheService' -import { ErrorService } from './ErrorService' import { ProposalInCreation, ProposalLifespan } from './ProposalService' import RpcService from './RpcService' const DELEGATION_STRATEGY_NAME = 'delegation' -const SLOW_RESPONSE_TIME_THRESHOLD_IN_MS = 8000 // 8 seconds -const SNAPSHOT_STATUS_CACHE_KEY = 'SNAPSHOT_STATUS' export class SnapshotService { - public static async getStatus(): Promise { - const cachedStatus = CacheService.get(SNAPSHOT_STATUS_CACHE_KEY) - if (cachedStatus) { - return cachedStatus - } - return { scoresStatus: UNKNOWN_STATUS, graphQlStatus: UNKNOWN_STATUS } - } - - public static async ping() { - try { - const { status: graphQlStatus, addressesSample } = await this.pingGraphQl() - const scoresStatus = await this.pingScores(addressesSample) - const snapshotStatus = { scoresStatus, graphQlStatus } - CacheService.set(SNAPSHOT_STATUS_CACHE_KEY, snapshotStatus) - logger.log('Snapshot status:', snapshotStatus) - } catch (error) { - ErrorService.report('Unable to determine snapshot status', { error, category: ErrorCategory.Snapshot }) - } - } - - private static async pingScores(addressesSample: string[]): Promise { - let scoresHealth = ServiceHealth.Normal - const responseTime = await SnapshotApi.get().ping(addressesSample) - if (responseTime === -1) { - scoresHealth = ServiceHealth.Failing - } else if (responseTime > SLOW_RESPONSE_TIME_THRESHOLD_IN_MS) { - scoresHealth = ServiceHealth.Slow - } - return { health: scoresHealth, responseTime } - } - - private static async pingGraphQl(): Promise<{ status: ServiceStatus; addressesSample: string[] }> { - let health = ServiceHealth.Normal - const { responseTime, addressesSample } = await SnapshotGraphql.get().ping() - if (responseTime === -1) { - health = ServiceHealth.Failing - } else if (responseTime > SLOW_RESPONSE_TIME_THRESHOLD_IN_MS) { - health = ServiceHealth.Slow - } - return { status: { health, responseTime }, addressesSample } - } - static async createProposal( proposalInCreation: ProposalInCreation, proposalId: string, diff --git a/src/services/SnapshotStatusService.ts b/src/services/SnapshotStatusService.ts new file mode 100644 index 000000000..b63a4ba43 --- /dev/null +++ b/src/services/SnapshotStatusService.ts @@ -0,0 +1,107 @@ +import chalk from 'chalk' + +import { SnapshotApi } from '../clients/SnapshotApi' +import { SnapshotGraphql } from '../clients/SnapshotGraphql' +import { ServiceHealth, ServiceStatus, SnapshotStatus, UNKNOWN_STATUS } from '../clients/SnapshotTypes' +import { SNAPSHOT_STATUS_ERROR_RATE_THRESHOLD, SNAPSHOT_STATUS_MAX_ERROR_BUFFER_SIZE } from '../constants' +import { ErrorCategory } from '../utils/errorCategories' + +import CacheService from './CacheService' +import { ErrorService } from './ErrorService' + +const SNAPSHOT_STATUS_CACHE_KEY = 'SNAPSHOT_STATUS' + +export class SnapshotStatusService { + private static scoresRequestResults: boolean[] = [] + private static graphQlRequestResults: boolean[] = [] + + public static async getStatus(): Promise { + const cachedStatus = CacheService.get(SNAPSHOT_STATUS_CACHE_KEY) + if (cachedStatus) { + return cachedStatus + } + return { scoresStatus: UNKNOWN_STATUS, graphQlStatus: UNKNOWN_STATUS } + } + + public static async ping() { + try { + const { status: graphQlStatus, addressesSample } = await this.measureGraphQlErrorRate() + const scoresStatus = await this.measureScoresErrorRate(addressesSample) + const snapshotStatus = { scoresStatus, graphQlStatus } + CacheService.set(SNAPSHOT_STATUS_CACHE_KEY, snapshotStatus) + this.logOnRecentError(snapshotStatus) + } catch (error) { + ErrorService.report('Unable to determine snapshot status', { error, category: ErrorCategory.Snapshot }) + } + } + + private static async measureGraphQlErrorRate(): Promise<{ status: ServiceStatus; addressesSample: string[] }> { + let requestSuccessful = true + const { responseTime, addressesSample } = await SnapshotGraphql.get().ping() + if (responseTime === -1) { + requestSuccessful = false + } + this.graphQlRequestResults.push(requestSuccessful) + const { errorRate, health } = this.calculateErrorRate(this.graphQlRequestResults) + + return { status: { health, responseTime, errorRate }, addressesSample } + } + + private static async measureScoresErrorRate(addressesSample: string[]): Promise { + let requestSuccessful = true + const responseTime = await SnapshotApi.get().ping(addressesSample) + if (responseTime === -1) { + requestSuccessful = false + } + this.scoresRequestResults.push(requestSuccessful) + + const { errorRate, health } = this.calculateErrorRate(this.scoresRequestResults) + + return { health, responseTime, errorRate } + } + + private static calculateErrorRate(responsesBuffer: boolean[]) { + if (responsesBuffer.length > SNAPSHOT_STATUS_MAX_ERROR_BUFFER_SIZE) { + responsesBuffer.shift() + } + const errorRate = responsesBuffer.filter((requestSuccessful) => !requestSuccessful).length / responsesBuffer.length + + let health = ServiceHealth.Normal + if (errorRate > SNAPSHOT_STATUS_ERROR_RATE_THRESHOLD) { + health = ServiceHealth.Failing + } + return { errorRate, health } + } + + private static visualizeRequestResults(requestResults: boolean[]) { + const numRows = 1 + const numCols = requestResults.length + const greenPoint = chalk.green('●') + const redPoint = chalk.red('●') + let matrix = '' + for (let i = 0; i < numRows; i++) { + for (let j = 0; j < numCols; j++) { + const index = i * numCols + j + if (index < requestResults.length) { + const result = requestResults[index] + matrix += result ? greenPoint : redPoint + } else { + matrix += ' ' + } + matrix += ' ' + } + matrix += '\n' + } + console.log(matrix) + } + + private static logOnRecentError(snapshotStatus: SnapshotStatus) { + if (this.scoresRequestResults.includes(false) || this.graphQlRequestResults.includes(false)) { + console.log('\u26A1', 'Snapshot Status', '\u26A1') + console.log('Scores: ', JSON.stringify(snapshotStatus.scoresStatus)) + this.visualizeRequestResults(this.scoresRequestResults) + console.log('GraphQl: ', JSON.stringify(snapshotStatus.graphQlStatus)) + this.visualizeRequestResults(this.graphQlRequestResults) + } + } +}