fix: snapshot status banner (#1323)

* chore: error rate measuring WIP

* chore: show banner if error rate > 0.25, log snapshot status if a recent request failed, refactors

* refactor: extract to snapshot status service

* refactor: extract snapshot status config to env vars

.env.example

@@ -37,6 +37,11 @@ GATSBY_SNAPSHOT_DELEGATE_CONTRACT_ADDRESS=0x469788fE6E9E9681C6ebF3bF78e7Fd26Fc01
 GATSBY_SNAPSHOT_ADDRESS=
 SNAPSHOT_PRIVATE_KEY=
 
+# Snapshot Status
+SNAPSHOT_STATUS_ENABLED='false'
+SNAPSHOT_STATUS_MAX_ERROR_BUFFER_SIZE=30
+SNAPSHOT_STATUS_ERROR_RATE_THRESHOLD=0.33
+
 # Discourse integration
 GATSBY_DISCOURSE_CATEGORY=14
 GATSBY_DISCOURSE_USER=dao

gatsby-browser.js

@@ -32,7 +32,6 @@ import { SEGMENT_KEY, SSO_URL } from "./src/constants"
 import { flattenMessages } from "./src/utils/intl"
 import en from "./src/intl/en.json"
 import SnapshotStatus from "./src/components/Debug/SnapshotStatus"
-import SNAPSHOT_STATUS_ENABLED from "./src/constants"
 
 const queryClient = new QueryClient()
 
@@ -51,7 +50,7 @@ export const wrapPageElement = ({ element, props }) => {
       <IntlProvider defaultLocale='en' locale='en' messages={flattenMessages(en)}>
         <IdentityModalContextProvider>
           <BurgerMenuStatusContextProvider>
-            {SNAPSHOT_STATUS_ENABLED && <SnapshotStatus />}
+            <SnapshotStatus />
             <Layout {...props} rightMenu={<Navbar />}>
               {element}
             </Layout>

src/back/jobs/PingSnapshot.ts

@@ -1,8 +1,5 @@
-import { SNAPSHOT_STATUS_ENABLED } from '../../constants'
-import { SnapshotService } from '../../services/SnapshotService'
+import { SnapshotStatusService } from '../../services/SnapshotStatusService'
 
 export async function pingSnapshot() {
-  if (SNAPSHOT_STATUS_ENABLED) {
-    await SnapshotService.ping()
-  }
+  await SnapshotStatusService.ping()
 }

src/back/routes/snapshot.ts

@@ -5,6 +5,7 @@ import { Request } from 'express'
 
 import { SnapshotVote } from '../../clients/SnapshotTypes'
 import { SnapshotService } from '../../services/SnapshotService'
+import { SnapshotStatusService } from '../../services/SnapshotStatusService'
 import {
   validateAddress,
   validateDates,
@@ -26,7 +27,7 @@ export default routes((router) => {
 })
 
 async function getStatus(req: Request) {
-  return await SnapshotService.getStatus()
+  return await SnapshotStatusService.getStatus()
 }
 
 async function getConfig(req: Request<{ spaceName?: string }>) {

src/clients/SnapshotApi.ts

@@ -188,8 +188,8 @@ export class SnapshotApi {
     return Number(time.toString().slice(0, -3))
   }
 
-  async ping(addressesSample: string[]) {
-    const addresses = addressesSample.length === 0 ? DEBUG_ADDRESSES : addressesSample
+  async ping(addressesSample?: string[]) {
+    const addresses = !addressesSample || addressesSample.length === 0 ? DEBUG_ADDRESSES : addressesSample
     try {
       const { formattedAddresses, spaceName, network, strategies, scoreApiUrl } = await this.prepareScoresQueryArgs(
         addresses

src/clients/SnapshotTypes.ts

@@ -134,10 +134,9 @@ export enum StrategyOrder {
 
 export enum ServiceHealth {
   Normal = 'normal',
-  Slow = 'slow',
   Failing = 'failing',
   Unknown = 'unknown',
 }
-export type ServiceStatus = { health: ServiceHealth; responseTime: number }
+export type ServiceStatus = { health: ServiceHealth; responseTime: number; errorRate: number }
 export type SnapshotStatus = { scoresStatus: ServiceStatus; graphQlStatus: ServiceStatus }
-export const UNKNOWN_STATUS = { health: ServiceHealth.Unknown, responseTime: 0 }
+export const UNKNOWN_STATUS = { health: ServiceHealth.Unknown, responseTime: 0, errorRate: 0 }

src/components/Debug/SnapshotStatus.tsx

@@ -4,14 +4,15 @@ import classNames from 'classnames'
 
 import { Governance } from '../../clients/Governance'
 import { ServiceHealth, SnapshotStatus as SnapshotServiceStatus } from '../../clients/SnapshotTypes'
+import { SNAPSHOT_STATUS_ENABLED } from '../../constants'
 import { useBurgerMenu } from '../../hooks/useBurgerMenu'
 import useFormatMessage from '../../hooks/useFormatMessage'
 import Markdown from '../Common/Typography/Markdown'
 import WarningTriangle from '../Icon/WarningTriangle'
 
 import './SnapshotStatus.css'
 
-const PING_INTERVAL_IN_MS = 10000 // 10 seconds
+const PING_INTERVAL_IN_MS = 30000 // 30 seconds
 
 function logIfNotNormal(status: SnapshotServiceStatus) {
   if (status.scoresStatus.health !== ServiceHealth.Normal || status.graphQlStatus.health !== ServiceHealth.Normal) {
@@ -27,8 +28,7 @@ export default function SnapshotStatus() {
   const updateServiceStatus = async () => {
     const status = await Governance.get().getSnapshotStatus()
     logIfNotNormal(status)
-    const show =
-      status.scoresStatus.health === ServiceHealth.Slow || status.scoresStatus.health === ServiceHealth.Failing
+    const show = status.scoresStatus.health === ServiceHealth.Failing && SNAPSHOT_STATUS_ENABLED
     setShowTopBar(show)
     setStatus((prev) => ({ ...prev, snapshotStatusBarOpen: show }))
   }

src/constants.ts

@@ -38,4 +38,7 @@ export const DEBUG_ADDRESSES = (process.env.DEBUG_ADDRESSES || '')
   .split(',')
   .filter(isEthereumAddress)
   .map((address) => address.toLowerCase())
-export const SNAPSHOT_STATUS_ENABLED = false
+
+export const SNAPSHOT_STATUS_ENABLED = process.env.SNAPSHOT_STATUS_ENABLED === 'true'
+export const SNAPSHOT_STATUS_MAX_ERROR_BUFFER_SIZE = Number(process.env.SNAPSHOT_STATUS_MAX_ERROR_BUFFER_SIZE || 30)
+export const SNAPSHOT_STATUS_ERROR_RATE_THRESHOLD = Number(process.env.SNAPSHOT_STATUS_ERROR_RATE_THRESHOLD || 0.33)

src/server.ts

@@ -45,7 +45,7 @@ const jobs = manager()
 jobs.cron('@eachMinute', finishProposal)
 jobs.cron('@eachMinute', activateProposals)
 jobs.cron('@eachMinute', publishBids)
-jobs.cron('@eachMinute', pingSnapshot)
+jobs.cron('@each10Second', pingSnapshot)
 jobs.cron('@daily', updateGovernanceBudgets)
 jobs.cron('@daily', runAirdropJobs)
 jobs.cron('@monthly', giveTopVoterBadges)

src/services/SnapshotService.ts

@@ -3,76 +3,20 @@ import isNumber from 'lodash/isNumber'
 
 import { SnapshotApi, SnapshotReceipt } from '../clients/SnapshotApi'
 import { SnapshotGraphql } from '../clients/SnapshotGraphql'
-import {
-  DetailedScores,
-  ServiceHealth,
-  ServiceStatus,
-  SnapshotProposal,
-  SnapshotStatus,
-  SnapshotVote,
-  UNKNOWN_STATUS,
-  VpDistribution,
-} from '../clients/SnapshotTypes'
+import { DetailedScores, SnapshotProposal, SnapshotVote, VpDistribution } from '../clients/SnapshotTypes'
 import * as templates from '../entities/Proposal/templates'
 import { proposalUrl, snapshotProposalUrl } from '../entities/Proposal/utils'
 import { SNAPSHOT_SPACE } from '../entities/Snapshot/constants'
 import { isSameAddress } from '../entities/Snapshot/utils'
 import { inBackground } from '../helpers'
 import { Avatar } from '../utils/Catalyst/types'
-import { ErrorCategory } from '../utils/errorCategories'
 
-import CacheService from './CacheService'
-import { ErrorService } from './ErrorService'
 import { ProposalInCreation, ProposalLifespan } from './ProposalService'
 import RpcService from './RpcService'
 
 const DELEGATION_STRATEGY_NAME = 'delegation'
-const SLOW_RESPONSE_TIME_THRESHOLD_IN_MS = 8000 // 8 seconds
-const SNAPSHOT_STATUS_CACHE_KEY = 'SNAPSHOT_STATUS'
 
 export class SnapshotService {
-  public static async getStatus(): Promise<SnapshotStatus | undefined> {
-    const cachedStatus = CacheService.get<SnapshotStatus>(SNAPSHOT_STATUS_CACHE_KEY)
-    if (cachedStatus) {
-      return cachedStatus
-    }
-    return { scoresStatus: UNKNOWN_STATUS, graphQlStatus: UNKNOWN_STATUS }
-  }
-
-  public static async ping() {
-    try {
-      const { status: graphQlStatus, addressesSample } = await this.pingGraphQl()
-      const scoresStatus = await this.pingScores(addressesSample)
-      const snapshotStatus = { scoresStatus, graphQlStatus }
-      CacheService.set(SNAPSHOT_STATUS_CACHE_KEY, snapshotStatus)
-      logger.log('Snapshot status:', snapshotStatus)
-    } catch (error) {
-      ErrorService.report('Unable to determine snapshot status', { error, category: ErrorCategory.Snapshot })
-    }
-  }
-
-  private static async pingScores(addressesSample: string[]): Promise<ServiceStatus> {
-    let scoresHealth = ServiceHealth.Normal
-    const responseTime = await SnapshotApi.get().ping(addressesSample)
-    if (responseTime === -1) {
-      scoresHealth = ServiceHealth.Failing
-    } else if (responseTime > SLOW_RESPONSE_TIME_THRESHOLD_IN_MS) {
-      scoresHealth = ServiceHealth.Slow
-    }
-    return { health: scoresHealth, responseTime }
-  }
-
-  private static async pingGraphQl(): Promise<{ status: ServiceStatus; addressesSample: string[] }> {
-    let health = ServiceHealth.Normal
-    const { responseTime, addressesSample } = await SnapshotGraphql.get().ping()
-    if (responseTime === -1) {
-      health = ServiceHealth.Failing
-    } else if (responseTime > SLOW_RESPONSE_TIME_THRESHOLD_IN_MS) {
-      health = ServiceHealth.Slow
-    }
-    return { status: { health, responseTime }, addressesSample }
-  }
-
   static async createProposal(
     proposalInCreation: ProposalInCreation,
     proposalId: string,

src/services/SnapshotStatusService.ts

@@ -0,0 +1,107 @@
+import chalk from 'chalk'
+
+import { SnapshotApi } from '../clients/SnapshotApi'
+import { SnapshotGraphql } from '../clients/SnapshotGraphql'
+import { ServiceHealth, ServiceStatus, SnapshotStatus, UNKNOWN_STATUS } from '../clients/SnapshotTypes'
+import { SNAPSHOT_STATUS_ERROR_RATE_THRESHOLD, SNAPSHOT_STATUS_MAX_ERROR_BUFFER_SIZE } from '../constants'
+import { ErrorCategory } from '../utils/errorCategories'
+
+import CacheService from './CacheService'
+import { ErrorService } from './ErrorService'
+
+const SNAPSHOT_STATUS_CACHE_KEY = 'SNAPSHOT_STATUS'
+
+export class SnapshotStatusService {
+  private static scoresRequestResults: boolean[] = []
+  private static graphQlRequestResults: boolean[] = []
+
+  public static async getStatus(): Promise<SnapshotStatus | undefined> {
+    const cachedStatus = CacheService.get<SnapshotStatus>(SNAPSHOT_STATUS_CACHE_KEY)
+    if (cachedStatus) {
+      return cachedStatus
+    }
+    return { scoresStatus: UNKNOWN_STATUS, graphQlStatus: UNKNOWN_STATUS }
+  }
+
+  public static async ping() {
+    try {
+      const { status: graphQlStatus, addressesSample } = await this.measureGraphQlErrorRate()
+      const scoresStatus = await this.measureScoresErrorRate(addressesSample)
+      const snapshotStatus = { scoresStatus, graphQlStatus }
+      CacheService.set(SNAPSHOT_STATUS_CACHE_KEY, snapshotStatus)
+      this.logOnRecentError(snapshotStatus)
+    } catch (error) {
+      ErrorService.report('Unable to determine snapshot status', { error, category: ErrorCategory.Snapshot })
+    }
+  }
+
+  private static async measureGraphQlErrorRate(): Promise<{ status: ServiceStatus; addressesSample: string[] }> {
+    let requestSuccessful = true
+    const { responseTime, addressesSample } = await SnapshotGraphql.get().ping()
+    if (responseTime === -1) {
+      requestSuccessful = false
+    }
+    this.graphQlRequestResults.push(requestSuccessful)
+    const { errorRate, health } = this.calculateErrorRate(this.graphQlRequestResults)
+
+    return { status: { health, responseTime, errorRate }, addressesSample }
+  }
+
+  private static async measureScoresErrorRate(addressesSample: string[]): Promise<ServiceStatus> {
+    let requestSuccessful = true
+    const responseTime = await SnapshotApi.get().ping(addressesSample)
+    if (responseTime === -1) {
+      requestSuccessful = false
+    }
+    this.scoresRequestResults.push(requestSuccessful)
+
+    const { errorRate, health } = this.calculateErrorRate(this.scoresRequestResults)
+
+    return { health, responseTime, errorRate }
+  }
+
+  private static calculateErrorRate(responsesBuffer: boolean[]) {
+    if (responsesBuffer.length > SNAPSHOT_STATUS_MAX_ERROR_BUFFER_SIZE) {
+      responsesBuffer.shift()
+    }
+    const errorRate = responsesBuffer.filter((requestSuccessful) => !requestSuccessful).length / responsesBuffer.length
+
+    let health = ServiceHealth.Normal
+    if (errorRate > SNAPSHOT_STATUS_ERROR_RATE_THRESHOLD) {
+      health = ServiceHealth.Failing
+    }
+    return { errorRate, health }
+  }
+
+  private static visualizeRequestResults(requestResults: boolean[]) {
+    const numRows = 1
+    const numCols = requestResults.length
+    const greenPoint = chalk.green('●')
+    const redPoint = chalk.red('●')
+    let matrix = ''
+    for (let i = 0; i < numRows; i++) {
+      for (let j = 0; j < numCols; j++) {
+        const index = i * numCols + j
+        if (index < requestResults.length) {
+          const result = requestResults[index]
+          matrix += result ? greenPoint : redPoint
+        } else {
+          matrix += ' '
+        }
+        matrix += ' '
+      }
+      matrix += '\n'
+    }
+    console.log(matrix)
+  }
+
+  private static logOnRecentError(snapshotStatus: SnapshotStatus) {
+    if (this.scoresRequestResults.includes(false) || this.graphQlRequestResults.includes(false)) {
+      console.log('\u26A1', 'Snapshot Status', '\u26A1')
+      console.log('Scores: ', JSON.stringify(snapshotStatus.scoresStatus))
+      this.visualizeRequestResults(this.scoresRequestResults)
+      console.log('GraphQl: ', JSON.stringify(snapshotStatus.graphQlStatus))
+      this.visualizeRequestResults(this.graphQlRequestResults)
+    }
+  }
+}