Merge pull request #950 from autonomys/feat/improve-leaderboard-index…

…ers-and-tasks-similarly Improve leaderboard indexers and tasks similarly
autonomys · Nov 14, 2024 · 8e541eb · 8e541eb
2 parents a4b8bb8 + 30e8cb1
commit 8e541eb
Show file tree

Hide file tree

Showing 10 changed files with 79 additions and 199 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -248,7 +248,6 @@ services:
       - -f=/${NETWORK_ID}/leaderboard
       - --db-schema=leaderboard
       - --workers=1
-      - --unsafe
       - --batch-size=15
       - --unfinalized-blocks=true
       - --disable-historical=false

diff --git a/indexers/mainnet/leaderboard/project.ts b/indexers/mainnet/leaderboard/project.ts
@@ -156,14 +156,6 @@ const project: SubstrateProject = {
               method: "WithdrewStake",
             },
           },
-          {
-            kind: SubstrateHandlerKind.Block,
-            handler: "handleSortAndRankLeaderboard",
-            filter: {
-              // This is the frequency of the leaderboard sort and rank
-              modulo: 50,
-            },
-          },
         ],
       },
     },

diff --git a/indexers/mainnet/leaderboard/src/mappings/helper.ts b/indexers/mainnet/leaderboard/src/mappings/helper.ts
diff --git a/indexers/mainnet/leaderboard/src/mappings/mappingHandlers.ts b/indexers/mainnet/leaderboard/src/mappings/mappingHandlers.ts
@@ -1,6 +1,5 @@
-import { SubstrateBlock, SubstrateEvent } from "@subql/types";
+import { SubstrateEvent } from "@subql/types";
 import * as db from "./db";
-import { sortAndRankLeaderboard } from "./helper";
 
 export async function handleTransferEvent(
   event: SubstrateEvent
@@ -370,14 +369,3 @@ export async function handleWithdrewStakeEvent(
     timestamp
   );
 }
-
-export async function handleSortAndRankLeaderboard(
-  block: SubstrateBlock
-): Promise<void> {
-  const {
-    block: {
-      header: { number },
-    },
-  } = block;
-  await sortAndRankLeaderboard(BigInt(number.toString()));
-}
diff --git a/indexers/taskboard/src/tasks/consensus.ts b/indexers/taskboard/src/tasks/consensus.ts
@@ -1,4 +1,3 @@
-import { Job } from "bull";
 import { Pool, PoolClient } from "pg";
 import { connectToDB, queries } from "../utils/db";
 import { getStoredValue, setRedisValue } from "../utils/store";
@@ -9,87 +8,84 @@ interface ConsensusResult {
   query: string[];
 }
 
-interface JobData {}
-
 const LAST_BLOCK_NUMBER_KEY = "consensusUniqueRowsMapping:lastBlockNumber";
 const BLOCK_NUMBER_INCREMENT = 12; // 12 is the number of blocks to process at a time
 
-export const consensusUniqueRowsMapping = async (
-  job: Job<JobData>
-): Promise<ConsensusResult> => {
-  const pool: Pool = await connectToDB();
-
-  const result: ConsensusResult = {
-    blockNumber: 0,
-    updatedTables: [],
-    query: [],
-  };
-
-  try {
-    const client: PoolClient = await pool.connect();
-
-    // To-Do: Implement the logic to update the consensus tables
-    // Tables:
-    // - consensus_account_profiles
-    // - consensus_account_rewards
-
-    // Logic:
-    // - For new accounts, create a new row in the consensus_account_profile and consensus_account_rewards
-    // - For each rewards row, increment the total rewards value and counts for the corresponding account and reward type
-
-    // Get the last block number
-    const lastBlockNumber = await getStoredValue(LAST_BLOCK_NUMBER_KEY).then(
-      (reply) => (reply ? parseInt(reply) : 0)
-    );
+export const consensusUniqueRowsMapping =
+  async (): Promise<ConsensusResult> => {
+    const pool: Pool = await connectToDB();
 
-    const blockNumber = lastBlockNumber + BLOCK_NUMBER_INCREMENT;
-    result.blockNumber = blockNumber;
+    const result: ConsensusResult = {
+      blockNumber: 0,
+      updatedTables: [],
+      query: [],
+    };
 
     try {
-      await client.query("BEGIN");
-
-      // Execute queries
-      const [
-        sectionsResult,
-        extrinsicModuleResult,
-        eventModuleResult,
-        logResult,
-        accountResult,
-      ] = await Promise.all([
-        client.query(queries.consensusSectionsQuery, [blockNumber]),
-        client.query(queries.consensusExtrinsicModulesQuery, [blockNumber]),
-        client.query(queries.consensusEventModulesQuery, [blockNumber]),
-        client.query(queries.consensusLogKindsQuery, [blockNumber]),
-        client.query(queries.consensusAccountsQuery, [blockNumber]),
-      ]);
-
-      // Track updated tables
-      if (sectionsResult.rows.length > 0)
-        result.updatedTables.push("consensus_sections");
-      if (extrinsicModuleResult.rows.length > 0)
-        result.updatedTables.push("consensus_extrinsic_module");
-      if (eventModuleResult.rows.length > 0)
-        result.updatedTables.push("consensus_event_module");
-      if (logResult.rows.length > 0)
-        result.updatedTables.push("consensus_log_kinds");
-      if (accountResult.rows.length > 0)
-        result.updatedTables.push("consensus_accounts");
-
-      await client.query("COMMIT");
-
-      // Update the last block number
-      await setRedisValue(LAST_BLOCK_NUMBER_KEY, blockNumber.toString());
+      const client: PoolClient = await pool.connect();
+
+      // To-Do: Implement the logic to update the consensus tables
+      // Tables:
+      // - consensus_account_profiles
+      // - consensus_account_rewards
+
+      // Logic:
+      // - For new accounts, create a new row in the consensus_account_profile and consensus_account_rewards
+      // - For each rewards row, increment the total rewards value and counts for the corresponding account and reward type
+
+      // Get the last block number
+      const lastBlockNumber = await getStoredValue(LAST_BLOCK_NUMBER_KEY).then(
+        (reply) => (reply ? parseInt(reply) : 0)
+      );
+
+      const blockNumber = lastBlockNumber + BLOCK_NUMBER_INCREMENT;
+      result.blockNumber = blockNumber;
+
+      try {
+        await client.query("BEGIN");
+
+        // Execute queries
+        const [
+          sectionsResult,
+          extrinsicModuleResult,
+          eventModuleResult,
+          logResult,
+          accountResult,
+        ] = await Promise.all([
+          client.query(queries.consensusSectionsQuery, [blockNumber]),
+          client.query(queries.consensusExtrinsicModulesQuery, [blockNumber]),
+          client.query(queries.consensusEventModulesQuery, [blockNumber]),
+          client.query(queries.consensusLogKindsQuery, [blockNumber]),
+          client.query(queries.consensusAccountsQuery, [blockNumber]),
+        ]);
+
+        // Track updated tables
+        if (sectionsResult.rows.length > 0)
+          result.updatedTables.push("consensus_sections");
+        if (extrinsicModuleResult.rows.length > 0)
+          result.updatedTables.push("consensus_extrinsic_module");
+        if (eventModuleResult.rows.length > 0)
+          result.updatedTables.push("consensus_event_module");
+        if (logResult.rows.length > 0)
+          result.updatedTables.push("consensus_log_kinds");
+        if (accountResult.rows.length > 0)
+          result.updatedTables.push("consensus_accounts");
+
+        await client.query("COMMIT");
+
+        // Update the last block number
+        await setRedisValue(LAST_BLOCK_NUMBER_KEY, blockNumber.toString());
+      } catch (err) {
+        await client.query("ROLLBACK");
+        console.error("Error updating consensus tables:", err);
+        throw new Error(`Failed to update consensus tables: ${err}`);
+      } finally {
+        client.release();
+      }
+
+      return result;
     } catch (err) {
-      await client.query("ROLLBACK");
-      console.error("Error updating consensus tables:", err);
+      console.error("Error in consensus:", err);
       throw new Error(`Failed to update consensus tables: ${err}`);
-    } finally {
-      client.release();
     }
-
-    return result;
-  } catch (err) {
-    console.error("Error in consensus:", err);
-    throw new Error(`Failed to update consensus tables: ${err}`);
-  }
-};
+  };
diff --git a/indexers/taskboard/src/tasks/index.ts b/indexers/taskboard/src/tasks/index.ts
@@ -30,6 +30,10 @@ export const tasks: Tasks = {
   leaderboardSortAndRank: {
     handler: leaderboardSortAndRank,
     concurrency: 1,
+    cron: {
+      pattern: "*/2 * * * *", // Runs every 2 minutes
+      enabled: true,
+    },
   },
   updateAccount: {
     handler: updateAccount,

diff --git a/indexers/taskboard/src/tasks/leaderboardSortAndRank.ts b/indexers/taskboard/src/tasks/leaderboardSortAndRank.ts
@@ -2,31 +2,20 @@ import { Pool, PoolClient } from "pg";
 import { LEADERBOARD_ENTRY_TYPE } from "../constants";
 import { connectToDB, entryTypeToTable, queries } from "../utils/db";
 
-interface Job {
-  data: {
-    blockNumber: number;
-  };
-}
-
 interface UpdatedTable {
   table: string;
   rowCount: number;
 }
 
 interface LeaderboardResult {
-  blockNumber: number;
   updatedTables: UpdatedTable[];
   query: string[];
 }
 
-export const leaderboardSortAndRank = async (
-  job: Job
-): Promise<LeaderboardResult> => {
-  const { blockNumber } = job.data;
+export const leaderboardSortAndRank = async (): Promise<LeaderboardResult> => {
   const pool: Pool = await connectToDB();
 
   const result: LeaderboardResult = {
-    blockNumber,
     updatedTables: [],
     query: [],
   };

diff --git a/indexers/taurus/leaderboard/project.ts b/indexers/taurus/leaderboard/project.ts
@@ -156,14 +156,6 @@ const project: SubstrateProject = {
               method: "WithdrewStake",
             },
           },
-          {
-            kind: SubstrateHandlerKind.Block,
-            handler: "handleSortAndRankLeaderboard",
-            filter: {
-              // This is the frequency of the leaderboard sort and rank
-              modulo: 50,
-            },
-          },
         ],
       },
     },

diff --git a/indexers/taurus/leaderboard/src/mappings/helper.ts b/indexers/taurus/leaderboard/src/mappings/helper.ts
diff --git a/indexers/taurus/leaderboard/src/mappings/mappingHandlers.ts b/indexers/taurus/leaderboard/src/mappings/mappingHandlers.ts
@@ -1,6 +1,5 @@
-import { SubstrateBlock, SubstrateEvent } from "@subql/types";
+import { SubstrateEvent } from "@subql/types";
 import * as db from "./db";
-import { sortAndRankLeaderboard } from "./helper";
 
 export async function handleTransferEvent(
   event: SubstrateEvent
@@ -370,14 +369,3 @@ export async function handleWithdrewStakeEvent(
     timestamp
   );
 }
-
-export async function handleSortAndRankLeaderboard(
-  block: SubstrateBlock
-): Promise<void> {
-  const {
-    block: {
-      header: { number },
-    },
-  } = block;
-  await sortAndRankLeaderboard(BigInt(number.toString()));
-}