From d8a74a3325d0efe443a54c85b3ac301c60a93945 Mon Sep 17 00:00:00 2001 From: mtpoly <128363128+mt-polygon-technology@users.noreply.github.com> Date: Wed, 16 Aug 2023 11:54:53 +0200 Subject: [PATCH 01/10] Create sonar-project.properties sonarqube set up --- sonar-project.properties | 1 + 1 file changed, 1 insertion(+) create mode 100644 sonar-project.properties diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..6c791a9 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1 @@ +sonar.projectKey=maticnetwork_chainflow_AYn0jzKkuJjRFCFKZwPS From e44595052c0c41824ae4e6cd1740ab9140923a31 Mon Sep 17 00:00:00 2001 From: mtpoly <128363128+mt-polygon-technology@users.noreply.github.com> Date: Wed, 16 Aug 2023 11:57:33 +0200 Subject: [PATCH 02/10] Create security-build.yml sonarqube set up --- .github/workflows/security-build.yml | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/security-build.yml diff --git a/.github/workflows/security-build.yml b/.github/workflows/security-build.yml new file mode 100644 index 0000000..62203a6 --- /dev/null +++ b/.github/workflows/security-build.yml @@ -0,0 +1,32 @@ +name: Security Build +on: + push: + branches: + - main # or the name of your main branch + workflow_dispatch: {} + +jobs: + sonarqube: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # Disabling shallow clone is recommended for improving relevancy of reporting. + fetch-depth: 0 + + # Triggering SonarQube analysis as results of it are required by Quality Gate check. + - name: SonarQube Scan + uses: sonarsource/sonarqube-scan-action@master + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + + # Check the Quality Gate status. + - name: SonarQube Quality Gate check + id: sonarqube-quality-gate-check + uses: sonarsource/sonarqube-quality-gate-action@master + # Force to fail step after specific time. + timeout-minutes: 5 + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} From 7178496273f08baaeaf6f20738c04f390961ff65 Mon Sep 17 00:00:00 2001 From: nivida Date: Thu, 17 Aug 2023 16:03:31 +0200 Subject: [PATCH 03/10] code style and typos fixed in readme --- README.md | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index f8720e8..6125bea 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ ChainFlow block producers encompass three distinct types of producers, each desi "bootstrap.servers": '', "security.protocol": "plaintext", blockSubscriptionTimeout: 120000, - type: 'blocks:poller'. + type: 'blocks:poller', { error: (error: KafkaError | BlockProducerError) => {}, closed: () => {} // On broker connection closed @@ -211,7 +211,7 @@ ChainFlow block producers encompass three distinct types of producers, each desi maxRetries: '', mongoUrl: '', "bootstrap.servers": '', - "security.protocol": "plaintext", + "security.protocol": "plaintext" }) producer.on("blockProducer.fatalError", (error) => { @@ -271,7 +271,7 @@ const producer = new SynchronousProducer( coder: { fileName: "matic_transfer", packageName: "matictransferpackage", - messageType: "MaticTransferBlock", + messageType: "MaticTransferBlock" } } ); @@ -302,11 +302,11 @@ const producer = produce( packageName: "matictransferpackage", messageType: "MaticTransferBlock", }, - type: "synchronous" // use 'synchronous'. if synchronous producer is needed, + type: "synchronous", // use 'synchronous'. if synchronous producer is needed { emitter: () => { this.produceEvent("", ""); - } + }, error: (error: KafkaError | BlockProducerError) => {}, closed: () => {} // On broker connection closed } @@ -332,14 +332,7 @@ const producer = new AsynchronousProducer( coder: { fileName: "matic_transfer", packageName: "matictransferpackage", - messageType: "MaticTransferBlock", - }, - { - emitter: () => { - this.produceEvent("", ""); - } - error: (error: KafkaError | BlockProducerError) => {}, - closed: () => {} // On broker connection closed + messageType: "MaticTransferBlock" } } ); @@ -368,13 +361,13 @@ const producer = produce( coder: { fileName: "matic_transfer", packageName: "matictransferpackage", - messageType: "MaticTransferBlock", + messageType: "MaticTransferBlock" }, - type: "asynchronous" + type: "asynchronous", { emitter: () => { this.produceEvent("", ""); - } + }, error: (error: KafkaError | BlockProducerError) => {}, closed: () => {} // On broker connection closed } @@ -480,7 +473,7 @@ const consumer = new SynchronousConsumer( coders: { fileName: "block", packageName: "blockpackage", - messageType: "Block", + messageType: "Block" } } ); @@ -508,7 +501,7 @@ consume( "coderConfig": { fileName: "block", packageName: "blockpackage", - messageType: "Block", + messageType: "Block" }, type: 'synchronous' }, @@ -541,7 +534,7 @@ const consumer = new AsynchronousConsumer( coders: { fileName: "block", packageName: "blockpackage", - messageType: "Block", + messageType: "Block" } } ); @@ -647,7 +640,7 @@ const consumerConfig = { packageName: "blockpackage", messageType: "Block", }, - type: "synchronous", + type: "synchronous" }; // Start consuming messages from the Kafka topic. @@ -666,7 +659,7 @@ consume(consumerConfig, { closed: () => { Logger.info(`Subscription is ended.`); throw new Error("Consumer stopped."); - }, + } }); @@ -736,7 +729,7 @@ const loggerConfig = { }, winston: { // Any additional Winston configuration options can be provided here - }, + } }; // Create the singleton logger instance with the specified configuration From 8c6942829e016b17fef970939c62a25f7c383bf0 Mon Sep 17 00:00:00 2001 From: nivida Date: Mon, 28 Aug 2023 13:44:45 +0200 Subject: [PATCH 04/10] unused IKafkaCoderConfig import removed --- public/kafka/consumer/asynchronous_consumer.ts | 1 - public/kafka/consumer/synchronous_consumer.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/public/kafka/consumer/asynchronous_consumer.ts b/public/kafka/consumer/asynchronous_consumer.ts index 9e8701f..6243296 100644 --- a/public/kafka/consumer/asynchronous_consumer.ts +++ b/public/kafka/consumer/asynchronous_consumer.ts @@ -1,5 +1,4 @@ import { AsynchronousConsumer as InternalAsynchronousConsumer } from "@internal/kafka/consumer/asynchronous_consumer.js"; -import { IKafkaCoderConfig } from "@internal/interfaces/kafka_coder_config.js"; import { IConsumerConfig } from "@internal/interfaces/consumer_config.js"; import { Coder } from "@internal/coder/protobuf_coder.js"; import { ICoderConfig } from "@internal/interfaces/coder_config.js"; diff --git a/public/kafka/consumer/synchronous_consumer.ts b/public/kafka/consumer/synchronous_consumer.ts index f865dd3..b54506a 100644 --- a/public/kafka/consumer/synchronous_consumer.ts +++ b/public/kafka/consumer/synchronous_consumer.ts @@ -1,5 +1,4 @@ import { SynchronousConsumer as InternalSynchronousConsumer } from "@internal/kafka/consumer/synchronous_consumer.js"; -import { IKafkaCoderConfig } from "@internal/interfaces/kafka_coder_config.js"; import { IConsumerConfig } from "@internal/interfaces/consumer_config.js"; import { Coder } from "@internal/coder/protobuf_coder.js"; import { ICoderConfig } from "@internal/interfaces/coder_config.js"; From 5c6d229e509444b3911a821b7ca8aa726f705c7d Mon Sep 17 00:00:00 2001 From: nivida Date: Mon, 28 Aug 2023 13:51:12 +0200 Subject: [PATCH 05/10] Empty arrow function removed --- internal/kafka/producer/synchronous_producer.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/kafka/producer/synchronous_producer.ts b/internal/kafka/producer/synchronous_producer.ts index c05179d..0c5e324 100644 --- a/internal/kafka/producer/synchronous_producer.ts +++ b/internal/kafka/producer/synchronous_producer.ts @@ -59,7 +59,8 @@ export class SynchronousProducer extends AbstractProducer { timestamp?: number ): Promise { return new Promise(async (resolve, reject) => { - let deliveryListener: (error: LibrdKafkaError, report: DeliveryReport) => void = () => { }; + let deliveryListener: (error: LibrdKafkaError, report: DeliveryReport) => void; + try { const identifier = { time: Date.now() }; const timer = setTimeout(() => { From 90a4480868d66f882f60709c892dd93fd765418a Mon Sep 17 00:00:00 2001 From: nivida Date: Mon, 28 Aug 2023 14:01:09 +0200 Subject: [PATCH 06/10] type declaration fixed --- internal/kafka/producer/synchronous_producer.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/kafka/producer/synchronous_producer.ts b/internal/kafka/producer/synchronous_producer.ts index 0c5e324..e2ea6bc 100644 --- a/internal/kafka/producer/synchronous_producer.ts +++ b/internal/kafka/producer/synchronous_producer.ts @@ -59,8 +59,8 @@ export class SynchronousProducer extends AbstractProducer { timestamp?: number ): Promise { return new Promise(async (resolve, reject) => { - let deliveryListener: (error: LibrdKafkaError, report: DeliveryReport) => void; - + let deliveryListener: undefined | ((error: LibrdKafkaError, report: DeliveryReport) => void); + try { const identifier = { time: Date.now() }; const timer = setTimeout(() => { @@ -89,7 +89,9 @@ export class SynchronousProducer extends AbstractProducer { return; } }; + this.on("delivery-report", deliveryListener); + await this.sendToInternalProducer( key, message, @@ -103,7 +105,9 @@ export class SynchronousProducer extends AbstractProducer { this.poll(); }, 100); } catch (error) { - this.removeListener("delivery-report", deliveryListener); + if (deliveryListener) { + this.removeListener("delivery-report", deliveryListener); + } throw KafkaError.createUnknown(error); } From 0efa2c5b66915e9b0b0ffbcecf6f95aab3ea5ff4 Mon Sep 17 00:00:00 2001 From: nivida Date: Mon, 28 Aug 2023 14:15:34 +0200 Subject: [PATCH 07/10] fix --- internal/kafka/producer/synchronous_producer.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/kafka/producer/synchronous_producer.ts b/internal/kafka/producer/synchronous_producer.ts index e2ea6bc..12c57ba 100644 --- a/internal/kafka/producer/synchronous_producer.ts +++ b/internal/kafka/producer/synchronous_producer.ts @@ -59,7 +59,8 @@ export class SynchronousProducer extends AbstractProducer { timestamp?: number ): Promise { return new Promise(async (resolve, reject) => { - let deliveryListener: undefined | ((error: LibrdKafkaError, report: DeliveryReport) => void); + // @ts-ignore + let deliveryListener: (error: LibrdKafkaError, report: DeliveryReport) => void; try { const identifier = { time: Date.now() }; From 3485316f762fd11c0a84d9a31deb6c75867a9faa Mon Sep 17 00:00:00 2001 From: nivida Date: Mon, 28 Aug 2023 14:18:19 +0200 Subject: [PATCH 08/10] ts-ignore added because the type is only used internally --- internal/kafka/producer/synchronous_producer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/kafka/producer/synchronous_producer.ts b/internal/kafka/producer/synchronous_producer.ts index 12c57ba..f39250f 100644 --- a/internal/kafka/producer/synchronous_producer.ts +++ b/internal/kafka/producer/synchronous_producer.ts @@ -106,6 +106,7 @@ export class SynchronousProducer extends AbstractProducer { this.poll(); }, 100); } catch (error) { + // @ts-ignore if (deliveryListener) { this.removeListener("delivery-report", deliveryListener); } From d61c568fc471dc58680e683d7e88f13ab4e5c88b Mon Sep 17 00:00:00 2001 From: krishna9810 Date: Thu, 21 Sep 2023 20:11:06 +0530 Subject: [PATCH 09/10] quick node block getter changes --- .../abstract_block_subscription.ts | 6 ++++-- .../block_subscription/block_subscription.ts | 5 +++-- internal/interfaces/block_producer_config.ts | 1 + .../block_producers/quicknode_block_producer.ts | 9 ++++++--- schemas/global_exit_root.proto | 13 +++++++++++++ .../quicknode_block_producer.test.ts | 3 ++- .../abstract_block_subscription.test.ts | 16 +++++++++++++++- 7 files changed, 44 insertions(+), 9 deletions(-) create mode 100644 schemas/global_exit_root.proto diff --git a/internal/block_subscription/abstract_block_subscription.ts b/internal/block_subscription/abstract_block_subscription.ts index 2a5f5e3..855c070 100644 --- a/internal/block_subscription/abstract_block_subscription.ts +++ b/internal/block_subscription/abstract_block_subscription.ts @@ -42,7 +42,7 @@ export abstract class AbstractBlockSubscription extends Queue 0 + ? (await this.eth.getBlock('latest')).number - this.blockDelay + : (await this.eth.getBlock('finalized')).number; this.nextBlock = startBlock; this.lastBlockHash = ""; this.lastReceivedBlockNumber = startBlock - 1; diff --git a/internal/block_subscription/block_subscription.ts b/internal/block_subscription/block_subscription.ts index 97cdfcb..798d225 100644 --- a/internal/block_subscription/block_subscription.ts +++ b/internal/block_subscription/block_subscription.ts @@ -30,9 +30,10 @@ export class BlockSubscription extends AbstractBlockSubscription { protected rpcWsEndpoints: string[] = [], protected maxRetries: number = 0, private blockGetterType: "quicknode_block_getter" | "erigon_block_getter" | "block_getter" = "block_getter", - timeout?: number + timeout?: number, + blockDelay?: number, ) { - super(eth, timeout); + super(eth, timeout, blockDelay); this.setWorkers(); } diff --git a/internal/interfaces/block_producer_config.ts b/internal/interfaces/block_producer_config.ts index e625c50..780fbba 100644 --- a/internal/interfaces/block_producer_config.ts +++ b/internal/interfaces/block_producer_config.ts @@ -8,4 +8,5 @@ export interface IBlockProducerConfig extends IProducerConfig { maxRetries?: number, blockPollingTimeout?: number, blockSubscriptionTimeout?: number, + blockDelay?: number } diff --git a/public/block_producers/quicknode_block_producer.ts b/public/block_producers/quicknode_block_producer.ts index 138d415..761ee14 100644 --- a/public/block_producers/quicknode_block_producer.ts +++ b/public/block_producers/quicknode_block_producer.ts @@ -3,7 +3,7 @@ import { IProducedBlock, ProducedBlocksModel, IProducedBlocksModel } from "@inte import { BlockSubscription } from "@internal/block_subscription/block_subscription.js"; import { IBlockProducerConfig } from "@internal/interfaces/block_producer_config.js"; import { IProducerConfig } from "@internal/interfaces/producer_config.js"; -import { BlockGetter } from "@internal/block_getters/block_getter.js"; +import { QuickNodeBlockGetter } from "@internal/block_getters/quicknode_block_getter.js"; import { Coder } from "@internal/coder/protobuf_coder.js"; import { Database } from "@internal/mongo/database.js"; import Eth from "web3-eth"; @@ -29,6 +29,7 @@ export class QuickNodeBlockProducer extends BlockProducer { const mongoUrl = config.mongoUrl || "mongodb://localhost:27017/chain-flow"; const maxReOrgDepth = config.maxReOrgDepth || 0; const maxRetries = config.maxRetries || 0; + const blockDelay = config.blockDelay || 0; const blockSubscriptionTimeout = config.blockSubscriptionTimeout; // Has to be done or Kafka complains later @@ -37,6 +38,7 @@ export class QuickNodeBlockProducer extends BlockProducer { delete config.mongoUrl; delete config.maxReOrgDepth; delete config.maxRetries; + delete config.blockDelay; delete config.blockSubscriptionTimeout; //@ts-ignore @@ -67,9 +69,10 @@ export class QuickNodeBlockProducer extends BlockProducer { endpoints, maxRetries, "quicknode_block_getter", - blockSubscriptionTimeout + blockSubscriptionTimeout, + blockDelay ), - new BlockGetter(eth, maxRetries), + new QuickNodeBlockGetter(eth, maxRetries), database, database.model>( "ProducedBlocks", diff --git a/schemas/global_exit_root.proto b/schemas/global_exit_root.proto new file mode 100644 index 0000000..b31c929 --- /dev/null +++ b/schemas/global_exit_root.proto @@ -0,0 +1,13 @@ +package globalexitrootpackage; +syntax = "proto3"; + +message ExitRoots { + string mainnet = 1; + string rollUp = 2; +} + +message GlobalExitRoot { + uint64 blockNumber = 1; + uint64 timestamp = 2; + repeated ExitRoots data = 3; +} diff --git a/tests/block_producer/quicknode_block_producer.test.ts b/tests/block_producer/quicknode_block_producer.test.ts index cfcd4f2..d82c30e 100644 --- a/tests/block_producer/quicknode_block_producer.test.ts +++ b/tests/block_producer/quicknode_block_producer.test.ts @@ -139,7 +139,8 @@ describe("Block Producer", () => { ["rpc.com", "rpc2.com"], 0, "quicknode_block_getter", - 60000 + 60000, + 0 ); }); diff --git a/tests/block_subscription/abstract_block_subscription.test.ts b/tests/block_subscription/abstract_block_subscription.test.ts index 477aa02..2215d9b 100644 --- a/tests/block_subscription/abstract_block_subscription.test.ts +++ b/tests/block_subscription/abstract_block_subscription.test.ts @@ -62,7 +62,7 @@ describe("Abstract Block Subscription", () => { subscriber.unsubscribe(); }); - test("Must call get block to determine if backfilling is required", async () => { + test("Must call get block(finalized) to determine if backfilling is required", async () => { mockedEthObject.getBlock.mockResolvedValueOnce({ number: 0 } as BlockTransactionObject); expect( await subscriber.subscribe(observer, 0) @@ -71,6 +71,20 @@ describe("Abstract Block Subscription", () => { expect(mockedEthObject.getBlock).toBeCalledWith("finalized"); }); + test("Must call get block(latest) to determine if backfilling is required and if block delay is greater than 0", async () => { + subscriber = new BlockSubscription( + mockedEthObject, + 60000, + 256 + ); + mockedEthObject.getBlock.mockResolvedValueOnce({ number: 0 } as BlockTransactionObject); + expect( + await subscriber.subscribe(observer, 0) + ).toEqual(undefined); + + expect(mockedEthObject.getBlock).toBeCalledWith("latest"); + }); + test("If the difference between last block and finalized block is more than 50, log subscription must not be called but backfill", async () => { mockedEthObject.getBlock.mockResolvedValueOnce({ number: 51 } as BlockTransactionObject); expect( From 6d3243400988f63dea90ea9eadf11a1ef86960f4 Mon Sep 17 00:00:00 2001 From: Nitin Mittal Date: Fri, 22 Sep 2023 16:12:44 +0400 Subject: [PATCH 10/10] fix: tests --- public/block_producers/quicknode_block_producer.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/public/block_producers/quicknode_block_producer.ts b/public/block_producers/quicknode_block_producer.ts index 00683f8..6d805fa 100644 --- a/public/block_producers/quicknode_block_producer.ts +++ b/public/block_producers/quicknode_block_producer.ts @@ -29,7 +29,6 @@ export class QuickNodeBlockProducer extends BlockProducer { const mongoUrl = config.mongoUrl || "mongodb://localhost:27017/chain-flow"; const maxReOrgDepth = config.maxReOrgDepth || 0; const maxRetries = config.maxRetries || 0; - const blockDelay = config.blockDelay || 0; const blockSubscriptionTimeout = config.blockSubscriptionTimeout; const blockDelay = config.blockDelay || 0; const alternateEndpoint = config.alternateEndpoint;