diff --git a/.github/workflows/security-build.yml b/.github/workflows/security-build.yml new file mode 100644 index 0000000..62203a6 --- /dev/null +++ b/.github/workflows/security-build.yml @@ -0,0 +1,32 @@ +name: Security Build +on: + push: + branches: + - main # or the name of your main branch + workflow_dispatch: {} + +jobs: + sonarqube: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # Disabling shallow clone is recommended for improving relevancy of reporting. + fetch-depth: 0 + + # Triggering SonarQube analysis as results of it are required by Quality Gate check. + - name: SonarQube Scan + uses: sonarsource/sonarqube-scan-action@master + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + + # Check the Quality Gate status. + - name: SonarQube Quality Gate check + id: sonarqube-quality-gate-check + uses: sonarsource/sonarqube-quality-gate-action@master + # Force to fail step after specific time. + timeout-minutes: 5 + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} diff --git a/README.md b/README.md index 02ba151..a9a3cf2 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ Chain Indexer Framework block producers encompass three distinct types of produc "bootstrap.servers": '', "security.protocol": "plaintext", blockSubscriptionTimeout: 120000, - type: 'blocks:poller'. + type: 'blocks:poller', { error: (error: KafkaError | BlockProducerError) => {}, closed: () => {} // On broker connection closed @@ -215,7 +215,7 @@ Chain Indexer Framework block producers encompass three distinct types of produc maxRetries: '', mongoUrl: '', "bootstrap.servers": '', - "security.protocol": "plaintext", + "security.protocol": "plaintext" }) producer.on("blockProducer.fatalError", (error) => { @@ -275,7 +275,7 @@ const producer = new SynchronousProducer( coder: { fileName: "matic_transfer", packageName: "matictransferpackage", - messageType: "MaticTransferBlock", + messageType: "MaticTransferBlock" } } ); @@ -306,11 +306,11 @@ const producer = produce( packageName: "matictransferpackage", messageType: "MaticTransferBlock", }, - type: "synchronous" // use 'synchronous'. if synchronous producer is needed, + type: "synchronous", // use 'synchronous'. if synchronous producer is needed { emitter: () => { this.produceEvent("", ""); - } + }, error: (error: KafkaError | BlockProducerError) => {}, closed: () => {} // On broker connection closed } @@ -336,14 +336,7 @@ const producer = new AsynchronousProducer( coder: { fileName: "matic_transfer", packageName: "matictransferpackage", - messageType: "MaticTransferBlock", - }, - { - emitter: () => { - this.produceEvent("", ""); - } - error: (error: KafkaError | BlockProducerError) => {}, - closed: () => {} // On broker connection closed + messageType: "MaticTransferBlock" } } ); @@ -372,13 +365,13 @@ const producer = produce( coder: { fileName: "matic_transfer", packageName: "matictransferpackage", - messageType: "MaticTransferBlock", + messageType: "MaticTransferBlock" }, - type: "asynchronous" + type: "asynchronous", { emitter: () => { this.produceEvent("", ""); - } + }, error: (error: KafkaError | BlockProducerError) => {}, closed: () => {} // On broker connection closed } @@ -484,7 +477,7 @@ const consumer = new SynchronousConsumer( coders: { fileName: "block", packageName: "blockpackage", - messageType: "Block", + messageType: "Block" } } ); @@ -512,7 +505,7 @@ consume( "coderConfig": { fileName: "block", packageName: "blockpackage", - messageType: "Block", + messageType: "Block" }, type: 'synchronous' }, @@ -545,7 +538,7 @@ const consumer = new AsynchronousConsumer( coders: { fileName: "block", packageName: "blockpackage", - messageType: "Block", + messageType: "Block" } } ); @@ -651,7 +644,7 @@ const consumerConfig = { packageName: "blockpackage", messageType: "Block", }, - type: "synchronous", + type: "synchronous" }; // Start consuming messages from the Kafka topic. @@ -670,7 +663,7 @@ consume(consumerConfig, { closed: () => { Logger.info(`Subscription is ended.`); throw new Error("Consumer stopped."); - }, + } }); @@ -740,7 +733,7 @@ const loggerConfig = { }, winston: { // Any additional Winston configuration options can be provided here - }, + } }; // Create the singleton logger instance with the specified configuration diff --git a/internal/kafka/producer/synchronous_producer.ts b/internal/kafka/producer/synchronous_producer.ts index c05179d..f39250f 100644 --- a/internal/kafka/producer/synchronous_producer.ts +++ b/internal/kafka/producer/synchronous_producer.ts @@ -59,7 +59,9 @@ export class SynchronousProducer extends AbstractProducer { timestamp?: number ): Promise { return new Promise(async (resolve, reject) => { - let deliveryListener: (error: LibrdKafkaError, report: DeliveryReport) => void = () => { }; + // @ts-ignore + let deliveryListener: (error: LibrdKafkaError, report: DeliveryReport) => void; + try { const identifier = { time: Date.now() }; const timer = setTimeout(() => { @@ -88,7 +90,9 @@ export class SynchronousProducer extends AbstractProducer { return; } }; + this.on("delivery-report", deliveryListener); + await this.sendToInternalProducer( key, message, @@ -102,7 +106,10 @@ export class SynchronousProducer extends AbstractProducer { this.poll(); }, 100); } catch (error) { - this.removeListener("delivery-report", deliveryListener); + // @ts-ignore + if (deliveryListener) { + this.removeListener("delivery-report", deliveryListener); + } throw KafkaError.createUnknown(error); } diff --git a/public/block_producers/quicknode_block_producer.ts b/public/block_producers/quicknode_block_producer.ts index d50e4c6..6d805fa 100644 --- a/public/block_producers/quicknode_block_producer.ts +++ b/public/block_producers/quicknode_block_producer.ts @@ -3,7 +3,7 @@ import { IProducedBlock, ProducedBlocksModel, IProducedBlocksModel } from "@inte import { BlockSubscription } from "@internal/block_subscription/block_subscription.js"; import { IBlockProducerConfig } from "@internal/interfaces/block_producer_config.js"; import { IProducerConfig } from "@internal/interfaces/producer_config.js"; -import { BlockGetter } from "@internal/block_getters/block_getter.js"; +import { QuickNodeBlockGetter } from "@internal/block_getters/quicknode_block_getter.js"; import { Coder } from "@internal/coder/protobuf_coder.js"; import { Database } from "@internal/mongo/database.js"; import Eth from "web3-eth"; @@ -40,6 +40,7 @@ export class QuickNodeBlockProducer extends BlockProducer { delete config.mongoUrl; delete config.maxReOrgDepth; delete config.maxRetries; + delete config.blockDelay; delete config.blockSubscriptionTimeout; delete config.blockDelay; delete config.alternateEndpoint; @@ -79,7 +80,7 @@ export class QuickNodeBlockProducer extends BlockProducer { alternateEndpoint, rpcTimeout ), - new BlockGetter(eth, maxRetries), + new QuickNodeBlockGetter(eth, maxRetries), database, database.model>( "ProducedBlocks", diff --git a/public/kafka/consumer/asynchronous_consumer.ts b/public/kafka/consumer/asynchronous_consumer.ts index 9e8701f..6243296 100644 --- a/public/kafka/consumer/asynchronous_consumer.ts +++ b/public/kafka/consumer/asynchronous_consumer.ts @@ -1,5 +1,4 @@ import { AsynchronousConsumer as InternalAsynchronousConsumer } from "@internal/kafka/consumer/asynchronous_consumer.js"; -import { IKafkaCoderConfig } from "@internal/interfaces/kafka_coder_config.js"; import { IConsumerConfig } from "@internal/interfaces/consumer_config.js"; import { Coder } from "@internal/coder/protobuf_coder.js"; import { ICoderConfig } from "@internal/interfaces/coder_config.js"; diff --git a/public/kafka/consumer/synchronous_consumer.ts b/public/kafka/consumer/synchronous_consumer.ts index f865dd3..b54506a 100644 --- a/public/kafka/consumer/synchronous_consumer.ts +++ b/public/kafka/consumer/synchronous_consumer.ts @@ -1,5 +1,4 @@ import { SynchronousConsumer as InternalSynchronousConsumer } from "@internal/kafka/consumer/synchronous_consumer.js"; -import { IKafkaCoderConfig } from "@internal/interfaces/kafka_coder_config.js"; import { IConsumerConfig } from "@internal/interfaces/consumer_config.js"; import { Coder } from "@internal/coder/protobuf_coder.js"; import { ICoderConfig } from "@internal/interfaces/coder_config.js"; diff --git a/schemas/global_exit_root.proto b/schemas/global_exit_root.proto new file mode 100644 index 0000000..b31c929 --- /dev/null +++ b/schemas/global_exit_root.proto @@ -0,0 +1,13 @@ +package globalexitrootpackage; +syntax = "proto3"; + +message ExitRoots { + string mainnet = 1; + string rollUp = 2; +} + +message GlobalExitRoot { + uint64 blockNumber = 1; + uint64 timestamp = 2; + repeated ExitRoots data = 3; +} diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..6c791a9 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1 @@ +sonar.projectKey=maticnetwork_chainflow_AYn0jzKkuJjRFCFKZwPS diff --git a/tests/block_subscription/abstract_block_subscription.test.ts b/tests/block_subscription/abstract_block_subscription.test.ts index 477aa02..2215d9b 100644 --- a/tests/block_subscription/abstract_block_subscription.test.ts +++ b/tests/block_subscription/abstract_block_subscription.test.ts @@ -62,7 +62,7 @@ describe("Abstract Block Subscription", () => { subscriber.unsubscribe(); }); - test("Must call get block to determine if backfilling is required", async () => { + test("Must call get block(finalized) to determine if backfilling is required", async () => { mockedEthObject.getBlock.mockResolvedValueOnce({ number: 0 } as BlockTransactionObject); expect( await subscriber.subscribe(observer, 0) @@ -71,6 +71,20 @@ describe("Abstract Block Subscription", () => { expect(mockedEthObject.getBlock).toBeCalledWith("finalized"); }); + test("Must call get block(latest) to determine if backfilling is required and if block delay is greater than 0", async () => { + subscriber = new BlockSubscription( + mockedEthObject, + 60000, + 256 + ); + mockedEthObject.getBlock.mockResolvedValueOnce({ number: 0 } as BlockTransactionObject); + expect( + await subscriber.subscribe(observer, 0) + ).toEqual(undefined); + + expect(mockedEthObject.getBlock).toBeCalledWith("latest"); + }); + test("If the difference between last block and finalized block is more than 50, log subscription must not be called but backfill", async () => { mockedEthObject.getBlock.mockResolvedValueOnce({ number: 51 } as BlockTransactionObject); expect(