Skip to content

Commit

Permalink
Throttle block consumption (#156)
Browse files Browse the repository at this point in the history
# Description
This PR adds a new optional config parameter to each network, allowing
us to define some block throttling.

the setting is called `processEveryNumBlocks` : 
* Throttle block processing to only process blocks every N blocks. Set
to 1 to process every block, 2 to process every other block, etc.

Also, note that even if we skip a block, it doesn't mean we don't have
any work on it. We still need to check if there's new orders coming in.
So what is actually skipped is only the programmatic order checks and
posting orders.

Also, it persists all Prometheus metrics, so it should count as a
"consumed block" for our consumption rate alerts, meaning we should see
the block consumption rate progressing faster.

<img width="1708" alt="image"
src="https://github.com/user-attachments/assets/3e4e9b96-c141-416d-b399-e315abc355f8">



This is an example on how to use it: `"processEveryNumBlocks": 40,`
This would do polling and post orders once every 40 blocks. In arbitrum
this should be every 8 seconds

```yaml
{
  "networks": [
    {      
      "name": "arbitrum-one",
      "rpc": "wss://your-rpc/arbitrum",
      "deploymentBlock": 204704802,
      "processEveryNumBlocks": 40,
      "filterPolicy": {
        "defaultAction": "DROP",
        "handlers": {
          "0x44569Cbd4E10dd5e97293337964Eff32d58ed352": "ACCEPT",
          "0x519BA24e959E33b3B6220CA98bd353d8c2D89920": "ACCEPT",
          "0x6cF1e9cA41f7611dEf408122793c358a3d11E5a5": "ACCEPT",
          "0xd3338f21c89745e46af56aeaf553cf96ba9bc66f": "ACCEPT",
          "0xE8212F30C28B4AAB467DF3725C14d6e89C2eB967": "ACCEPT",
          "0xB148F40fff05b5CE6B22752cf8E454B556f7a851": "ACCEPT"
        }
      },
      "watchdogTimeout": 120
    }
  ]
}
```
  • Loading branch information
anxolin authored Jul 22, 2024
1 parent ac00dd2 commit 90ecbf5
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 24 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM node:alpine AS build
FROM node:22.2-alpine AS build

WORKDIR /usr/src/app

Expand Down
59 changes: 36 additions & 23 deletions src/services/chain.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ export class ChainContext {
readonly dryRun: boolean;
readonly watchdogTimeout: number;
readonly addresses?: string[];
readonly processEveryNumBlocks: number;

private sync: ChainSync = ChainSync.SYNCING;
static chains: Chains = {};

Expand Down Expand Up @@ -106,6 +108,7 @@ export class ChainContext {
this.deploymentBlock = deploymentBlock;
this.pageSize = pageSize ?? PAGE_SIZE_DEFAULT;
this.dryRun = dryRun;
this.processEveryNumBlocks = options.processEveryNumBlocks ?? 1;
this.watchdogTimeout = watchdogTimeout ?? WATCHDOG_TIMEOUT_DEFAULT_SECS;
this.addresses = owners;

Expand Down Expand Up @@ -168,7 +171,7 @@ export class ChainContext {
* @returns the run promises for what needs to be watched
*/
public async warmUp(oneShot?: boolean) {
const { provider, chainId } = this;
const { provider, chainId, processEveryNumBlocks } = this;
const log = getLogger("chainContext:warmUp", chainId.toString());
let { lastProcessedBlock } = this.registry;
const { pageSize } = this;
Expand Down Expand Up @@ -213,7 +216,13 @@ export class ChainContext {
log.info(
`🔄 Start sync with from block ${fromBlock} to ${toBlock}. Pending ${
toBlock - fromBlock
} blocks (~${Math.ceil((toBlock - fromBlock) / pageSize)} pages)`
} blocks (~${Math.ceil(
(toBlock - fromBlock) / pageSize
)} pages, processing every ${
processEveryNumBlocks > 1
? processEveryNumBlocks + " blocks"
: "block"
})`
);
}

Expand Down Expand Up @@ -311,9 +320,8 @@ export class ChainContext {
let lastBlockReceived = lastProcessedBlock;
provider.on("block", async (blockNumber: number) => {
try {
const block = await provider.getBlock(blockNumber);

log.debug(`New block ${blockNumber}`);
const block = await provider.getBlock(blockNumber);

// Set the block time metric
const _blockTime = block.timestamp - lastBlockReceived.timestamp;
Expand Down Expand Up @@ -434,7 +442,7 @@ async function processBlock(
blockNumberOverride?: number,
blockTimestampOverride?: number
) {
const { provider, chainId } = context;
const { provider, chainId, processEveryNumBlocks } = context;
const timer = metrics.processBlockDurationSeconds
.labels(context.chainId.toString())
.startTimer();
Expand Down Expand Up @@ -463,24 +471,29 @@ async function processBlock(
}
}

// run action
const result = await checkForAndPlaceOrder(
context,
block,
blockNumberOverride,
blockTimestampOverride
)
.then(() => true)
.catch(() => {
hasErrors = true;
log.error(`Error running "checkForAndPlaceOrder" action`);
return false;
});
log.debug(
`Result of "checkForAndPlaceOrder" action for block ${
block.number
}: ${_formatResult(result)}`
);
// Decide if we should process this block
const shouldProcessBlock = block.number % processEveryNumBlocks === 0;

// Check programmatic orders and place orders if necessary
if (shouldProcessBlock) {
const result = await checkForAndPlaceOrder(
context,
block,
blockNumberOverride,
blockTimestampOverride
)
.then(() => true)
.catch(() => {
hasErrors = true;
log.error(`Error running "checkForAndPlaceOrder" action`);
return false;
});
log.debug(
`Result of "checkForAndPlaceOrder" action for block ${
block.number
}: ${_formatResult(result)}`
);
}

timer();
if (hasErrors) {
Expand Down
6 changes: 6 additions & 0 deletions src/types/config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@
"watchdogTimeout": {
"type": "integer"
},
"processEveryNumBlocks": {
"type": "integer",
"minimum": 1,
"description": "Throttle block processing to only process blocks every N blocks. Set to 1 to process every block (default), 2 to process every other block, etc.",
"default": 1
},
"orderBookApi": {
"type": "string",
"format": "uri"
Expand Down
4 changes: 4 additions & 0 deletions src/types/types.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ export interface Config {
rpc: string;
deploymentBlock: number;
watchdogTimeout?: number;
/**
* Throttle block processing to only process blocks every N blocks. Set to 1 to process every block, 2 to process every other block, etc.
*/
processEveryNumBlocks?: number;
orderBookApi?: string;
pageSize?: number;
filterPolicy: {
Expand Down

0 comments on commit 90ecbf5

Please sign in to comment.