Skip to content

Commit

Permalink
Merge branch 'main' into chore/switch-to-frequency-standalone-node
Browse files Browse the repository at this point in the history
  • Loading branch information
@JoeCap08055
JoeCap08055 authored Oct 25, 2024
2 parents 0dcd614 + d1f4cf0 commit 73b7a28
Show file tree
Hide file tree
Showing 3 changed files with 133 additions and 46 deletions.
106 changes: 80 additions & 26 deletions libs/account-lib/src/utils/blockchain-scanner.service.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,55 @@
import { Injectable, Logger } from '@nestjs/common';
import { Test } from '@nestjs/testing';
import { Hash, SignedBlock } from '@polkadot/types/interfaces';
import { BlockchainService } from '#blockchain/blockchain.service';
import { BlockHash, BlockNumber, SignedBlock } from '@polkadot/types/interfaces';
import { DEFAULT_REDIS_NAMESPACE, getRedisToken, InjectRedis } from '@songkeys/nestjs-redis';
import { Redis } from 'ioredis';
import { FrameSystemEventRecord } from '@polkadot/types/lookup';
import { BlockchainScannerService } from './blockchain-scanner.service';
import { FrameSystemEventRecord } from '@polkadot/types/lookup';
import { mockApiPromise } from '#testlib/polkadot-api.mock.spec';
import { BlockchainService, NONCE_SERVICE_REDIS_NAMESPACE } from '#blockchain/blockchain.service';
import { IBlockchainNonProviderConfig } from '#blockchain/blockchain.config';
import { GenerateMockConfigProvider } from '#testlib/utils.config-tests';
import { EventEmitterModule } from '@nestjs/event-emitter';
import { AnyNumber } from '@polkadot/types/types';

jest.mock('@polkadot/api', () => {
const originalModule = jest.requireActual<typeof import('@polkadot/api')>('@polkadot/api');
return {
__esModules: true,
WsProvider: jest.fn().mockImplementation(() => originalModule.WsProvider),
ApiPromise: jest.fn().mockImplementation(() => ({
...originalModule.ApiPromise,
...mockApiPromise,
})),
};
});
const mockBlockchainConfigProvider = GenerateMockConfigProvider<IBlockchainNonProviderConfig>('blockchain', {
frequencyTimeoutSecs: 10,
frequencyApiWsUrl: new URL('ws://localhost:9944'),
isDeployedReadOnly: false,
});

const mockRedis = {
get: jest.fn(),
set: jest.fn(),
defineCommand: jest.fn(),
};

const mockDefaultRedisProvider = {
provide: getRedisToken(DEFAULT_REDIS_NAMESPACE),
useValue: { get: jest.fn(), set: jest.fn() },
useValue: mockRedis,
};

const mockNonceRedisProvider = {
provide: getRedisToken(NONCE_SERVICE_REDIS_NAMESPACE),
useValue: mockRedis,
};

const mockBlockHash = {
toString: jest.fn(() => '0x1234'),
some: () => true,
};
isEmpty: false,
} as unknown as BlockHash;

const mockSignedBlock = {
block: {
Expand All @@ -26,29 +60,11 @@ const mockSignedBlock = {
},
};

Object.defineProperty(mockBlockHash, 'isEmpty', {
get: jest.fn(() => false),
});

const mockEmptyBlockHash = {
toString: jest.fn(() => '0x00000'),
some: () => false,
};
Object.defineProperty(mockEmptyBlockHash, 'isEmpty', {
get: jest.fn(() => true),
});
const mockBlockchainService = {
isReady: jest.fn(() => Promise.resolve()),
getBlock: jest.fn((_blockHash?: string | Hash) => mockSignedBlock as unknown as SignedBlock),
getBlockHash: jest.fn((blockNumber: number) => (blockNumber > 1 ? mockEmptyBlockHash : mockBlockHash)),
getLatestFinalizedBlockNumber: jest.fn(),
getEvents: jest.fn(() => []),
};

const mockBlockchainServiceProvider = {
provide: BlockchainService,
useValue: mockBlockchainService,
};
isEmpty: true,
} as unknown as BlockHash;

@Injectable()
class ScannerService extends BlockchainScannerService {
Expand All @@ -67,10 +83,48 @@ describe('BlockchainScannerService', () => {

beforeAll(async () => {
const moduleRef = await Test.createTestingModule({
providers: [mockRedis, Logger, mockBlockchainServiceProvider, ScannerService],
imports: [
EventEmitterModule.forRoot({
// Use this instance throughout the application
global: true,
// set this to `true` to use wildcards
wildcard: false,
// the delimiter used to segment namespaces
delimiter: '.',
// set this to `true` if you want to emit the newListener event
newListener: false,
// set this to `true` if you want to emit the removeListener event
removeListener: false,
// the maximum amount of listeners that can be assigned to an event
maxListeners: 10,
// show event name in memory leak message when more than maximum amount of listeners is assigned
verboseMemoryLeak: false,
// disable throwing uncaughtException if an error event is emitted and it has no listeners
ignoreErrors: false,
}),
],
providers: [
mockDefaultRedisProvider,
mockNonceRedisProvider,
mockBlockchainConfigProvider,
Logger,
BlockchainService,
ScannerService,
],
}).compile();
moduleRef.enableShutdownHooks();
service = moduleRef.get<ScannerService>(ScannerService);
blockchainService = moduleRef.get<BlockchainService>(BlockchainService);
const mockApi: any = await blockchainService.getApi();
jest.spyOn(blockchainService, 'getBlock').mockResolvedValue(mockSignedBlock as unknown as SignedBlock);
jest
.spyOn(blockchainService, 'getBlockHash')
.mockImplementation((blockNumber: BlockNumber | AnyNumber) =>
Promise.resolve((blockNumber as unknown as number) > 1 ? mockEmptyBlockHash : mockBlockHash),
);
jest.spyOn(blockchainService, 'getLatestFinalizedBlockNumber');
jest.spyOn(blockchainService, 'getEvents').mockResolvedValue([]);
mockApi.emit('connected'); // keeps the test suite from hanging when finished
});

describe('scan', () => {
Expand Down
50 changes: 38 additions & 12 deletions libs/account-lib/src/utils/blockchain-scanner.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,20 @@ export interface IBlockchainScanParameters {
}

export class EndOfChainError extends Error {}
export class SkipBlockError extends Error {}

function eventName({ event: { section, method } }: FrameSystemEventRecord) {
return `${section}.${method}`;
}

export abstract class BlockchainScannerService {
private scanIsPaused = false;

protected scanInProgress = false;

protected chainEventHandlers = new Map<
string,
((block: SignedBlock, event: FrameSystemEventRecord) => any | Promise<any>)[]
((block: SignedBlock, event: FrameSystemEventRecord) => unknown | Promise<unknown>)[]
>();

private readonly lastSeenBlockNumberKey: string;
Expand All @@ -37,6 +40,20 @@ export abstract class BlockchainScannerService {
protected readonly logger: Logger,
) {
this.lastSeenBlockNumberKey = `${this.constructor.name}:${LAST_SEEN_BLOCK_NUMBER_KEY}`;
this.blockchainService.on('chain.disconnected', () => {
this.paused = true;
});
this.blockchainService.on('chain.connected', () => {
this.paused = false;
});
}

protected get paused() {
return this.scanIsPaused;
}

private set paused(p: boolean) {
this.scanIsPaused = p;
}

public get scanParameters() {
Expand All @@ -57,7 +74,6 @@ export abstract class BlockchainScannerService {
}

try {
await this.blockchainService.isReady();
// Only scan blocks if initial conditions met
await this.checkInitialScanParameters();

Expand All @@ -76,26 +92,36 @@ export abstract class BlockchainScannerService {
this.logger.verbose(`Starting scan from block #${currentBlockNumber}`);

// eslint-disable-next-line no-constant-condition
while (true) {
await this.checkScanParameters(currentBlockNumber, currentBlockHash); // throws when end-of-chain reached
const block = await this.blockchainService.getBlock(currentBlockHash);
const blockEvents = await this.blockchainService.getEvents(currentBlockHash);
await this.handleChainEvents(block, blockEvents);
await this.processCurrentBlock(block, blockEvents);
while (!this.paused) {
try {
await this.checkScanParameters(currentBlockNumber, currentBlockHash); // throws when end-of-chain reached
const block = await this.blockchainService.getBlock(currentBlockHash);
const blockEvents = await this.blockchainService.getEvents(currentBlockHash);
await this.handleChainEvents(block, blockEvents);
await this.processCurrentBlock(block, blockEvents);
} catch (err) {
if (!(err instanceof SkipBlockError)) {
throw err;
}
this.logger.debug(`Skipping block ${currentBlockNumber}`);
}
await this.setLastSeenBlockNumber(currentBlockNumber);

// Move to the next block
currentBlockNumber += 1;
currentBlockHash = await this.blockchainService.getBlockHash(currentBlockNumber);
}
} catch (e: any) {
} catch (e) {
if (e instanceof EndOfChainError) {
this.logger.debug(e.message);
return;
}

this.logger.error('Unexpected error scanning chain', JSON.stringify(e), e?.stack);
throw e;
// Don't throw if scan paused; that's WHY it's paused
if (!this.paused) {
this.logger.error(JSON.stringify(e));
throw e;
}
} finally {
this.scanInProgress = false;
}
Expand Down Expand Up @@ -130,7 +156,7 @@ export abstract class BlockchainScannerService {

public registerChainEventHandler(
events: string[],
callback: (block: SignedBlock, blockEvents: FrameSystemEventRecord) => any | Promise<any>,
callback: (block: SignedBlock, blockEvents: FrameSystemEventRecord) => unknown | Promise<unknown>,
) {
events.forEach((event) => {
const handlers = new Set(this.chainEventHandlers.get(event) || []);
Expand Down
23 changes: 15 additions & 8 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,21 @@
"generate:swagger-ui:graph": "npx --yes @redocly/cli build-docs openapi-specs/graph.openapi.json --output=./docs/graph/index.html",
"pregenerate:swagger-ui:graph": "npx --yes @redocly/cli build-docs openapi-specs/graph-webhooks.openapi.yaml --output=./docs/graph/webhooks.html",
"generate:swagger-ui": "npm run generate:swagger-ui:account ; npm run generate:swagger-ui:content-publishing ; npm run generate:swagger-ui:content-watcher ; npm run generate:swagger-ui:graph",
"test:account": "dotenvx run -f env-files/account.template.env -- jest 'account*'",
"test:content-publishing": "dotenvx run -f env-files/content-publishing.template.env -- jest 'content-publishing*'",
"test:content-watcher": "dotenvx run -f env-files/content-watcher.template.env -- jest 'content-watcher*'",
"test:graph": "dotenvx run -f env-files/graph.template.env -- jest 'graph*'",
"test:libs:blockchain": "dotenvx run -f env-files/graph.template.env -- jest --runInBand 'libs/blockchain*'",
"test:libs:cache": "dotenvx run -f env-files/graph.template.env -- jest 'libs/cache*'",
"test:libs:utils": "jest 'libs/utils*'",
"test:libs": "npm run test:libs:blockchain ; npm run test:libs:cache; npm run test:libs:utils;",
"test:account": "dotenvx run -f env-files/account.template.env -- jest 'apps/account-api' 'apps/account-worker'",
"test:content-publishing": "dotenvx run -f env-files/content-publishing.template.env -- jest 'apps/content-publishing-api' 'apps/content-publishing-worker'",
"test:content-watcher": "dotenvx run -f env-files/content-watcher.template.env -- jest 'apps/content-watcher'",
"test:graph": "dotenvx run -f env-files/graph.template.env -- jest 'apps/graph-api' 'apps/graph-worker'",
"test:libs:account": "dotenvx run -f env-files/account.template.env -- jest 'libs/account-lib'",
"test:libs:blockchain": "dotenvx run -f env-files/graph.template.env -- jest 'libs/blockchain'",
"test:libs:cache": "dotenvx run -f env-files/graph.template.env -- jest 'libs/cache'",
"test:libs:config": "dotenvx run -f env-files/graph.template.env -- jest 'libs/config'",
"test:libs:content-publishing": "dotenvx run -f env-files/content-publishing.template.env -- jest 'libs/content-publishing-lib'",
"test:libs:content-watcher": "dotenvx run -f env-files/content-watcher.template.env -- jest 'libs/content-watcher-lib'",
"test:libs:graph": "dotenvx run -f env-files/graph.template.env -- jest 'libs/graph-lib'",
"test:libs:queue": "dotenvx run -f env-files/graph.template.env -- jest 'libs/queue'",
"test:libs:storage": "dotenvx run -f env-files/content-publishing.template.env -- jest 'libs/storage'",
"test:libs:utils": "jest 'libs/utils'",
"test:libs": "dotenvx run -f env-files/graph.template.env -f env-files/content-publishing.template.env -- jest 'libs/'",
"test": "npm run test:account ; npm run test:content-publishing ; npm run test:content-watcher ; npm run test:graph ; npm run test:libs",
"test:verbose": "jest --coverage --verbose",
"test:e2e:account": "dotenvx run -f env-files/account.template.env -- jest --runInBand --detectOpenHandles --testRegex 'account-api/.*\\.e2e-spec\\.ts'",
Expand Down

0 comments on commit 73b7a28

Please sign in to comment.