Refactor content-watcher service configs (#554)

This PR refactors `ConfigModule` and `ConfigService` for
`account-service`.

- Eliminate custom config service class and rely simply on built-in
NestJS `ConfigModule`, using namespaced configs for individual modules
- Move config property configuration into module-specific config, ie
'blockchain', 'cache', 'queue', 'account-api', 'account-worker'. (NOTE,
as each module is refactored in the future, these module configs will be
merged with the corresponding module configs from the other services,
but for now they will still be duplicated across the services)
- Update & add unit tests to cover new config setup

Closes #547

apps/content-watcher/src/api.config.spec.ts

@@ -0,0 +1,37 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { describe, it, expect, beforeAll } from '@jest/globals';
+import apiConfig, { IContentWatcherApiConfig } from './api.config';
+import configSetup from '#testlib/utils.config-tests';
+
+const { setupConfigService, shouldFailBadValues } = configSetup<IContentWatcherApiConfig>(apiConfig);
+
+describe('Content Watcher API config', () => {
+  const ALL_ENV: { [key: string]: string | undefined } = {
+    API_PORT: undefined,
+  };
+
+  beforeAll(() => {
+    Object.keys(ALL_ENV).forEach((key) => {
+      ALL_ENV[key] = process.env[key];
+    });
+  });
+
+  describe('invalid environment', () => {
+    it('invalid API port should fail', async () => shouldFailBadValues(ALL_ENV, 'API_PORT', [-1, 'bad port']));
+  });
+
+  describe('valid environment', () => {
+    let apiConf: IContentWatcherApiConfig;
+    beforeAll(async () => {
+      apiConf = await setupConfigService(ALL_ENV);
+    });
+
+    it('should be defined', () => {
+      expect(apiConf).toBeDefined();
+    });
+
+    it('should get API port', async () => {
+      expect(apiConf.apiPort).toStrictEqual(parseInt(ALL_ENV.API_PORT, 10));
+    });
+  });
+});

apps/content-watcher/src/api.config.ts

@@ -0,0 +1,18 @@
+import { JoiUtils } from '#config';
+import { registerAs } from '@nestjs/config';
+import Joi from 'joi';
+
+export interface IContentWatcherApiConfig {
+  apiPort: number;
+}
+
+export default registerAs('api', (): IContentWatcherApiConfig => {
+  const configs: JoiUtils.JoiConfig<IContentWatcherApiConfig> = {
+    apiPort: {
+      value: process.env.API_PORT,
+      joi: Joi.number().min(0).default(3000),
+    },
+  };
+
+  return JoiUtils.validate<IContentWatcherApiConfig>(configs);
+});

apps/content-watcher/src/api.module.ts

@@ -11,26 +11,33 @@ import { CrawlerModule } from '#content-watcher-lib/crawler/crawler.module';
 import { IPFSProcessorModule } from '#content-watcher-lib/ipfs/ipfs.module';
 import { PubSubModule } from '#content-watcher-lib/pubsub/pubsub.module';
 import { ScannerModule } from '#content-watcher-lib/scanner/scanner.module';
-import { AppConfigModule } from '#content-watcher-lib/config/config.module';
-import { AppConfigService } from '#content-watcher-lib/config/config.service';
 import { ContentWatcherQueues as QueueConstants } from '#types/constants/queue.constants';
 import { QueueModule } from '#content-watcher-lib/queues/queue.module';
 import { CacheModule } from '#content-watcher-lib/cache/cache.module';
+import cacheConfig, { ICacheConfig } from '#content-watcher-lib/cache/cache.config';
+import { ConfigModule } from '@nestjs/config';
+import apiConfig from './api.config';
+import blockchainConfig from '#content-watcher-lib/blockchain/blockchain.config';
+import queueConfig from '#content-watcher-lib/queues/queue.config';
+import ipfsConfig from '#content-watcher-lib/ipfs/ipfs.config';
+import scannerConfig from '#content-watcher-lib/scanner/scanner.config';
+import pubsubConfig from '#content-watcher-lib/pubsub/pubsub.config';
 
 @Module({
   imports: [
-    AppConfigModule,
+    ConfigModule.forRoot({
+      isGlobal: true,
+      load: [apiConfig, blockchainConfig, cacheConfig, queueConfig, ipfsConfig, scannerConfig, pubsubConfig],
+    }),
     ScheduleModule.forRoot(),
     BlockchainModule,
     ScannerModule,
     CrawlerModule,
     IPFSProcessorModule,
     PubSubModule,
     CacheModule.forRootAsync({
-      useFactory: (configService: AppConfigService) => [
-        { url: configService.redisUrl.toString(), keyPrefix: configService.cacheKeyPrefix },
-      ],
-      inject: [AppConfigService],
+      useFactory: (conf: ICacheConfig) => [{ url: conf.redisUrl, keyPrefix: conf.cacheKeyPrefix }],
+      inject: [cacheConfig.KEY],
     }),
     QueueModule,
 

apps/content-watcher/src/build-openapi.ts

@@ -13,11 +13,12 @@ process.env.REDIS_URL = dummyUrl;
 process.env.FREQUENCY_URL = dummyUrl;
 process.env.IPFS_ENDPOINT = dummyUrl;
 process.env.IPFS_GATEWAY_URL = dummyUrl;
+process.env.CACHE_KEY_PREFIX = 'content-watcher:';
 
 // eslint-disable-next-line
 import { ApiModule } from './api.module';
 // eslint-disable-next-line
-import { apiFile, generateSwaggerDoc } from '#content-watcher-lib/config/swagger_config';
+import { apiFile, generateSwaggerDoc } from './swagger_config';
 
 async function bootstrap() {
   const app = await NestFactory.create(ApiModule, {

apps/content-watcher/src/main.ts

@@ -2,8 +2,8 @@ import { NestFactory } from '@nestjs/core';
 import { Logger, ValidationPipe } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import { ApiModule } from './api.module';
-import { initSwagger } from '#content-watcher-lib/config/swagger_config';
-import { AppConfigService } from '#content-watcher-lib/config/config.service';
+import { initSwagger } from './swagger_config';
+import apiConfig, { IContentWatcherApiConfig } from './api.config';
 
 const logger = new Logger('main');
 
@@ -27,7 +27,7 @@ async function bootstrap() {
   const app = await NestFactory.create(ApiModule, {
     logger: process.env.DEBUG ? ['error', 'warn', 'log', 'verbose', 'debug'] : ['error', 'warn', 'log'],
   });
-  const configService = app.get<AppConfigService>(AppConfigService);
+  const config = app.get<IContentWatcherApiConfig>(apiConfig.KEY);
 
   // Get event emitter & register a shutdown listener
   const eventEmitter = app.get<EventEmitter2>(EventEmitter2);
@@ -41,8 +41,8 @@ async function bootstrap() {
     app.enableShutdownHooks();
     app.useGlobalPipes(new ValidationPipe());
     await initSwagger(app, '/docs/swagger');
-    logger.log(`Listening on port ${configService.apiPort}`);
-    await app.listen(configService.apiPort);
+    logger.log(`Listening on port ${config.apiPort}`);
+    await app.listen(config.apiPort);
   } catch (e) {
     await app.close();
     logger.log('****** MAIN CATCH ********');

docker-compose.yaml

@@ -34,6 +34,7 @@ x-content-watcher-env: &content-watcher-env
   STARTING_BLOCK: 759882
   BLOCKCHAIN_SCAN_INTERVAL_SECONDS: 6
   WEBHOOK_FAILURE_THRESHOLD: 4
+  CACHE_KEY_PREFIX: 'content-watcher:'
 
 x-graph-service-env: &graph-service-env
   DEBOUNCE_SECONDS: 10

libs/account-lib/src/cache/cache.config.spec.ts

@@ -36,7 +36,7 @@ describe('Cache module config', () => {
     });
 
     it('should get redis url', () => {
-      expect(new URL(cacheConf.redisUrl)).toStrictEqual(new URL(ALL_ENV.REDIS_URL));
+      expect(cacheConf.redisUrl).toStrictEqual(ALL_ENV.REDIS_URL);
     });
 
     it('should get cache key prefix', () => {

libs/content-publishing-lib/src/cache/cache.config.spec.ts

@@ -36,7 +36,7 @@ describe('Cache module config', () => {
     });
 
     it('should get redis url', () => {
-      expect(new URL(cacheConf.redisUrl)).toStrictEqual(new URL(ALL_ENV.REDIS_URL));
+      expect(cacheConf.redisUrl).toStrictEqual(ALL_ENV.REDIS_URL);
     });
 
     it('should get cache key prefix', () => {

libs/content-watcher-lib/src/blockchain/blockchain.config.spec.ts

@@ -0,0 +1,39 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { describe, it, expect, beforeAll } from '@jest/globals';
+import blockchainConfig, { IBlockchainConfig } from './blockchain.config';
+import configSetup from '#testlib/utils.config-tests';
+
+const { setupConfigService, validateMissing, shouldFailBadValues } = configSetup<IBlockchainConfig>(blockchainConfig);
+
+describe('Blockchain module config', () => {
+  const ALL_ENV: { [key: string]: string | undefined } = {
+    FREQUENCY_URL: undefined,
+  };
+
+  beforeAll(() => {
+    Object.keys(ALL_ENV).forEach((key) => {
+      ALL_ENV[key] = process.env[key];
+    });
+  });
+
+  describe('invalid environment', () => {
+    it('missing frequency url should fail', async () => validateMissing(ALL_ENV, 'FREQUENCY_URL'));
+    it('invalid frequency url should fail', async () => shouldFailBadValues(ALL_ENV, 'FREQUENCY_URL', ['invalid url']));
+  });
+
+  describe('valid environment', () => {
+    let blockchainConf: IBlockchainConfig;
+    beforeAll(async () => {
+      blockchainConf = await setupConfigService(ALL_ENV);
+    });
+
+    it('should be defined', () => {
+      expect(blockchainConf).toBeDefined();
+    });
+
+    it('should get frequency url', () => {
+      const expectedUrl = new URL(ALL_ENV.FREQUENCY_URL).toString();
+      expect(blockchainConf.frequencyUrl?.toString()).toStrictEqual(expectedUrl);
+    });
+  });
+});

libs/content-watcher-lib/src/blockchain/blockchain.config.ts

@@ -0,0 +1,21 @@
+import { registerAs } from '@nestjs/config';
+import Joi from 'joi';
+import * as JoiUtil from '#config/joi-utils';
+
+export interface IBlockchainConfig {
+  frequencyUrl: URL;
+}
+
+export default registerAs('blockchain', (): IBlockchainConfig => {
+  const configs: JoiUtil.JoiConfig<IBlockchainConfig> = {
+    frequencyUrl: {
+      value: process.env.FREQUENCY_URL,
+      joi: Joi.string()
+        .uri({ scheme: ['http', 'https', 'ws', 'wss'] })
+        .required()
+        .custom((v) => new URL(v)),
+    },
+  };
+
+  return JoiUtil.validate<IBlockchainConfig>(configs);
+});

libs/content-watcher-lib/src/blockchain/blockchain.service.ts

@@ -1,27 +1,27 @@
 /* eslint-disable no-underscore-dangle */
-import { Injectable, Logger, OnApplicationBootstrap, OnApplicationShutdown } from '@nestjs/common';
+import { Inject, Injectable, Logger, OnApplicationBootstrap, OnApplicationShutdown } from '@nestjs/common';
 import { ApiPromise, ApiRx, HttpProvider, WsProvider } from '@polkadot/api';
 import { firstValueFrom } from 'rxjs';
 import { options } from '@frequency-chain/api-augment';
 import { KeyringPair } from '@polkadot/keyring/types';
 import { BlockHash, BlockNumber, SignedBlock } from '@polkadot/types/interfaces';
 import { SubmittableExtrinsic } from '@polkadot/api/types';
 import { AnyNumber, ISubmittableResult } from '@polkadot/types/types';
-import { AppConfigService } from '../config/config.service';
 import { Extrinsic } from './extrinsic';
+import blockchainConfig, { IBlockchainConfig } from './blockchain.config';
 
 @Injectable()
 export class BlockchainService implements OnApplicationBootstrap, OnApplicationShutdown {
   public api: ApiRx;
 
   public apiPromise: ApiPromise;
 
-  private configService: AppConfigService;
+  private config: IBlockchainConfig;
 
   private logger: Logger;
 
   public async onApplicationBootstrap() {
-    const providerUrl = this.configService.frequencyUrl!;
+    const providerUrl = this.config.frequencyUrl!;
     let provider: any;
     if (/^ws/.test(providerUrl.toString())) {
       provider = new WsProvider(providerUrl.toString());
@@ -54,8 +54,8 @@ export class BlockchainService implements OnApplicationBootstrap, OnApplicationS
     await Promise.all(promises);
   }
 
-  constructor(configService: AppConfigService) {
-    this.configService = configService;
+  constructor(@Inject(blockchainConfig.KEY) config: IBlockchainConfig) {
+    this.config = config;
     this.logger = new Logger(this.constructor.name);
   }
 

libs/content-watcher-lib/src/cache/cache.config.spec.ts

@@ -0,0 +1,46 @@
+/* eslint-disable import/no-extraneous-dependencies */
+import { describe, it, expect, beforeAll } from '@jest/globals';
+import cacheConfig, { ICacheConfig } from './cache.config';
+import configSetup from '#testlib/utils.config-tests';
+
+const { setupConfigService, validateMissing, shouldFailBadValues } = configSetup<ICacheConfig>(cacheConfig);
+
+describe('Cache module config', () => {
+  const ALL_ENV: { [key: string]: string | undefined } = {
+    REDIS_URL: undefined,
+    CACHE_KEY_PREFIX: undefined,
+  };
+
+  beforeAll(() => {
+    Object.keys(ALL_ENV).forEach((key) => {
+      ALL_ENV[key] = process.env[key];
+    });
+  });
+
+  describe('invalid environment', () => {
+    it('missing redis url should fail', async () => validateMissing(ALL_ENV, 'REDIS_URL'));
+
+    it('invalid redis url should fail', async () => shouldFailBadValues(ALL_ENV, 'REDIS_URL', ['invalid url']));
+
+    it('missing cache key prefix should fail', async () => validateMissing(ALL_ENV, 'CACHE_KEY_PREFIX'));
+  });
+
+  describe('valid environment', () => {
+    let cacheConf: ICacheConfig;
+    beforeAll(async () => {
+      cacheConf = await setupConfigService(ALL_ENV);
+    });
+
+    it('should be defined', () => {
+      expect(cacheConf).toBeDefined();
+    });
+
+    it('should get redis url', () => {
+      expect(cacheConf.redisUrl).toStrictEqual(new URL(ALL_ENV.REDIS_URL).toString());
+    });
+
+    it('should get cache key prefix', () => {
+      expect(cacheConf.cacheKeyPrefix).toStrictEqual(ALL_ENV.CACHE_KEY_PREFIX?.toString());
+    });
+  });
+});

libs/content-watcher-lib/src/cache/cache.config.ts

@@ -0,0 +1,23 @@
+import { JoiUtils } from '#config';
+import { registerAs } from '@nestjs/config';
+import Joi from 'joi';
+
+export interface ICacheConfig {
+  redisUrl: string;
+  cacheKeyPrefix: string;
+}
+
+export default registerAs('cache', (): ICacheConfig => {
+  const configs: JoiUtils.JoiConfig<ICacheConfig> = {
+    redisUrl: {
+      value: process.env.REDIS_URL,
+      joi: Joi.string().uri().required(),
+    },
+    cacheKeyPrefix: {
+      value: process.env.CACHE_KEY_PREFIX,
+      joi: Joi.string().required(),
+    },
+  };
+
+  return JoiUtils.validate<ICacheConfig>(configs);
+});