Merge branch 'main' into chore/handle-chain-rpc-node-disconnect

.github/workflows/release.yml

@@ -117,3 +117,10 @@ jobs:
           push: ${{ needs.set_variables.outputs.test_run != 'true'}}
           file: Docker/Dockerfile.${{ matrix.service }}
           tags: ${{ steps.cp-tags.outputs.tags }}
+      - name: Update DockerHub Description
+        uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae
+        with:
+          username: ${{secrets.DOCKERHUB_USERNAME}}
+          password: ${{secrets.DOCKERHUB_TOKEN}}
+          repository: ${{env.DOCKER_HUB_PROFILE}}/${{matrix.service}}-service
+          readme-filepath: Docker/${{matrix.service}}-service.overview.md

Docker/account-service.overview.md

@@ -0,0 +1,33 @@
+# Account Service Overview
+
+Account Service is a service enabling easy interaction with [DSNP](https://dsnp.org/) accounts on [Frequency](https://docs.frequency.xyz/). Accounts are identified by the Frequency MSA Id.
+
+## Key Features
+
+- **Account creation using [SIWF](https://github.com/ProjectLibertyLabs/siwf)**
+  - Includes behind the scenes delegation to the provider
+- **Get User and Provider Account(s)**
+- **Handle (aka username) claiming**
+  - Create
+  - Change
+- **Keys Handling**
+  - Add keys to an account
+  - Get keys
+- **Get delegation info for an MSA Id**
+- **Easy integration with Web3 wallets**
+
+## Running the Account Service
+
+For information on running the Account Service, please refer to the [Account Service Documentation](https://projectlibertylabs.github.io/gateway/Run/AccountService/RunAccountService.html).
+
+## Detailed Documentation
+
+For more detailed information about the Account Service, including setup, configuration, and API documentation, please refer to the [Account Service Documentation](https://projectlibertylabs.github.io/gateway/Build/AccountService/AccountService.html).
+
+## 🚀 REST API Docs
+
+- [REST API](https://projectlibertylabs.github.io/account-service)
+
+### Environment Variables
+
+The Account Service depends on specific environment variables to function correctly. These variables are used to configure the service and are required for the service to run. Please refer to the [Account Service Configuration](https://projectlibertylabs.github.io/gateway/Build/AccountService/AccountService.html#configuration) documentation for a complete list of required environment variables.

Docker/content-publishing-service.overview.md

@@ -0,0 +1,37 @@
+# Content Publishing Service Overview
+
+The Content Publishing Service is part of the [Gateway](https://projectlibertylabs.github.io/gateway) collection of services that provides a familiar REST API to allow uploading content and publishing announcements to the [Frequency](https://docs.frequency.xyz/) chain. The service handles all of the necessary blockchain interaction and allows clients to interact using a familiar, Web2-friendly interface.
+
+## Key Features
+
+### API
+
+- **Upload asset files**
+- **Create a broadcast(post)**
+- **Create a reply**
+- **Create a reaction**
+- **Update existing content**
+- **Delete existing content (tombstones)**
+- **Update a user profile**
+
+### Development API
+
+- **Get the details of a job**
+- **Get asset**
+- **Populate a queue with dummy announcement data**
+
+## Running the Content Publishing Service
+
+For information on running the Content Publishing Service, please refer to the [Content Publishing Service Documentation](https://projectlibertylabs.github.io/gateway/Run/ContentPublishing/RunContentPublishing.html).
+
+## Detailed Documentation
+
+For more detailed information about the Content Publishing Service, including setup, configuration, and API documentation, please refer to the [Content Publishing Service Documentation](https://projectlibertylabs.github.io/gateway/Build/ContentPublishing/ContentPublishing.html).
+
+## 🚀 REST API Docs
+
+- [REST API](https://projectlibertylabs.github.io/content-publishing-service)
+
+### Environment Variables
+
+The Content Publishing Service depends on specific environment variables to function correctly. These variables are used to configure the service and are required for the service to run. Please refer to the [Content Publishing Service Configuration](https://projectlibertylabs.github.io/gateway/Build/ContentPublishing/ContentPublishing.html#configuration) documentation for a complete list of required environment variables.

Docker/content-watcher-service.overview.md

@@ -0,0 +1,31 @@
+# Content Watcher Service Overview
+
+The Content Watcher Service is part of the [Gateway](https://projectlibertylabs.github.io/gateway) collection of services that provides a familiar callback API to retrieve content and publishing announcements from the [Frequency](https://docs.frequency.xyz/) chain. The service handles all of the necessary blockchain interaction and allows clients to interact using a familiar, Web2-friendly interface.
+
+## Key Features
+
+### API
+
+- **Parse DSNP Messages on Frequency**
+- **Send content to registered webhooks**
+- **Filter via Simple Schema and MSA Id**
+
+### Scanner API
+
+- **Start, Stop, and restart scanning**
+
+## Running the Content Watcher Service
+
+For information on running the Content Watcher Service, please refer to the [Content Watcher Service Documentation](https://projectlibertylabs.github.io/gateway/Run/ContentWatcher/RunContentWatcher.html).
+
+## Detailed Documentation
+
+For more detailed information about the Content Watcher Service, including setup, configuration, and API documentation, please refer to the [Content Watcher Service Documentation](https://projectlibertylabs.github.io/gateway/Build/ContentWatcher/ContentWatcher.html).
+
+## 🚀 REST API Doc
+
+- [REST API](https://projectlibertylabs.github.io/content-watcher-service)
+
+### Environment Variables
+
+The Content Watcher Service depends on specific environment variables to function correctly. These variables are used to configure the service and are required for the service to run. Please refer to the [Content Watcher Service Configuration](https://projectlibertylabs.github.io/gateway/Build/ContentWatcher/ContentWatcher.html#configuration) documentation for a complete list of required environment variables.

Docker/graph-service.overview.md

@@ -0,0 +1,25 @@
+# Graph Service Overview
+
+The Graph Service is a service enabling easy interaction with [DSNP](https://dsnp.org/) private and public graphs on [Frequency](https://docs.frequency.xyz/).
+
+## Key Features
+
+- **Fetch Graphs**
+- **Update Graphs**
+- **Watch Graphs**
+
+## Running the Graph Service
+
+For information on running the Graph Service, please refer to the [Graph Service Documentation](https://projectlibertylabs.github.io/gateway/Run/GraphService/RunGraphService.html).
+
+## Detailed Documentation
+
+For more detailed information about the Graph Service, including setup, configuration, and API documentation, please refer to the [Graph Service Documentation](https://projectlibertylabs.github.io/gateway/Build/GraphService/GraphService.html).
+
+## 🚀 REST API Doc
+
+- [REST API](https://projectlibertylabs.github.io/graph-service)
+
+### Environment Variables
+
+The Graph Service depends on specific environment variables to function correctly. These variables are used to configure the service and are required for the service to run. Please refer to the [Graph Service Configuration](https://projectlibertylabs.github.io/gateway/Build/GraphService/GraphService.html#configuration) documentation for a complete list of required environment variables.

Makefile

@@ -90,3 +90,12 @@ start-graph-worker: update-packages
 .PHONY: setup-account
 setup-account:
 	@cd apps/account-api/test/setup && npm install && npm run main
+
+
+.PHONY: test-content-watcher-services-start
+test-content-watcher-services-start:
+	@scripts/content-watcher/local-setup.sh -n cw-e2e -i
+
+.PHONY: test-content-watcher-services-stop
+test-content-watcher-services-stop:
+	@scripts/content-watcher/local-setup.sh -n cw-e2e -d

apps/content-watcher/src/controllers/v1/search.controller.ts

@@ -12,7 +12,10 @@ export class SearchControllerV1 {
 
   @Post()
   @HttpCode(HttpStatus.OK)
-  @ApiOperation({ summary: 'Search for DSNP content by id, start/end block, and filters' })
+  @ApiOperation({
+    summary:
+      'Search for DSNP content by id and filters, starting from `upperBound` block and going back for `blockCount` number of blocks',
+  })
   @ApiOkResponse({
     description: 'Returns a jobId to be used to retrieve the results',
     type: SearchResponseDto,

apps/content-watcher/src/controllers/v1/webhook.controller.ts

@@ -3,7 +3,7 @@ import {
   WebhookRegistrationDto,
   WebhookRegistrationResponseDto,
 } from '#types/dtos/content-watcher/subscription.webhook.dto';
-import { Body, Controller, Delete, Get, HttpStatus, Logger, Put } from '@nestjs/common';
+import { Body, Controller, Delete, Get, HttpStatus, Logger, Post } from '@nestjs/common';
 import { ApiOkResponse, ApiOperation, ApiTags } from '@nestjs/swagger';
 
 @Controller('v1/webhooks')
@@ -15,7 +15,7 @@ export class WebhookControllerV1 {
     this.logger = new Logger(this.constructor.name);
   }
 
-  @Put()
+  @Post()
   @ApiOperation({ summary: 'Register a webhook to be called when new content is encountered on the chain' })
   async registerWebhook(@Body() webhookRegistrationDto: WebhookRegistrationDto) {
     this.logger.debug(`Registering webhook ${JSON.stringify(webhookRegistrationDto)}`);

apps/content-watcher/src/crawler/crawler.service.ts

@@ -30,27 +30,33 @@ export class CrawlerService extends BaseConsumer {
   async process(job: Job<ContentSearchRequestDto, any, string>): Promise<void> {
     this.logger.log(`Processing crawler job ${job.id}: ${JSON.stringify(job.data)}`);
 
+    // we crawl the blocks in reverse because we can not determine how many blocks back
+    // we can access from the node
     try {
-      let { startBlock } = job.data;
-      if (!startBlock) {
-        startBlock = await this.blockchainService.getLatestFinalizedBlockNumber();
-        // eslint-disable-next-line no-param-reassign
-        job.data.startBlock = startBlock;
-        this.logger.debug(`No starting block specified; starting from end of chain at block ${startBlock}`);
+      let { upperBoundBlock } = job.data;
+      const latestBlock = await this.blockchainService.getLatestFinalizedBlockNumber();
+      if (!upperBoundBlock) {
+        upperBoundBlock = latestBlock;
+        this.logger.debug(`No starting block specified; starting from end of chain at block ${upperBoundBlock}`);
       }
+      upperBoundBlock = Math.min(upperBoundBlock, latestBlock);
+      // eslint-disable-next-line no-param-reassign
+      job.data.upperBoundBlock = upperBoundBlock;
       // Make sure blockCount is not longer than the current chain length
-      if (job.data.blockCount >= startBlock) {
+      if (job.data.blockCount >= upperBoundBlock) {
         // eslint-disable-next-line no-param-reassign
-        job.data.blockCount = startBlock;
+        job.data.blockCount = upperBoundBlock;
       }
-      let blockList = new Array(job.data.blockCount).fill(0).map((_v, index) => startBlock - index);
+      this.logger.debug(`Searching backwards from ${upperBoundBlock} for ${job.data.blockCount} blocks!`);
+      let blockList = new Array(job.data.blockCount).fill(0).map((_v, index) => upperBoundBlock - index);
       blockList.reverse();
 
       // Process block list in chunks so as not to overload the async queue
       await this.processBlockList(
         job.data.clientReferenceId,
         blockList.slice(0, CRAWLER_BLOCK_CHUNK_SIZE),
         job.data.filters,
+        job.data.webhookUrl,
       );
       blockList = blockList.slice(CRAWLER_BLOCK_CHUNK_SIZE);
 
@@ -73,7 +79,12 @@ export class CrawlerService extends BaseConsumer {
     }
   }
 
-  private async processBlockList(clientReferenceId: string, blockList: number[], filters: ChainWatchOptionsDto) {
+  private async processBlockList(
+    clientReferenceId: string,
+    blockList: number[],
+    filters: ChainWatchOptionsDto,
+    webHookUrl: string,
+  ) {
     this.logger.debug(`Processing block list ${Math.min(...blockList)}...${Math.max(...blockList)}`);
     await Promise.all(
       blockList.map(async (blockNumber) => {
@@ -82,7 +93,7 @@ export class CrawlerService extends BaseConsumer {
           this.logger.debug(`Found ${messages.length} messages for block ${blockNumber}`);
         }
         // eslint-disable-next-line no-await-in-loop
-        await ChainEventProcessorService.queueIPFSJobs(messages, this.ipfsQueue, clientReferenceId);
+        await ChainEventProcessorService.queueIPFSJobs(messages, this.ipfsQueue, clientReferenceId, webHookUrl);
       }),
     );
   }