Mono-Repo Merge Content Watcher Service (#248)

Merging Content Watcher Service from
https://github.com/AmplicaLabs/content-watcher-service/

.github/workflows/content-watcher-build.yml

@@ -0,0 +1,68 @@
+name: "[Content Watcher] Build And Test"
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: "Build"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "npm"
+          registry-url: "https://registry.npmjs.org"
+          cache-dependency-path: services/content-watcher/package-lock.json
+      - name: Install dependencies
+        working-directory: services/content-watcher
+        run: npm ci
+      - name: Build NestJS
+        working-directory: services/content-watcher
+        run: npm run build
+  test_jest:
+    name: "Test"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "npm"
+          registry-url: "https://registry.npmjs.org"
+          cache-dependency-path: services/content-watcher/package-lock.json
+      - name: Install dependencies
+        working-directory: services/content-watcher
+        run: npm ci
+      - name: Run Jest
+        working-directory: services/content-watcher
+        run: npm run test
+  check_licenses:
+    name: "Dependency License Check"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "npm"
+          registry-url: "https://registry.npmjs.org"
+          cache-dependency-path: services/content-watcher/package-lock.json
+      - name: Install dependencies
+        working-directory: services/content-watcher
+        run: npm ci
+      - name: License Check
+        working-directory: services/content-watcher
+        # List all the licenses and error out if it is not one of the supported licenses
+        run: npx license-report --fields=name --fields=licenseType | jq 'map(select(.licenseType | IN("MIT", "Apache-2.0", "ISC", "BSD-3-Clause", "BSD-2-Clause", "(Apache-2.0 AND MIT)") | not)) | if length == 0 then halt else halt_error(1) end'

.github/workflows/content-watcher-release.yml

@@ -0,0 +1,76 @@
+name: "[Content Watcher] Release"
+run-name: "[Content Watcher] Cut Release ${{github.event.inputs.release-version || github.ref_name}}"
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+on:
+  push:
+    tags:
+      - "content-watcher-v[0-9]+.[0-9]+.[0-9]+" # ex. v1.0.0
+      - "content-watcher-v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" # ex. v1.1.0-rc1
+      - "content-watcher-v0.0.1" # used for testing only
+      - "content-watcher-v0.0.1-rc[0-9]+" # used for testing only
+  workflow_dispatch:
+    inputs:
+      release-version:
+        description: "Release version (content-watcher-v#.#.#[-rc#])"
+        required: true
+
+env:
+  NEW_RELEASE_TAG_FROM_UI: ${{github.event.inputs.release-version}}
+  TEST_RUN: ${{startsWith(github.event.inputs.release-version || github.ref_name, 'content-watcher-v0.0.1')}}
+  DOCKER_HUB_PROFILE: amplicalabs
+  IMAGE_NAME: content-watcher-service
+
+jobs:
+  build-and-publish-container-image:
+    name: Build and publish container image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Validate Version Tag
+        if: env.NEW_RELEASE_TAG_FROM_UI != ''
+        shell: bash
+        run: |
+          version=${{env.NEW_RELEASE_TAG_FROM_UI}}
+          echo "Release version entered in UI: $version"
+          regex='^content-watcher-v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-rc[1-9]\d*)?$'
+          if [[ ! $version =~ $regex ]]; then
+            echo "ERROR: Entered version $version is not valid."
+            echo "Please use content-watcher-v#.#.#[-rc#] format."
+            exit 1
+          fi
+          echo "valid-version=true" >> $GITHUB_OUTPUT
+      - name: Check Out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{env.NEW_RELEASE_TAG_FROM_UI}}
+      - name: Set up tags for cp image
+        id: cp-tags
+        uses: docker/metadata-action@v5
+        with:
+          flavor: |
+            latest=auto
+          images: |
+            ${{env.DOCKER_HUB_PROFILE}}/${{env.IMAGE_NAME}}
+          tags: |
+            type=semver,pattern={{version}}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: |
+            linux/amd64
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{secrets.DOCKERHUB_USERNAME_FC}}
+          password: ${{secrets.DOCKERHUB_TOKEN_FC}}
+      - name: Build and Push content-watcher-service Image
+        uses: docker/build-push-action@v5
+        with:
+          context: services/content-watcher
+          platforms: linux/amd64
+          push: ${{env.TEST_RUN != 'true'}}
+          file: services/content-watcher/Dockerfile
+          tags: ${{ steps.cp-tags.outputs.tags }}

services/content-watcher/.dockerignore

@@ -0,0 +1,35 @@
+# Include any files or directories that you don't want to be copied to your
+# container here (e.g., local build artifacts, temporary files, etc.).
+#
+# For more help, visit the .dockerignore file reference guide at
+# https://docs.docker.com/go/build-context-dockerignore/
+
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/.next
+**/.cache
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/charts
+**/docker-compose*
+**/compose.y*ml
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+**/build
+**/dist
+LICENSE
+README.md
+env.template

services/content-watcher/.env.content-publishing-service

@@ -0,0 +1,12 @@
+CHAIN_ENVIRONMENT=dev
+PROVIDER_ID=1
+PROVIDER_ACCOUNT_SEED_PHRASE=//Alice
+FILE_UPLOAD_MAX_SIZE_IN_BYTES=10000000 # ~10Mb
+ASSET_EXPIRATION_INTERVAL_SECONDS=300
+BATCH_INTERVAL_SECONDS=12
+BATCH_MAX_COUNT=1000
+ASSET_UPLOAD_VERIFICATION_DELAY_SECONDS=5
+CAPACITY_LIMIT={"type":"percentage", "value":80}
+REDIS_URL=redis://redis:6379
+FREQUENCY_URL=ws://frequency:9944
+IPFS_ENDPOINT=http://ipfs:5001

services/content-watcher/.env.docker.dev

@@ -0,0 +1,39 @@
+# Tweak values for local development
+# Content Publishing Service in Docker Compose will override these values with `.env.content-publishing-service`
+
+# URL to IPFS endpoint
+# IPFS_ENDPOINT="https://ipfs.infura.io:5001"
+IPFS_ENDPOINT="http://ipfs:5001"
+
+# If using Infura with auth required for read access, put Project ID here, or leave blank for Kubo RPC
+# IPFS_BASIC_AUTH_USER=
+
+# If using Infura with auth required for read access, put auth token here, or leave blank for Kubo RPC
+# IPFS_BASIC_AUTH_SECRET=
+
+# IPFS gateway URL. '[CID]' is a token that will be replaced with an actual content ID
+# IPFS_GATEWAY_URL="https://ipfs.io/ipfs/[CID]"
+IPFS_GATEWAY_URL="http://ipfs:8080/ipfs/[CID]"
+
+# Blockchain node address
+FREQUENCY_URL=ws://frequency:9944
+
+# Redis URL
+REDIS_URL=redis://redis:6379
+
+# How many seconds to delay between successive scans of the chain
+# for new content (after end of chain is reached)
+BLOCKCHAIN_SCAN_INTERVAL_SECONDS=12
+
+# Max number of jobs allowed on the queue before
+# blockchain scan will be paused to allow queue to drain
+QUEUE_HIGH_WATER=1000
+
+# Number of retry attempts if a registered webhook call fails
+WEBHOOK_FAILURE_THRESHOLD=4
+
+# Number of seconds between webhook retry attempts when failing
+WEBHOOK_RETRY_INTERVAL_SECONDS=10
+
+# Port that the application REST endpoints listen on
+API_PORT=3000

services/content-watcher/.gitignore

@@ -0,0 +1,10 @@
+node_modules
+dist
+.env*
+!.env.docker.dev
+!.env.content-publishing-service
+.vscode
+coverage
+.idea
+docs/*.html
+*.bkp

services/content-watcher/.prettierrc

@@ -0,0 +1,10 @@
+{
+  "arrowParens": "always",
+  "bracketSpacing": true,
+  "semi": true,
+  "trailingComma": "all",
+  "singleQuote": true,
+  "printWidth": 180,
+  "tabWidth": 2,
+  "useTabs": false
+}

services/content-watcher/.tool-versions

@@ -0,0 +1 @@
+nodejs 20.12.2

services/content-watcher/Dockerfile

@@ -0,0 +1,33 @@
+# Use a multi-stage build for efficiency
+FROM node:20 AS builder
+
+WORKDIR /app
+
+COPY package*.json ./
+
+RUN npm ci
+
+COPY . .
+
+# Build the application
+RUN npm run build
+
+# Production stage
+FROM node:20
+
+WORKDIR /app
+
+COPY --from=builder /app/dist ./dist
+COPY package*.json ./
+
+RUN npm ci --omit=dev
+
+# We want jq and curl in the final image, but we don't need the support files
+RUN apt-get update && \
+	apt-get install -y jq curl tini && \
+	apt-get clean && \
+	rm -rf /usr/share/doc /usr/share/man /usr/share/zsh
+
+EXPOSE 3000
+
+ENTRYPOINT ["/usr/bin/tini", "--", "node", "dist/apps/api/main.js"]

services/content-watcher/ENVIRONMENT.md

@@ -0,0 +1,19 @@
+# Environment Variables
+
+This application recognizes the following environment variables:
+
+| Name                               | Description                                                                                                     |       Range/Type       | Required? |     Default      |
+| ---------------------------------- | --------------------------------------------------------------------------------------------------------------- | :--------------------: | :-------: | :--------------: |
+| `API_PORT`                         | HTTP port that the application listens on                                                                       |      1025 - 65535      |           |       3000       |
+| `BLOCKCHAIN_SCAN_INTERVAL_SECONDS` | How many seconds to delay between successive scans of the chain for new content (after end of chain is reached) |          > 0           |           |        12        |
+| `CACHE_KEY_PREFIX`                 | Prefix to use for Redis cache keys                                                                              |         string         |           | content-watcher: |
+| `FREQUENCY_URL`                    | Blockchain node address                                                                                         | http(s): or ws(s): URL |     Y     |                  |
+| `IPFS_BASIC_AUTH_SECRET`           | If required for read requests, put Infura auth token here, or leave blank for default Kubo RPC                  |         string         |     N     |      blank       |
+| `IPFS_BASIC_AUTH_USER`             | If required for read requests, put Infura Project ID here, or leave blank for default Kubo RPC                  |         string         |     N     |      blank       |
+| `IPFS_ENDPOINT`                    | URL to IPFS endpoint                                                                                            |          URL           |     Y     |                  |
+| `IPFS_GATEWAY_URL`                 | IPFS gateway URL. '[CID]' is a token that will be replaced with an actual content ID                            |      URL template      |     Y     |                  |
+| `QUEUE_HIGH_WATER`                 | Max number of jobs allowed on the '' before blockchain scan will be paused to allow queue to drain              |         >= 100         |           |       1000       |
+| `REDIS_URL`                        | Connection URL for Redis                                                                                        |          URL           |     Y     |
+| `STARTING_BLOCK`                   | Block number from which the service will start scanning the chain                                               |          > 0           |           |        1         |
+| `WEBHOOK_FAILURE_THRESHOLD`        | Number of failures allowing in the provider webhook before the service is marked down                           |          > 0           |           |        3         |
+| `WEBHOOK_RETRY_INTERVAL_SECONDS`   | Number of seconds between provider webhook retry attempts when failing                                          |          > 0           |           |        10        |

services/content-watcher/INSTALLING.md

@@ -0,0 +1,65 @@
+# INSTALLING
+
+## A Note about Redis Persistence
+
+The application requires a Redis server that is configured with `Append-only file` persistence. This is so that application state can be maintained across Redis restarts. Notes on how to configure this are included below for each type of deployment.
+
+
+## Deploying using prebuilt Docker images
+
+### Standalone (complete) image
+
+The standalone container image is meant to be a complete solution for a provider. It contains a single instance of the main application, plus a pre-configured Redis server. Simply download the latest [container image](https://hub.docker.com/r/amplicalabs/content-watcher-service/) and deploy using your favorite container management system.
+```
+    docker pull amplicalabs/content-watcher-service:standalone-latest
+```
+
+The internal Redis server included in the complete image is already configured for persistence; it is simply necessary to configure your container pod to map the directory `/var/lib/redis` to a persistent storage volume.
+
+#### Note: The internal redis server runs as user:group 100:102, so mapped volume permissions must at minimum allow write access to this user. How this is provisioned will depend on the specifics of your persistent storage infrastructure. If this is not configured properly, the redis server will fail to start, and the application upon launch will throw `ECONNREFUSED` errors.
+
+Follow the instructions below for [configuration](#configuration), with the exception that you should _not_ modify `REDIS_URL`, as it already points to the internal Redis server.
+
+### App-only image
+
+The app-only image is meant to be used for providers who would rather utilize a Redis instance in their own (or their cloud infrastructure provider's) external Redis instance or service. To download the latest [container image](https://hub.docker.com/r/amplicalabs/content-watcher-service/), simply:
+```
+    docker pull amplicalabs/content-watcher-service:apponly-latest
+```
+In this case, you need to ensure that the following settings are configured in your Redis instance:
+```
+appendonly true
+dir <base directory for Redis storage>
+appendonlydir <subdirectory of base directory where append-only persistence files are stored>
+```
+
+You must also minimally map `appendonlydir` (or the entire `dir`) to a persistent storage volume in your infrastructure environment
+
+## Building and Deploying the Application
+
+If you choose to build & deploy the application yourself, simply install the prerequisites:
+* NodeJS 18
+
+To build the application:
+```
+    npm run build
+```
+
+To run the application:
+```
+    npm start
+```
+
+## Configuration
+
+For the application to start & run correctly, it is necessary to configure the environment with certain parameters. These should be injected into a container pod if running in a containerized environment.
+
+The following is a list of environment variables that may be set to control the application's behavior and environment. The complete list can always be referenced [here](./env.template)
+
+|Variable|required?|Description|Default|
+|-|-|-|-|
+|`FREQUENCY_URL`|**yes**|Blockchain URL|_none_|
+|`STARTING_BLOCK`|**maybe**|Starting block for scanner to scan from|_none_|
+|`REDIS_URL`|**yes**|URL used to connect to Redis instance|_none_<br/>\*preset to the internal Redis URL in the standalone container|
+|`BLOCKCHAIN_SCAN_INTERVAL_SECONDS`|no|# of seconds to wait in between scans of the blockchain|12|
+|`QUEUE_HIGH_WATER`|no|# of pending queue entries to allow before pausing blockchain scanning until the next scan cycle|1000|