Mono-Repo Merge Content Publishing Service (#247)

Merging Content Publishing Service from
https://github.com/AmplicaLabs/content-publishing-service

.github/workflows/content-publishing-build.yml

@@ -0,0 +1,68 @@
+name: "[Content-Publishing] Build And Test"
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: "Build"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "npm"
+          registry-url: "https://registry.npmjs.org"
+          cache-dependency-path: services/content-publishing/package-lock.json
+      - name: Install dependencies
+        working-directory: services/content-publishing
+        run: npm ci
+      - name: Build NestJS
+        working-directory: services/content-publishing
+        run: npm run build
+  test_jest:
+    name: "Test"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "npm"
+          registry-url: "https://registry.npmjs.org"
+          cache-dependency-path: services/content-publishing/package-lock.json
+      - name: Install dependencies
+        working-directory: services/content-publishing
+        run: npm ci
+      - name: Run Jest
+        working-directory: services/content-publishing
+        run: npm run test
+  check_licenses:
+    name: "Dependency License Check"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "npm"
+          registry-url: "https://registry.npmjs.org"
+          cache-dependency-path: services/content-publishing/package-lock.json
+      - name: Install dependencies
+        working-directory: services/content-publishing
+        run: npm ci
+      - name: License Check
+        working-directory: services/content-publishing
+        # List all the licenses and error out if it is not one of the supported licenses
+        run: npx license-report --fields=name --fields=licenseType | jq 'map(select(.licenseType | IN("MIT", "Apache-2.0", "ISC", "BSD-3-Clause", "BSD-2-Clause", "(Apache-2.0 AND MIT)") | not)) | if length == 0 then halt else halt_error(1) end'

.github/workflows/content-publishing-release.yml

@@ -0,0 +1,76 @@
+name: "[Content-Publishing] Release"
+run-name: "[Content-Publishing] Cut Release ${{github.event.inputs.release-version || github.ref_name}}"
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+on:
+  push:
+    tags:
+      - "content-publishing-v[0-9]+.[0-9]+.[0-9]+" # ex. v1.0.0
+      - "content-publishing-v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" # ex. v1.1.0-rc1
+      - "content-publishing-v0.0.1" # used for testing only
+      - "content-publishing-v0.0.1-rc[0-9]+" # used for testing only
+  workflow_dispatch:
+    inputs:
+      release-version:
+        description: "Release version (content-publishing-v#.#.#[-rc#])"
+        required: true
+
+env:
+  NEW_RELEASE_TAG_FROM_UI: ${{github.event.inputs.release-version}}
+  TEST_RUN: ${{startsWith(github.event.inputs.release-version || github.ref_name, 'v0.0.1')}}
+  DOCKER_HUB_PROFILE: amplicalabs
+  IMAGE_NAME: content-publishing-service
+
+jobs:
+  build-and-publish-container-image:
+    name: Build and publish container image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Validate Version Tag
+        if: env.NEW_RELEASE_TAG_FROM_UI != ''
+        shell: bash
+        run: |
+          version=${{env.NEW_RELEASE_TAG_FROM_UI}}
+          echo "Release version entered in UI: $version"
+          regex='^content-publishing-v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-rc[1-9]\d*)?$'
+          if [[ ! $version =~ $regex ]]; then
+            echo "ERROR: Entered version $version is not valid."
+            echo "Please use v#.#.#[-rc#] format."
+            exit 1
+          fi
+          echo "valid-version=true" >> $GITHUB_OUTPUT
+      - name: Check Out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{env.NEW_RELEASE_TAG_FROM_UI}}
+      - name: Set up tags for cp image
+        id: cp-tags
+        uses: docker/metadata-action@v5
+        with:
+          flavor: |
+            latest=auto
+          images: |
+            ${{env.DOCKER_HUB_PROFILE}}/${{env.IMAGE_NAME}}
+          tags: |
+            type=semver,pattern={{version}}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: |
+            linux/amd64
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{secrets.DOCKERHUB_USERNAME_FC}}
+          password: ${{secrets.DOCKERHUB_TOKEN_FC}}
+      - name: Build and Push Content-Publishing-Service Image
+        uses: docker/build-push-action@v5
+        with:
+          context: services/content-publishing
+          platforms: linux/amd64
+          push: ${{env.TEST_RUN != 'true'}}
+          file: services/content-publishing/Dockerfile
+          tags: ${{ steps.cp-tags.outputs.tags }}

services/content-publishing/.dockerignore

@@ -0,0 +1,7 @@
+Dockerfile
+.dockerignore
+node_modules
+npm-debug.log
+dist
+.env*
+env.template

services/content-publishing/.env.docker.dev

@@ -0,0 +1,51 @@
+# Copy this file to ".env.dev" and ".env.docker.dev", and then tweak values for local development
+
+# URL to IPFS endpoint
+# IPFS_ENDPOINT="https://ipfs.infura.io:5001"
+IPFS_ENDPOINT="http://127.0.0.1:5001"
+
+# If using Infura, put Project ID here, or leave blank for Kubo RPC
+# IPFS_BASIC_AUTH_USER=
+
+# If using Infura, put auth token here, or leave blank for Kubo RPC
+IPFS_BASIC_AUTH_SECRET=
+
+# IPFS gateway URL. '[CID]' is a token that will be replaced with an actual content ID
+# IPFS_GATEWAY_URL="https://ipfs.io/ipfs/[CID]"
+IPFS_GATEWAY_URL="http://127.0.0.1:8080/ipfs/[CID]"
+
+# Blockchain node address
+FREQUENCY_URL=ws://0.0.0.0:9944
+
+PROVIDER_ID=1
+# Redis URL
+REDIS_URL=redis://0.0.0.0:6379
+PROVIDER_ACCOUNT_SEED_PHRASE="//Alice"
+CAPACITY_LIMIT='{"type":"percentage", "value":80}'
+
+# Port that the application REST endpoints listen on
+API_PORT=3000
+
+# Environment for mapping announcement type to schema ID (use 'dev' for e2e tests)
+# Possible values: dev, rococo, testnet, mainnet
+CHAIN_ENVIRONMENT=dev
+
+# Max file size allowed for asset upload
+FILE_UPLOAD_MAX_SIZE_IN_BYTES=2000000000
+
+# Number of seconds to keep completed asset entrie in the cache
+# before expiring them
+ASSET_EXPIRATION_INTERVAL_SECONDS=300
+
+# Number of seconds between content publishing batches. This is so that
+# the service waits a reasonable amount of time for additional content to publishing
+# before submitting a batch--it represents a trade-off between maximum batch fullness
+# and minimal wait time for published content.
+BATCH_INTERVAL_SECONDS=12
+
+# Maximum number of items that can be submitted in a single batch
+BATCH_MAX_COUNT=1000
+
+# Base delay in seconds used for exponential backoff while waiting for
+# uploaded assets to be verified available before publishing a content notice.
+ASSET_UPLOAD_VERIFICATION_DELAY_SECONDS=5

services/content-publishing/.eslintrc.json

@@ -0,0 +1,64 @@
+{
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+  "extends": [
+    "airbnb-base",
+    "prettier"
+  ],
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "project": "./tsconfig.json",
+    "sourceType": "module"
+  },
+  "settings": {
+   "import/extensions": [
+      "error",
+      "ignorePackages",
+      {
+        "js": "never",
+        "jsx": "never",
+        "ts": "never",
+        "tsx": "never"
+      }
+   ],
+    "import/parsers": {
+      "@typescript-eslint/parser": [".ts", ".tsx"]
+    },
+    "import/resolver": {
+      "typescript": {
+        "directory": "./tsconfig.json"
+      },
+      "node": {
+        "extensions": [".js", ".jsx", ".ts", ".d.ts", ".tsx"]
+      }
+    },
+    "react": {
+        "version": "999.99.99"
+    }
+  },
+  "rules": {
+    "no-console": "off",
+   "import/extensions": [
+      "error",
+      "ignorePackages",
+      {
+        "js": "never",
+        "jsx": "never",
+        "ts": "never",
+        "tsx": "never"
+      }
+   ],
+  "import/no-unresolved": [2, { "commonjs": true, "amd": true }],
+    "import/named": 2,
+    "import/namespace": 2,
+    "import/default": 2,
+    "import/export": 2,
+    "import/prefer-default-export": "off",
+    "indent": "off",
+    "no-unused-vars": "off",
+    "prettier/prettier": 2
+  },
+  "plugins": ["import", "prettier"]
+}

services/content-publishing/.gitignore

@@ -0,0 +1,7 @@
+node_modules
+dist
+.env
+.vscode
+coverage
+.idea
+docs/*.html

services/content-publishing/.prettierrc

@@ -0,0 +1,10 @@
+{
+  "arrowParens": "always",
+  "bracketSpacing": true,
+  "semi": true,
+  "trailingComma": "all",
+  "singleQuote": true,
+  "printWidth": 180,
+  "tabWidth": 2,
+  "useTabs": false
+}

services/content-publishing/.tool-versions

@@ -0,0 +1 @@
+nodejs 20.12.2

services/content-publishing/Dockerfile

@@ -0,0 +1,38 @@
+# Use a multi-stage build for efficiency
+FROM node:20 AS builder
+
+WORKDIR /app
+
+COPY package*.json ./
+
+RUN npm install
+
+COPY . .
+
+# Build the application
+RUN npm run build
+
+# Production stage
+FROM node:20
+
+WORKDIR /app
+
+COPY --from=builder /app/dist ./dist
+COPY package*.json ./
+COPY ./lua ./lua
+COPY ./scripts/docker-entrypoint.sh ./
+RUN chmod +x ./docker-entrypoint.sh
+
+RUN npm ci --omit=dev
+
+# We want jq and curl in the final image, but we don't need the support files
+RUN apt-get update && \
+	apt-get install -y jq curl tini && \
+	apt-get clean && \
+	rm -rf /usr/share/doc /usr/share/man /usr/share/zsh
+
+EXPOSE 3000
+
+ENV START_PROCESS="api"
+
+ENTRYPOINT ["/usr/bin/tini", "--", "./docker-entrypoint.sh"]

services/content-publishing/ENVIRONMENT.md

@@ -0,0 +1,22 @@
+# Environment Variables
+
+This application recognizes the following environment variables:
+
+| Name                                      | Description                                                                                                                                                                                                              |             Range/Type             |  Required?   | Default |
+| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :--------------------------------: | :----------: | :-----: |
+| `API_PORT`                                | HTTP port that the application listens on                                                                                                                                                                                |            1025 - 65535            |              |  3000   |
+| `CAPACITY_LIMIT`                          | Maximum amount of provider capacity this app is allowed to use (per epoch) type: 'percentage' 'amount' value: number (may be percentage, ie '80', or absolute amount of capacity)                                        |  JSON [(example)](./env.template)  |      Y       |         |
+| `FREQUENCY_URL`                           | Blockchain node address                                                                                                                                                                                                  |       http(s): or ws(s): URL       |      Y       |         |
+| `PROVIDER_ACCOUNT_SEED_PHRASE`            | Seed phrase for provider MSA control key                                                                                                                                                                                 |               string               |      Y       |         |
+| `PROVIDER_ID`                             | Provider MSA ID                                                                                                                                                                                                          |              integer               |      Y       |         |
+| `REDIS_URL`                               | Connection URL for Redis                                                                                                                                                                                                 |                URL                 |      Y       |
+|`ASSET_EXPIRATION_INTERVAL_SECONDS`|Number of seconds to keep completed asset entrie in the cache before expiring them|> 0|Y||
+|`ASSET_UPLOAD_VERIFICATION_DELAY_SECONDS`|Base delay in seconds used for exponential backoff while waiting for uploaded assets to be verified available before publishing a content notice|>= 0|Y||
+|`BATCH_INTERVAL_SECONDS`|Number of seconds between content publishing batches. This is so that the service waits a reasonable amount of time for additional content to publishing before submitting a batch--it represents a trade-off between maximum batch fullness and minimal wait time for published content|> 0|Y||
+|`BATCH_MAX_COUNT`|Maximum number of items that can be submitted in a single batch|> 0|Y||
+|`CHAIN_ENVIRONMENT`|Environment for mapping announcement type to schema ID (use 'dev' for e2e tests)|dev\|rococo\|testnet\|mainnet|Y||
+|`FILE_UPLOAD_MAX_SIZE_IN_BYTES`|Max file size (in bytes) allowed for asset upload|> 0|Y||
+|`IPFS_BASIC_AUTH_SECRET`|If using Infura, put auth token here, or leave blank for Kubo RPC|string|N|blank|
+|`IPFS_BASIC_AUTH_USER`|If using Infura, put Project ID here, or leave blank for Kubo RPC|string|N|blank|
+|`IPFS_ENDPOINT`|URL to IPFS endpoint|URL|Y||
+|`IPFS_GATEWAY_URL`|IPFS gateway URL. '[CID]' is a token that will be replaced with an actual content ID|URL template|Y||