Merge pull request #954 from autonomys/main

Sync main with production
autonomys · Nov 15, 2024 · 1837d46 · 1837d46
2 parents 8828ec5 + 98ca591
commit 1837d46
Show file tree

Hide file tree

Showing 111 changed files with 232,387 additions and 14,218 deletions.
diff --git a/.env b/.env
@@ -5,10 +5,12 @@ NETWORK_ID=mainnet
 # Testnet - Taurus
 # NETWORK_ID=taurus
 
+LOG_LEVEL=debug
+
 # Mainnet
-NODE_DOCKER_TAG="mainnet-2024-nov-05"
+NODE_DOCKER_TAG=mainnet-2024-nov-13
 # Testnet - Taurus
-# NODE_DOCKER_TAG="taurus-candidate"
+# NODE_DOCKER_TAG=taurus-2024-nov-05
 
 DB_USER=postgres
 DB_DATABASE=postgres
@@ -17,11 +19,11 @@ DB_PORT=5432
 DB_HOST=postgres
 
 # Mainnet
-RPC_URLS="ws://caddy:80,wss://rpc-0.mainnet.subspace.network/ws,wss://rpc-1.mainnet.subspace.network/ws,wss://rpc-0.mainnet.autonomys.xyz/ws,wss://rpc-1.mainnet.autonomys.xyz/ws"
+RPC_URLS="ws://localhost:8000,wss://rpc-0.mainnet.subspace.network/ws,wss://rpc-1.mainnet.subspace.network/ws,wss://rpc-0.mainnet.autonomys.xyz/ws,wss://rpc-1.mainnet.autonomys.xyz/ws"
 CHAIN_ID="0x66455a580aabff303720aa83adbe6c44502922251c03ba73686d5245da9e21bd"
 
 # Testnet - Taurus
-# RPC_URLS="ws://caddy:80,wss://rpc-0.taurus.subspace.network/ws"
+# RPC_URLS="ws://localhost:8000,wss://rpc-0.taurus.subspace.network/ws"
 # CHAIN_ID="0x295aeafca762a304d92ee1505548695091f6082d3f0aa4d092ac3cd6397a6c5e"
 
 HASURA_GRAPHQL_PORT=8080
@@ -36,14 +38,18 @@ HASURA_GRAPHQL_CORS_DOMAIN="*"
 DICTIONARY_DIRECTORY=autonomys
 # Testnet - Taurus
 # DICTIONARY_DIRECTORY=autonomys-taurus
-DICTIONARY_SUBQUERY_NODE_PORT=3010
-DICTIONARY_URL="http://dictionary_subquery_node:3000"
+DICTIONARY_SUBQUERY_NODE_PORT=3000
+DICTIONARY_URL="http://localhost:3000"
 
 REDIS_PORT=6379
 
 BULL_USERNAME=bull
 BULL_PASSWORD=board
 BULL_PORT=3020
+BULL_SESSION_SECRET=keyboardcat
 
 NR_API_KEY=""
-NR_AGENT_IDENTIFIER=""
+NR_AGENT_IDENTIFIER=""
+
+SLACK_TOKEN=""
+SLACK_CONVERSATION_ID=""
diff --git a/.vscode/astral.code-workspace b/.vscode/astral.code-workspace
@@ -29,8 +29,8 @@
       "path": "../indexers/gemini-3h"
     },
     {
-      "name": "indexer - Gemini 3G",
-      "path": "../indexers/gemini-3g"
+      "name": "Taskboard",
+      "path": "../indexers/taskboard"
     }
   ],
   "settings": {
@@ -56,6 +56,7 @@
       "gemini",
       "graphiql",
       "leaderboard",
+      "polkadot",
       "siwe",
       "subscan",
       "subspace",

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -21,6 +21,7 @@
     "gemini",
     "graphiql",
     "leaderboard",
+    "polkadot",
     "siwe",
     "subscan",
     "subspace",

diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
   postgres:
     volumes:
@@ -28,6 +26,12 @@ services:
       NRIA_DISPLAY_NAME: "${NR_AGENT_IDENTIFIER}"
     restart: unless-stopped
 
+  node:
+    logging:
+      driver: loki
+      options:
+        loki-url: "https://logging.subspace.network/loki/api/v1/push"
+
   hasura:
     image: hasura/graphql-engine:v2.40.0
     logging:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,5 +1,3 @@
-version: "3.9"
-
 volumes:
   node-data: {}
   postgres_db: {}
@@ -16,32 +14,24 @@ services:
         hard: 65536 # Hard limit for open files
     image: caddy:latest
     ports:
-    - "80:9944"    # Map external 80 to Caddy's 9944 for Node/RPC
-    - "443:9944"   # Map external 443 to Caddy's 9945 for HTTPS
-    - "8081:8080"  # Map external 8081 to Caddy's 8080 for Hasura
-    - "3011:3001"  # Map external 3011 to Caddy's 3001 for Consensus
-    - "3012:3002"  # Map external 3012 to Caddy's 3002 for Leaderboard
-    - "3013:3003"  # Map external 3013 to Caddy's 3003 for Staking
+      - "127.0.0.1:8000:8000"
     volumes:
       - ./indexers/Caddyfile:/etc/caddy/Caddyfile # Mount the Caddyfile
       - caddy_data:/data # Volume for Let's Encrypt certificates
       - caddy_config:/config
-
-    environment:
-      - DOMAIN=subql.${NETWORK_ID}.subspace.network
     depends_on:
       - node
     restart: unless-stopped
 
   # Subspace Node
   node:
     image: ghcr.io/autonomys/node:${NODE_DOCKER_TAG}
-    platform: linux/amd64
     volumes:
       - node-data:/var/subspace:rw
     ports:
       - "30333:30333/tcp"
       - "30433:30433/tcp"
+      - "127.0.0.1:9944:9944/tcp"
     restart: unless-stopped
     command:
       [
@@ -50,8 +40,10 @@ services:
         "${NETWORK_ID}",
         "--base-path",
         "/var/subspace",
-        "--pot-external-entropy",
-        "00000000000000000000661e88ab45dc3b61daec5ccf3cc83532153589b133b9",
+        "--state-pruning",
+        "archive",
+        "--blocks-pruning",
+        "archive",
         "--listen-on",
         "/ip4/0.0.0.0/tcp/30333",
         "--dsn-listen-on",
@@ -66,7 +58,6 @@ services:
         "1000",
         "--rpc-max-connections",
         "20000",
-        "--farmer",
         "--name",
         "astral",
         "--sync",
@@ -83,6 +74,7 @@ services:
     volumes:
       - postgres_db:/var/lib/postgresql/data
       - ./indexers/db/docker-entrypoint-initdb.d/init-db.sql:/docker-entrypoint-initdb.d/init-db.sql
+      - ./indexers/db/postgresql.conf:/etc/postgresql/postgresql.conf
     restart: unless-stopped
     hostname: postgres
     environment:
@@ -97,12 +89,14 @@ services:
       interval: 5s
       timeout: 5s
       retries: 5
+    command: ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"]
 
   # Hasura GraphQL Engine
   hasura:
     image: hasura/graphql-engine:v2.40.0
     depends_on:
-      - "postgres"
+      postgres:
+        condition: service_healthy
     volumes:
       - ./indexers/db/migrations:/hasura-migrations
       - ./indexers/db/metadata:/hasura-metadata
@@ -140,6 +134,10 @@ services:
       # Hasura Migrations and Metadata
       HASURA_GRAPHQL_MIGRATIONS_DIR: /hasura-migrations
       HASURA_GRAPHQL_METADATA_DIR: /hasura-metadata
+
+      # Task Board Action Endpoint
+      HASURA_GRAPHQL_ACTION_TASK_BOARD: http://taskboard:${BULL_PORT}
+      HASURA_GRAPHQL_ACTION_TASK_BOARD_SESSION_SECRET: ${BULL_SESSION_SECRET}
     ports:
       - "${HASURA_GRAPHQL_PORT}:8080"
     command:
@@ -159,7 +157,7 @@ services:
       DB_PASS: ${DB_PASSWORD}
       DB_DATABASE: ${DB_DATABASE}
       DB_HOST: ${DB_HOST}
-      DB_PORT: 5432
+      DB_PORT: ${DB_PORT}
     volumes:
       - ./indexers/dictionary/${DICTIONARY_DIRECTORY}:/dictionary
     healthcheck:
@@ -175,17 +173,19 @@ services:
     profiles: [dictionary, indexers]
     image: onfinality/subql-query:latest
     ports:
-      - "${DICTIONARY_SUBQUERY_NODE_PORT}:3000"
+      - "127.0.0.1:${DICTIONARY_SUBQUERY_NODE_PORT}:3000"
     depends_on:
-      - "postgres"
-      - "dictionary_subquery_node"
+      "postgres":
+        condition: service_healthy
+      "dictionary_subquery_node":
+        condition: service_healthy
     restart: unless-stopped
     environment:
       DB_USER: ${DB_USER}
       DB_PASS: ${DB_PASSWORD}
       DB_DATABASE: ${DB_DATABASE}
       DB_HOST: ${DB_HOST}
-      DB_PORT: 5432
+      DB_PORT: ${DB_PORT}
     command:
       - --name=dictionary
       - --playground
@@ -195,22 +195,19 @@ services:
     profiles: [indexers]
     image: subquerynetwork/subql-node-substrate:latest
     ports:
-      - "3001:3000"
+      - "3001:3001"
     depends_on:
       "postgres":
         condition: service_healthy
-      "dictionary_subquery_node":
-        condition: service_healthy
     restart: unless-stopped
     environment:
       ENDPOINT: ${RPC_URLS}
       CHAIN_ID: ${CHAIN_ID}
-      DICTIONARY: http://dictionary_subquery_node:3000
       DB_USER: ${DB_USER}
       DB_PASS: ${DB_PASSWORD}
       DB_DATABASE: ${DB_DATABASE}
       DB_HOST: ${DB_HOST}
-      DB_PORT: 5432
+      DB_PORT: ${DB_PORT}
     volumes:
       - ./indexers/${NETWORK_ID}/consensus:/${NETWORK_ID}/consensus
     command:
@@ -222,8 +219,9 @@ services:
       - --batch-size=250
       - --unfinalized-blocks=true
       - --disable-historical=false
+      - --port 3001
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://consensus_subquery_node:3000/ready"]
+      test: ["CMD", "curl", "-f", "http://consensus_subquery_node:3001/ready"]
       interval: 3s
       timeout: 5s
       retries: 10
@@ -232,7 +230,7 @@ services:
     profiles: [indexers]
     image: subquerynetwork/subql-node-substrate:latest
     ports:
-      - "3002:3000"
+      - "3002:3002"
     depends_on:
       "postgres":
         condition: service_healthy
@@ -242,24 +240,25 @@ services:
     environment:
       ENDPOINT: ${RPC_URLS}
       CHAIN_ID: ${CHAIN_ID}
-      DICTIONARY: http://dictionary_subquery_node:3000
+      DICTIONARY: http://localhost:3000
       DB_USER: ${DB_USER}
       DB_PASS: ${DB_PASSWORD}
       DB_DATABASE: ${DB_DATABASE}
       DB_HOST: ${DB_HOST}
-      DB_PORT: 5432
+      DB_PORT: ${DB_PORT}
     volumes:
       - ./indexers/${NETWORK_ID}/leaderboard:/${NETWORK_ID}/leaderboard
     command:
       - ${SUB_COMMAND:-} # set SUB_COMMAND env variable to "test" to run tests
       - -f=/${NETWORK_ID}/leaderboard
       - --db-schema=leaderboard
       - --workers=1
-      - --unsafe
       - --batch-size=15
-      - --disable-historical=true
+      - --unfinalized-blocks=true
+      - --disable-historical=false
+      - --port 3002
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://leaderboard_subquery_node:3000/ready"]
+      test: ["CMD", "curl", "-f", "http://leaderboard_subquery_node:3002/ready"]
       interval: 3s
       timeout: 5s
       retries: 10
@@ -268,7 +267,7 @@ services:
     profiles: [indexers]
     image: subquerynetwork/subql-node-substrate:latest
     ports:
-      - "3003:3000"
+      - "3003:3003"
     depends_on:
       "postgres":
         condition: service_healthy
@@ -278,12 +277,12 @@ services:
     environment:
       ENDPOINT: ${RPC_URLS}
       CHAIN_ID: ${CHAIN_ID}
-      DICTIONARY: http://dictionary_subquery_node:3000
+      DICTIONARY: http://localhost:3000
       DB_USER: ${DB_USER}
       DB_PASS: ${DB_PASSWORD}
       DB_DATABASE: ${DB_DATABASE}
       DB_HOST: ${DB_HOST}
-      DB_PORT: 5432
+      DB_PORT: ${DB_PORT}
     volumes:
       - ./indexers/${NETWORK_ID}/staking:/${NETWORK_ID}/staking
     command:
@@ -294,8 +293,9 @@ services:
       - --unsafe
       - --batch-size=15
       - --disable-historical=true
+      - --port 3003
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://staking_subquery_node:3000/ready"]
+      test: ["CMD", "curl", "-f", "http://staking_subquery_node:3003/ready"]
       interval: 3s
       timeout: 5s
       retries: 10
@@ -327,18 +327,21 @@ services:
     build:
       context: ./indexers/taskboard
     depends_on:
+      postgres:
+        condition: service_healthy
       redis:
         condition: service_healthy
     volumes:
       - ./indexers/taskboard:/app
     hostname: taskboard
     restart: unless-stopped
     ports:
-      - "${BULL_PORT}:3000"
+      - "${BULL_PORT}:${BULL_PORT}"
     environment:
       BULL_USERNAME: ${BULL_USERNAME}
       BULL_PASSWORD: ${BULL_PASSWORD}
       BULL_PORT: ${BULL_PORT}
+      BULL_SESSION_SECRET: ${BULL_SESSION_SECRET}
 
       REDIS_HOST: redis
       REDIS_PORT: ${REDIS_PORT}
@@ -348,3 +351,9 @@ services:
       DB_DATABASE: ${DB_DATABASE}
       DB_HOST: ${DB_HOST}
       DB_PORT: ${DB_PORT}
+
+      NETWORK_ID: ${NETWORK_ID}
+      LOG_LEVEL: ${LOG_LEVEL}
+
+      SLACK_TOKEN: ${SLACK_TOKEN}
+      SLACK_CONVERSATION_ID: ${SLACK_CONVERSATION_ID}