squids add db pooling and replication

templates/terraform/explorer/base/config/pgcat.toml

@@ -0,0 +1,115 @@
+#
+# PgCat configuration file
+#
+
+#
+# General pooler settings
+[general]
+# What IP to run on, 0.0.0.0 means accessible from everywhere.
+host = "0.0.0.0"
+
+# Port to run on, same as PgBouncer used in this example.
+port = 6432
+
+# Whether to enable prometheus exporter or not.
+enable_prometheus_exporter = true
+
+# Port at which prometheus exporter listens on.
+prometheus_exporter_port = 9930
+
+# How long to wait before aborting a server connection (ms).
+connect_timeout = 5000
+
+# How much time to give `SELECT 1` health check query to return with a result (ms).
+healthcheck_timeout = 1000
+
+# How long to keep connection available for immediate re-use, without running a healthcheck query on it
+healthcheck_delay = 30000
+
+# How much time to give clients during shutdown before forcibly killing client connections (ms).
+shutdown_timeout = 60000
+
+# For how long to ban a server if it fails a health check (seconds).
+ban_time = 60 # seconds
+
+# If we should log client connections
+log_client_connections = false
+
+# If we should log client disconnections
+log_client_disconnections = false
+
+# TLS
+# tls_certificate = "server.cert"
+# tls_private_key = "server.key"
+
+# Credentials to access the virtual administrative database (pgbouncer or pgcat)
+# Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc..
+admin_username = "postgres"
+admin_password = "postgres"
+
+# pool
+# configs are structured as pool.<pool_name>
+# the pool_name is what clients use as database name when connecting
+# For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded"
+[pools.postgres]
+# Pool mode (see PgBouncer docs for more).
+# session: one server connection per connected client
+# transaction: one server connection per client transaction
+pool_mode = "transaction"
+
+# If the client doesn't specify, route traffic to
+# this role by default.
+#
+# any: round-robin between primary and replicas,
+# replica: round-robin between replicas only without touching the primary,
+# primary: all queries go to the primary unless otherwise specified.
+default_role = "any"
+
+# Query parser. If enabled, we'll attempt to parse
+# every incoming query to determine if it's a read or a write.
+# If it's a read query, we'll direct it to a replica. Otherwise, if it's a write,
+# we'll direct it to the primary.
+query_parser_enabled = true
+
+# If the query parser is enabled and this setting is enabled, we'll attempt to
+# infer the role from the query itself.
+query_parser_read_write_splitting = true
+
+# If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for
+# load balancing of read queries. Otherwise, the primary will only be used for write
+# queries. The primary can always be explicitly selected with our custom protocol.
+primary_reads_enabled = true
+
+# So what if you wanted to implement a different hashing function,
+# or you've already built one and you want this pooler to use it?
+#
+# Current options:
+#
+# pg_bigint_hash: PARTITION BY HASH (Postgres hashing function)
+# sha1: A hashing function based on SHA1
+#
+sharding_function = "pg_bigint_hash"
+
+# Credentials for users that may connect to this cluster
+[pools.postgres.users.0]
+username = "postgres"
+password = "postgres"
+# Maximum number of server connections that can be established for this user
+# The maximum number of connection from a single Pgcat process to any database in the cluster
+# is the sum of pool_size across all users.
+pool_size = 9
+
+# Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way.
+statement_timeout = 0
+
+# Shard 0
+[pools.postgres.shards.0]
+# [ host, port, role ]
+servers = [
+    [ "postgres", 5432, "primary" ],
+    [ "postgres", 5432, "replica" ]
+]
+# Database name (e.g. "postgres")
+database = "postgres"
+
+database = "postgres"

templates/terraform/explorer/base/config/postgresql.conf

@@ -50,3 +50,10 @@ pg_stat_statements.track = all
 
 # Locking Settings
 deadlock_timeout = 1s
+
+# Replication Settings
+wal_level = replica
+max_wal_senders = 10
+max_replication_slots = 10
+hot_standby = on
+max_standby_archive_delay = 30s

templates/terraform/explorer/base/scripts/create_squid_node_compose_file.sh

@@ -28,6 +28,35 @@ services:
       options:
         loki-url: "https://logging.subspace.network/loki/api/v1/push"
 
+  db_replica:
+    image: postgres:16
+    shm_size: 1gb
+    volumes:
+      - db_data_replica:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: squid-archive
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    command: >
+      bash -c "until pg_isready -h db_primary -p 5432 -U ${POSTGRES_USER}; do sleep 1; done;
+      pg_basebackup -h db_primary -D /var/lib/postgresql/data -U ${POSTGRES_USER} -P -v --wal-method=stream;
+      echo 'hot_standby = on' >> /var/lib/postgresql/data/postgresql.conf;
+      echo 'primary_conninfo = ''host=db_primary port=5432 user=${POSTGRES_USER} password=${POSTGRES_PASSWORD}''' >> /var/lib/postgresql/data/postgresql.conf;
+      pg_ctl -D /var/lib/postgresql/data -o '-c config_file=/var/lib/postgresql/data/postgresql.conf' start"
+    depends_on:
+      - db_primary
+
+  pgcat:
+    image: ghcr.io/postgresml/pgcat:e1e4929d439313d987c352b4517a6d99627f3e9c
+    command:
+      - "pgcat"
+      - "/etc/pgcat/pgcat.toml"
+    volumes:
+      - "$HOME/squid/postgresql/conf/pgcat.toml:/etc/pgcat/pgcat.toml"
+    ports:
+      - "6432:6432"
+      - "9930:9930"
+
   run-migrations:
     image: ghcr.io/subspace/blockexplorer-processor:\${DOCKER_TAG}
     restart: on-failure:5
@@ -37,6 +66,8 @@ services:
       DB_NAME: \${DB_NAME}
       # provide DB password
       DB_PASS: \${DB_PASS}
+      # provide DB port
+      DB_PORT: \${DB_PORT}
     depends_on:
       - db
     command: "npm run db:migrate"
@@ -54,6 +85,8 @@ services:
       DB_NAME: \${DB_NAME}
       # provide DB password
       DB_PASS: \${DB_PASS}
+      # provide DB port
+      DB_PORT: \${DB_PORT}
     depends_on:
       - db
       - run-migrations
@@ -76,6 +109,8 @@ services:
       DB_NAME: \${DB_NAME}
       # provide DB password
       DB_PASS: \${DB_PASS}
+      # provide DB port
+      DB_PORT: \${DB_PORT}
     ports:
       - "4350:4000"
     logging:

templates/terraform/explorer/base/scripts/create_squid_node_db_replica_compose_file.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+source $HOME/.bash_profile
+cat > $HOME/squid/docker-compose.yml << EOF
+version: "3.8"
+
+services:
+  db_replica:
+    image: postgres:16
+    shm_size: 1gb
+    volumes:
+      - db_data_replica:/var/lib/postgresql/data
+      - ./postgresql_replica.conf:/etc/postgresql/postgresql.conf:ro
+    environment:
+      POSTGRES_DB: squid-archive
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    ports:
+      - "5432:5432"
+    command: >
+      bash -c "
+      until pg_isready -h ${PRIMARY_HOST} -p 5432 -U ${POSTGRES_USER}; do sleep 1; done;
+      pg_basebackup -h ${PRIMARY_HOST} -D /var/lib/postgresql/data -U ${POSTGRES_USER} -P -v -R -X stream -S replica_slot;
+      echo 'primary_conninfo = ''host=${PRIMARY_HOST} port=5432 user=${POSTGRES_USER} password=${POSTGRES_PASSWORD}''' >> /var/lib/postgresql/data/postgresql.auto.conf;
+      touch /var/lib/postgresql/data/standby.signal;
+      postgres -c config_file=/etc/postgresql/postgresql.conf"
+
+  agent:
+    container_name: newrelic-infra
+    image: newrelic/infrastructure:latest
+    cap_add:
+      - SYS_PTRACE
+    network_mode: bridge
+    pid: host
+    privileged: true
+    volumes:
+      - "/:/host:ro"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+    environment:
+      NRIA_LICENSE_KEY: "\${NR_API_KEY}"
+      NRIA_DISPLAY_NAME: "squid-\${NETWORK_NAME}-db-replica"
+    restart: unless-stopped
+
+volumes:
+  db_data_replica:
+
+EOF

templates/terraform/explorer/base/scripts/install_docker.sh

@@ -23,3 +23,11 @@ fi
 
 sudo sysctl -p /etc/sysctl.conf
 sudo docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
+
+# install ufw-docker
+sudo wget -O /usr/local/bin/ufw-docker \
+https://github.com/chaifeng/ufw-docker/raw/master/ufw-docker
+sudo chmod +x /usr/local/bin/ufw-docker
+sudo ufw enable
+sudo ufw-docker install
+sudo systemctl restart ufw