feat: allow to customize the number of sync nodes

anyproto · Jun 29, 2024 · f6552a9 · f6552a9
1 parent 82b9ec1
commit f6552a9
Show file tree

Hide file tree

Showing 13 changed files with 101 additions and 182 deletions.
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,4 @@
 /repos/
 /.env
 /.env.override
+/docker-compose.yml
diff --git a/Dockerfile-generateconfig-env b/Dockerfile-generateconfig-env
diff --git a/Dockerfile-generateconfig-env-and-compose b/Dockerfile-generateconfig-env-and-compose
@@ -0,0 +1,6 @@
+# syntax=docker/dockerfile:1
+FROM python:3.11-alpine
+WORKDIR /code
+COPY docker-generateconfig/requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+ENTRYPOINT ["/code/docker-generateconfig/env-and-compose.py"]
diff --git a/Dockerfile-generateconfig-processing b/Dockerfile-generateconfig-processing
@@ -1,7 +1,9 @@
 # syntax=docker/dockerfile:1
-FROM alpine:3.18.4
-RUN apk add --no-cache bash yq perl python3 py3-yaml
+FROM python:3.11-alpine
+RUN apk add --no-cache bash yq perl
 WORKDIR /code
+COPY docker-generateconfig/requirements.txt requirements.txt
+RUN pip install -r requirements.txt
 HEALTHCHECK \
     --start-period=3s \
     --retries=10 \

diff --git a/Makefile b/Makefile
@@ -1,12 +1,12 @@
 .DEFAULT_GOAL := start
 
-generate_env:
-	docker buildx build --tag generateconfig-env --file Dockerfile-generateconfig-env .
+generate_env_and_compose:
+	docker buildx build --tag generateconfig-env-and-compose --file Dockerfile-generateconfig-env-and-compose .
 	docker run --rm \
 		--volume ${CURDIR}/:/code/ \
-		generateconfig-env
+		generateconfig-env-and-compose
 
-start: generate_env
+start: generate_env_and_compose
 	docker compose up --detach --remove-orphans
 	@echo "Done! Upload your self-hosted network configuration file ${CURDIR}/etc/client.yml into the client app"
 	@echo "See: https://doc.anytype.io/anytype-docs/data-and-security/self-hosting#switching-between-networks"

diff --git a/docker-compose.yml → docker-compose.yml.j2 b/docker-compose.yml → docker-compose.yml.j2
@@ -4,7 +4,6 @@ services:
     build:
       context: .
       dockerfile: Dockerfile-generateconfig-anyconf
-    stop_signal: SIGKILL
     volumes:
       - ./:/code
       - "${STORAGE_DIR}:/code/storage"
@@ -13,19 +12,18 @@ services:
   generateconfig-processing:
     depends_on:
       generateconfig-anyconf:
-        condition: service_healthy
+        condition: service_completed_successfully
     build:
       context: .
       dockerfile: Dockerfile-generateconfig-processing
-    stop_signal: SIGKILL
     volumes:
       - ./:/code
       - "${STORAGE_DIR}:/code/storage"
 
   mongo-1:
     depends_on:
       generateconfig-processing:
-        condition: service_healthy
+        condition: service_completed_successfully
     image: "mongo:${MONGO_VERSION}"
     command: ["--replSet", "${MONGO_REPLICA_SET}", "--port", "${MONGO_1_PORT}"]
     ports:
@@ -40,7 +38,7 @@ services:
   redis:
     depends_on:
       generateconfig-processing:
-        condition: service_healthy
+        condition: service_completed_successfully
     image: "redis/redis-stack-server:${REDIS_VERSION}"
     restart: always
     command: ["redis-server", "--port", "${REDIS_PORT}", "--dir", "/data/", "--appendonly", "yes", "--maxmemory", "256mb", "--maxmemory-policy", "noeviction", "--protected-mode", "no", "--loadmodule", "/opt/redis-stack/lib/redisbloom.so"]
@@ -61,35 +59,27 @@ services:
     environment:
       MINIO_ROOT_USER: "${AWS_ACCESS_KEY_ID}"
       MINIO_ROOT_PASSWORD: "${AWS_SECRET_ACCESS_KEY}"
+      MC_HOST_minio: http://${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}@minio:${MINIO_PORT}
     ports:
       - "${EXTERNAL_MINIO_PORT}:${MINIO_PORT}"
       - "${EXTERNAL_MINIO_WEB_PORT}:${MINIO_WEB_PORT}"
     volumes:
       - "${STORAGE_DIR}/minio:/data"
     healthcheck:
-      test: bash -c ':> /dev/tcp/127.0.0.1/${MINIO_PORT}' || exit 1
-      interval: 5s
-      timeout: 10s
+      test: bash -c 'mc version info "minio/${MINIO_BUCKET}" || mc mb "minio/${MINIO_BUCKET}"'
+      interval: 10s
+      timeout: 5s
       retries: 3
     networks:
       default:
         aliases:
           - "${MINIO_BUCKET}.minio" # <bucket-name>.<endpoint-name>
 
-  create-bucket:
-    image: minio/mc:latest
-    environment:
-      MC_HOST_minio: http://${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}@minio:${MINIO_PORT}
-    restart: "no"
-    depends_on:
-      - minio
-    command: [ "mc", "mb", "minio/${MINIO_BUCKET}"]
-
   any-sync-coordinator_bootstrap:
     image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}"
     depends_on:
       generateconfig-processing:
-        condition: service_healthy
+        condition: service_completed_successfully
       mongo-1:
         condition: service_healthy
     volumes:
@@ -101,7 +91,7 @@ services:
     image: "ghcr.io/anyproto/any-sync-coordinator:${ANY_SYNC_COORDINATOR_VERSION}"
     depends_on:
       generateconfig-processing:
-        condition: service_healthy
+        condition: service_completed_successfully
       mongo-1:
         condition: service_healthy
       any-sync-coordinator_bootstrap:
@@ -142,65 +132,27 @@ services:
           memory: 500M
     restart: on-failure
 
-  any-sync-node-1:
-    image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}"
-    depends_on:
-      any-sync-coordinator:
-        condition: service_started
-    ports:
-      - "${ANY_SYNC_NODE_1_PORT}:${ANY_SYNC_NODE_1_PORT}"
-      - "${ANY_SYNC_NODE_1_QUIC_PORT}:${ANY_SYNC_NODE_1_QUIC_PORT}/udp"
-      - 8081:8080
-      - 8001:8000
-    volumes:
-      - ./etc/any-sync-node-1/:/etc/any-sync-node/
-      - "${STORAGE_DIR}/any-sync-node-1/:/storage/"
-      - "${STORAGE_DIR}/networkStore/any-sync-node-1/:/networkStore/"
-    deploy:
-      resources:
-        limits:
-          memory: 500M
-    restart: on-failure
-
-  any-sync-node-2:
-    image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}"
-    depends_on:
-      any-sync-coordinator:
-        condition: service_started
-    ports:
-      - "${ANY_SYNC_NODE_2_PORT}:${ANY_SYNC_NODE_2_PORT}"
-      - "${ANY_SYNC_NODE_2_QUIC_PORT}:${ANY_SYNC_NODE_2_QUIC_PORT}/udp"
-      - 8082:8080
-      - 8002:8000
-    volumes:
-      - ./etc/any-sync-node-2/:/etc/any-sync-node/
-      - "${STORAGE_DIR}/any-sync-node-2/:/storage/"
-      - "${STORAGE_DIR}/networkStore/any-sync-node-2/:/networkStore/"
-    deploy:
-      resources:
-        limits:
-          memory: 500M
-    restart: on-failure
-
-  any-sync-node-3:
+{% for node in nodes %}
+  any-sync-node-{{ node }}:
     image: "ghcr.io/anyproto/any-sync-node:${ANY_SYNC_NODE_VERSION}"
     depends_on:
       any-sync-coordinator:
         condition: service_started
     ports:
-      - "${ANY_SYNC_NODE_3_PORT}:${ANY_SYNC_NODE_3_PORT}"
-      - "${ANY_SYNC_NODE_3_QUIC_PORT}:${ANY_SYNC_NODE_3_QUIC_PORT}/udp"
-      - 8083:8080
-      - 8003:8000
+      - "${ANY_SYNC_NODE_{{ node }}_PORT}:${ANY_SYNC_NODE_{{ node }}_PORT}"
+      - "${ANY_SYNC_NODE_{{ node }}_QUIC_PORT}:${ANY_SYNC_NODE_{{ node }}_QUIC_PORT}/udp"
+      - 81{{ '%02d' % node | int }}:8080
+      - 82{{ '%02d' % node | int }}:8000
     volumes:
-      - ./etc/any-sync-node-3/:/etc/any-sync-node/
-      - "${STORAGE_DIR}/any-sync-node-3/:/storage/"
-      - "${STORAGE_DIR}/networkStore/any-sync-node-3/:/networkStore/"
+      - ./etc/any-sync-node-{{ node }}/:/etc/any-sync-node/
+      - "${STORAGE_DIR}/any-sync-node-{{ node }}/:/storage/"
+      - "${STORAGE_DIR}/networkStore/any-sync-node-{{ node }}/:/networkStore/"
     deploy:
       resources:
         limits:
           memory: 500M
     restart: on-failure
+{% endfor %}
 
   any-sync-consensusnode:
     image: "ghcr.io/anyproto/any-sync-consensusnode:${ANY_SYNC_CONSENSUSNODE_VERSION}"
@@ -226,10 +178,8 @@ services:
     depends_on:
       - any-sync-consensusnode
       - any-sync-filenode
-      - any-sync-coordinator
-      - any-sync-node-1
-      - any-sync-node-2
-      - any-sync-node-3
+      - any-sync-coordinator{% for node in nodes %}
+      - any-sync-node-{{ node }}{% endfor %}
     volumes:
       - ./:/code
       - "${STORAGE_DIR}:/code/storage"

diff --git a/docker-generateconfig/anyconf.sh b/docker-generateconfig/anyconf.sh
@@ -3,7 +3,9 @@
 echo "INFO: $0 start"
 echo "INFO: loading .env file"
 source .env
+ANY_SYNC_NODE_INDEXES=($(compgen -v | grep -o '^ANY_SYNC_NODE_\d\+_ADDRESSES' | grep -o '\d\+' | sort -n))
 
+SYNC_NODE_TEMPLATE="$(pwd)/docker-generateconfig/etc/node-1.yml"
 echo "INFO: create persistent config dir='./storage/docker-generateconfig'"
 install -d ./storage/docker-generateconfig
 cd ./storage/docker-generateconfig
@@ -25,36 +27,40 @@ fi
 NETWORK_ID=$( cat .networkId)
 NETWORK_SIGNING_KEY=$( cat .networkSigningKey )
 
-if [[ -s account0.yml ]]; then
+if [[ -s account-coordinator.yml ]]; then
     echo "INFO: saved nodes and accounts configuration found, skipping"
 else
     echo "INFO: save nodes and accounts not found, createing"
-    anyconf generate-nodes \
-        --t tree \
-        --t tree \
-        --t tree \
-        --t coordinator \
-        --t file \
-        --t consensus \
-        --addresses ${ANY_SYNC_NODE_1_ADDRESSES} \
-        --addresses ${ANY_SYNC_NODE_2_ADDRESSES} \
-        --addresses ${ANY_SYNC_NODE_3_ADDRESSES} \
-        --addresses ${ANY_SYNC_COORDINATOR_ADDRESSES} \
-        --addresses ${ANY_SYNC_FILENODE_ADDRESSES} \
-        --addresses ${ANY_SYNC_CONSENSUSNODE_ADDRESSES} \
+    node_type_args=("--t" "coordinator" "--t" "consensus" "--t" "file")
+    node_addr_args=("--addresses" "${ANY_SYNC_COORDINATOR_ADDRESSES}" "--addresses" "${ANY_SYNC_CONSENSUSNODE_ADDRESSES}" "--addresses" "${ANY_SYNC_FILENODE_ADDRESSES}")
+    for i in ${!ANY_SYNC_NODE_INDEXES[@]}; do
+        index="${ANY_SYNC_NODE_INDEXES[$i]}"
+        key="ANY_SYNC_NODE_${index}_ADDRESSES"
+        node_type_args+=("--t" "tree")
+        node_addr_args+=("--addresses" "${!key}")
+    done
+    anyconf generate-nodes "${node_type_args[@]}" "${node_addr_args[@]}"
 
     if [ $? -ne 0 ]; then
         echo "ERROR: Failed to generate nodes and accounts!"
         exit 1
     fi
+
+    mv account0.yml account-coordinator.yml
+    mv account1.yml account-consensus.yml
+    mv account2.yml account-file.yml
+    for i in ${!ANY_SYNC_NODE_INDEXES[@]}; do
+        index="${ANY_SYNC_NODE_INDEXES[$i]}"
+        mv "account$((i+3)).yml" "account-node-${index}.yml"
+        cp "$SYNC_NODE_TEMPLATE" "node-${index}.yml"
+        sed -i "s/ANY_SYNC_NODE_1_/ANY_SYNC_NODE_${index}_/g" "node-${index}.yml"
+    done
 fi
 
 echo "INFO: yq processing yml files"
 yq --indent 2 --inplace 'del(.creationTime)' nodes.yml
 yq --indent 2 --inplace ".networkId |= \"${NETWORK_ID}\"" nodes.yml
-yq --indent 2 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account3.yml
-yq --indent 2 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account5.yml
+yq --indent 2 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account-coordinator.yml
+yq --indent 2 --inplace ".account.signingKey |= \"${NETWORK_SIGNING_KEY}\"" account-consensus.yml
 
 echo "INFO: $0 done"
-echo "INFO: starting nc as status service"
-nc -lk -p 8000 -e /bin/cat
diff --git a/docker-generateconfig/env.py → docker-generateconfig/env-and-compose.py b/docker-generateconfig/env.py → docker-generateconfig/env-and-compose.py
@@ -5,11 +5,14 @@
 import json
 import requests
 import re
+from jinja2 import Environment, FileSystemLoader
 
 cfg = {
-    'inputFile': '.env.common',
-    'overrideFile': '.env.override',
-    'outputFile': '.env',
+    'inputEnvFile': '.env.common',
+    'overrideEnvFile': '.env.override',
+    'outputEnvFile': '.env',
+    'templateComposeFile': 'docker-compose.yml.j2',
+    'outputComposeFile': 'docker-compose.yml',
     'overrideVarMap': {
         'ANY_SYNC_NODE_VERSION': 'pkg::any-sync-node',
         'ANY_SYNC_FILENODE_VERSION': 'pkg::any-sync-filenode',
@@ -24,22 +27,22 @@
 
 # load variables from inputFile
 envVars = dict()
-if os.path.exists(cfg['inputFile']) and os.path.getsize(cfg['inputFile']) > 0:
-    with open(cfg['inputFile']) as file:
+if os.path.exists(cfg['inputEnvFile']) and os.path.getsize(cfg['inputEnvFile']) > 0:
+    with open(cfg['inputEnvFile']) as file:
         for line in file:
             if line.startswith('#') or not line.strip():
                 continue
             key, value = line.strip().split('=', 1)
             if key in envVars:
-                print(f"WARNING: dublicate key={key} in env file={cfg['inputFile']}")
+                print(f"WARNING: dublicate key={key} in env file={cfg['inputEnvFile']}")
             envVars[key] = value
 else:
-    print(f"ERROR: file={cfg['inputFile']} not found or size=0")
+    print(f"ERROR: file={cfg['inputEnvFile']} not found or size=0")
     exit(1)
 
 # override variables from overrideFile
-if os.path.exists(cfg['overrideFile']) and os.path.getsize(cfg['overrideFile']) > 0:
-    with open(cfg['overrideFile']) as file:
+if os.path.exists(cfg['overrideEnvFile']) and os.path.getsize(cfg['overrideEnvFile']) > 0:
+    with open(cfg['overrideEnvFile']) as file:
         for line in file:
             if line.startswith('#') or not line.strip():
                 continue
@@ -77,6 +80,13 @@ def getLatestVersions(role):
                 envVars[key] = 'v'+str(lastVersionValue)
 
 # save in output file
-with open(cfg['outputFile'], 'w') as file:
+with open(cfg['outputEnvFile'], 'w') as file:
     for key, value in envVars.items():
         file.write(f"{key}={value}\n")
+
+# generate docker-compose.yml
+nodes = {key.split('_')[3] for key in envVars.keys() if key.startswith('ANY_SYNC_NODE_') and key.endswith('_HOST')}
+j2 = Environment(loader=FileSystemLoader('.')).get_template(cfg['templateComposeFile'])
+rendered = j2.render({'nodes': nodes})
+with open(cfg['outputComposeFile'], 'w') as f:
+    f.write(rendered)
diff --git a/docker-generateconfig/env-requirements.txt b/docker-generateconfig/env-requirements.txt
diff --git a/docker-generateconfig/etc/node-2.yml b/docker-generateconfig/etc/node-2.yml