From 834c19c25960b960947b1a3580fc9fae360cc01c Mon Sep 17 00:00:00 2001
From: dydxwill <119354122+dydxwill@users.noreply.github.com>
Date: Thu, 7 Dec 2023 15:26:59 -0500
Subject: [PATCH] [IND-510] setup containerized Docker env for e2e testing
 (#851)

---
 e2e-testing/.env                          |   1 +
 e2e-testing/docker-compose-e2e-test.yml   | 229 ++++++++++++++++++++++
 e2e-testing/run-containerized-env.sh      |   4 +
 protocol/Makefile                         |   6 +
 protocol/testing/e2etest-local/Dockerfile |  14 ++
 protocol/testing/e2etest-local/local.sh   | 173 ++++++++++++++++
 protocol/testing/genesis.sh               |  31 +++
 7 files changed, 458 insertions(+)
 create mode 100644 e2e-testing/.env
 create mode 100644 e2e-testing/docker-compose-e2e-test.yml
 create mode 100755 e2e-testing/run-containerized-env.sh
 create mode 100644 protocol/testing/e2etest-local/Dockerfile
 create mode 100755 protocol/testing/e2etest-local/local.sh

diff --git a/e2e-testing/.env b/e2e-testing/.env
new file mode 100644
index 0000000000..b7671ec50e
--- /dev/null
+++ b/e2e-testing/.env
@@ -0,0 +1 @@
+ETH_RPC_ENDPOINT=https://eth-sepolia.g.alchemy.com/v2/demo
diff --git a/e2e-testing/docker-compose-e2e-test.yml b/e2e-testing/docker-compose-e2e-test.yml
new file mode 100644
index 0000000000..0357b9dd30
--- /dev/null
+++ b/e2e-testing/docker-compose-e2e-test.yml
@@ -0,0 +1,229 @@
+version: '3'
+services:
+  kafka:
+    image: blacktop/kafka:2.6
+    ports:
+      - 9092:9092
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: kafka
+      KAFKA_CREATE_TOPICS:
+        "to-ender:1:1,\
+        to-vulcan:1:1,\
+        to-websockets-orderbooks:1:1,\
+        to-websockets-subaccounts:1:1,\
+        to-websockets-trades:1:1,\
+        to-websockets-markets:1:1,\
+        to-websockets-candles:1:1"
+      KAFKA_LISTENERS: INTERNAL://:9092,EXTERNAL_SAME_HOST://:29092
+      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,EXTERNAL_SAME_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL_SAME_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+      DD_AGENT_HOST: datadog-agent
+    healthcheck:
+      test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic to-websockets-candles --describe"]
+      interval: 5s
+      timeout: 20s
+      retries: 50
+  postgres:
+    build:
+      context: ../indexer
+      dockerfile: ../indexer/Dockerfile.postgres.local
+    ports:
+      - 5435:5432
+    environment:
+      POSTGRES_PASSWORD: dydxserver123
+      POSTGRES_USER: dydx_dev
+      DATADOG_POSTGRES_PASSWORD: dydxserver123
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U dydx_dev"]
+      interval: 5s
+      timeout: 20s
+      retries: 10
+  redis:
+    image: redis:5.0.6-alpine
+    ports:
+      - 6382:6379
+  dydxprotocold0:
+    image: local:e2etest-dydxprotocol
+    entrypoint:
+      - cosmovisor
+      - run
+      - start
+      - --log_level
+      # Note that only this validator has a log-level of `info`; other validators use `error` by default.
+      # Change to `debug` for more verbose log-level.
+      - info 
+      - --home
+      - /dydxprotocol/chain/.alice
+      - --p2p.persistent_peers 
+      - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656"
+      - --bridge-daemon-eth-rpc-endpoint
+      - "${ETH_RPC_ENDPOINT}"
+    environment:
+      - DAEMON_HOME=/dydxprotocol/chain/.alice
+    volumes:
+      - ../protocol/localnet/dydxprotocol0:/dydxprotocol/chain/.alice/data
+    ports:
+      - "26657:26657"
+      - "9090:9090"
+      - "1317:1317"
+  
+  # This is the Indexer connected node.
+  # TODO: remove stake and make this a full node.
+  dydxprotocold1:
+    image: local:e2etest-dydxprotocol
+    entrypoint:
+       - cosmovisor
+       - run
+       - start
+       - --log_level
+       - error
+       - --home
+       - /dydxprotocol/chain/.bob
+       - --p2p.persistent_peers 
+       - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,b69182310be02559483e42c77b7b104352713166@dydxprotocold1:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656"
+       - --non-validating-full-node=true
+       - --bridge-daemon-eth-rpc-endpoint
+       - "${ETH_RPC_ENDPOINT}"
+       - --indexer-kafka-conn-str
+       - "kafka:9092"
+    environment:
+      - DAEMON_HOME=/dydxprotocol/chain/.bob
+    volumes:
+      - ../protocol/localnet/dydxprotocol1:/dydxprotocol/chain/.bob/data
+    ports:
+      - "26658:26657"
+    depends_on:
+      kafka:
+        condition: service_healthy
+ 
+  dydxprotocold2:
+    image: local:e2etest-dydxprotocol
+    entrypoint:
+       - cosmovisor
+       - run
+       - start
+       - --log_level
+       - error
+       - --home
+       - /dydxprotocol/chain/.carl
+       - --p2p.persistent_peers
+       - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656"
+       - --bridge-daemon-eth-rpc-endpoint
+       - "${ETH_RPC_ENDPOINT}"
+    environment:
+      - DAEMON_HOME=/dydxprotocol/chain/.carl
+    volumes:
+      - ../protocol/localnet/dydxprotocol2:/dydxprotocol/chain/.carl/data
+  
+  dydxprotocold3:
+    image: local:e2etest-dydxprotocol
+    entrypoint:
+       - cosmovisor
+       - run
+       - start
+       - --log_level
+       - error
+       - --home
+       - /dydxprotocol/chain/.dave
+       - --p2p.persistent_peers 
+       - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656"
+       - --bridge-daemon-eth-rpc-endpoint
+       - "${ETH_RPC_ENDPOINT}"
+    environment:
+      - DAEMON_HOME=/dydxprotocol/chain/.dave
+    volumes:
+      - ../protocol/localnet/dydxprotocol3:/dydxprotocol/chain/.dave/data
+
+  postgres-package:
+    build:
+      context: ../indexer
+      dockerfile: ../indexer/Dockerfile.postgres-package.local
+    links:
+      - postgres
+    depends_on:
+      postgres:
+        condition: service_healthy
+  ender:
+    build:
+      context: ../indexer
+      dockerfile: ../indexer/Dockerfile.service.local
+      args:
+        service: ender
+    ports:
+      - 3001:3001
+    links:
+      - postgres
+    environment:
+      - REDIS_URL=redis://redis:6379
+    depends_on:
+      kafka:
+        condition: service_healthy
+      postgres-package:
+        condition: service_completed_successfully
+  comlink:
+    build:
+      context: ../indexer
+      dockerfile: ../indexer/Dockerfile.service.local
+      args:
+        service: comlink
+    environment:
+      - PORT=3002
+      - REDIS_URL=redis://redis:6379
+      - RATE_LIMIT_REDIS_URL=redis://redis:6379
+      - RATE_LIMIT_ENABLED=false
+      - INDEXER_LEVEL_GEOBLOCKING_ENABLED=false
+      - COMPLIANCE_DATA_CLIENT=PLACEHOLDER
+    ports:
+      - 3002:3002
+    links:
+      - postgres
+    depends_on:
+      postgres-package:
+        condition: service_completed_successfully
+  socks:
+    build:
+      context: ../indexer
+      dockerfile: ../indexer/Dockerfile.service.local
+      args:
+        service: socks
+    ports:
+      - 3003:3003
+    links:
+      - postgres
+    environment:
+      - WS_PORT=3003
+      - COMLINK_URL=host.docker.internal:3002
+    depends_on:
+      kafka:
+        condition: service_healthy
+      postgres-package:
+        condition: service_completed_successfully
+  roundtable:
+    build:
+      context: ../indexer
+      dockerfile: ../indexer/Dockerfile.service.local
+      args:
+        service: roundtable
+    ports:
+      - 3004:3004
+    links:
+      - postgres
+    depends_on:
+      kafka:
+        condition: service_healthy
+      postgres-package:
+        condition: service_completed_successfully
+  vulcan:
+    build:
+      context: ../indexer
+      dockerfile: ../indexer/Dockerfile.service.local
+      args:
+        service: vulcan
+    environment:
+      - REDIS_URL=redis://redis:6379
+    ports:
+      - 3005:3005
+    depends_on:
+      kafka:
+        condition: service_healthy
diff --git a/e2e-testing/run-containerized-env.sh b/e2e-testing/run-containerized-env.sh
new file mode 100755
index 0000000000..051b5c65f7
--- /dev/null
+++ b/e2e-testing/run-containerized-env.sh
@@ -0,0 +1,4 @@
+cd ../protocol
+make e2etest-build-image
+cd ../e2e-testing
+docker compose -f docker-compose-e2e-test.yml up
diff --git a/protocol/Makefile b/protocol/Makefile
index 0ceef16a91..c74912db58 100644
--- a/protocol/Makefile
+++ b/protocol/Makefile
@@ -373,9 +373,15 @@ localnet-compose-upd:
 	@docker build . -t local:dydxprotocol -f testing/testnet-local/Dockerfile --no-cache
 	@docker-compose -f docker-compose.yml up --force-recreate -d $(ARGS)
 
+build-e2etest-image:
+	@echo "Build e2e test image at commit ${GIT_COMMIT_HASH}"
+	@docker build . -t local:e2etest-dydxprotocol -f testing/e2etest-local/Dockerfile --no-cache
+
 localnet-start: localnet-init localnet-compose-up
 localnet-startd: localnet-init localnet-compose-upd
 
+e2etest-build-image: localnet-init build-e2etest-image
+
 # Continue the localnet with the same chain state.
 localnet-continue:
 	@docker-compose -f docker-compose.yml up $(ARGS)
diff --git a/protocol/testing/e2etest-local/Dockerfile b/protocol/testing/e2etest-local/Dockerfile
new file mode 100644
index 0000000000..345cf49f66
--- /dev/null
+++ b/protocol/testing/e2etest-local/Dockerfile
@@ -0,0 +1,14 @@
+FROM dydxprotocol-base
+
+COPY ./testing/e2etest-local/local.sh /dydxprotocol/local.sh
+COPY ./testing/genesis.sh /dydxprotocol/genesis.sh
+COPY ./testing/start.sh /dydxprotocol/start.sh
+COPY ./daemons/pricefeed/client/constants/testdata /dydxprotocol/exchange_config
+COPY ./testing/delaymsg_config /dydxprotocol/delaymsg_config
+
+RUN go install cosmossdk.io/tools/cosmovisor/cmd/cosmovisor@v1.4.0
+
+RUN /dydxprotocol/local.sh
+
+ENV DAEMON_NAME=dydxprotocold
+ENTRYPOINT ["cosmovisor", "run"]
diff --git a/protocol/testing/e2etest-local/local.sh b/protocol/testing/e2etest-local/local.sh
new file mode 100755
index 0000000000..0e1f28e4c2
--- /dev/null
+++ b/protocol/testing/e2etest-local/local.sh
@@ -0,0 +1,173 @@
+#!/bin/bash
+set -eo pipefail
+
+# This file initializes muliple validators for local and CI testing purposes.
+# This file should be run as part of `docker-compose-e2e-test.yml`.
+
+source "./genesis.sh"
+
+CHAIN_ID="localdydxprotocol"
+
+# Define mnemonics for all validators.
+MNEMONICS=(
+	# alice
+	# Consensus Address: dydxvalcons1zf9csp5ygq95cqyxh48w3qkuckmpealrw2ug4d
+	"merge panther lobster crazy road hollow amused security before critic about cliff exhibit cause coyote talent happy where lion river tobacco option coconut small"
+
+	# bob
+	# Consensus Address: dydxvalcons1s7wykslt83kayxuaktep9fw8qxe5n73ucftkh4
+	"color habit donor nurse dinosaur stable wonder process post perfect raven gold census inside worth inquiry mammal panic olive toss shadow strong name drum"
+
+	# carl
+	# Consensus Address: dydxvalcons1vy0nrh7l4rtezrsakaadz4mngwlpdmhy64h0ls
+	"school artefact ghost shop exchange slender letter debris dose window alarm hurt whale tiger find found island what engine ketchup globe obtain glory manage"
+
+	# dave
+	# Consensus Address: dydxvalcons1stjspktkshgcsv8sneqk2vs2ws0nw2wr272vtt
+	"switch boring kiss cash lizard coconut romance hurry sniff bus accident zone chest height merit elevator furnace eagle fetch quit toward steak mystery nest"
+)
+
+# Define node keys for all validators.
+NODE_KEYS=(
+	# Node ID: 17e5e45691f0d01449c84fd4ae87279578cdd7ec
+	"8EGQBxfGMcRfH0C45UTedEG5Xi3XAcukuInLUqFPpskjp1Ny0c5XvwlKevAwtVvkwoeYYQSe0geQG/cF3GAcUA=="
+
+	# Node ID: b69182310be02559483e42c77b7b104352713166
+	"3OZf5HenMmeTncJY40VJrNYKIKcXoILU5bkYTLzTJvewowU2/iV2+8wSlGOs9LoKdl0ODfj8UutpMhLn5cORlw=="
+
+	# Node ID: 47539956aaa8e624e0f1d926040e54908ad0eb44
+	"tWV4uEya9Xvmm/kwcPTnEQIV1ZHqiqUTN/jLPHhIBq7+g/5AEXInokWUGM0shK9+BPaTPTNlzv7vgE8smsFg4w=="
+
+	# Node ID: 5882428984d83b03d0c907c1f0af343534987052
+	"++C3kWgFAs7rUfwAHB7Ffrv43muPg0wTD2/UtSPFFkhtobooIqc78UiotmrT8onuT1jg8/wFPbSjhnKRThTRZg=="
+)
+
+# Define monikers for each validator. These are made up strings and can be anything.
+# This also controls in which directory the validator's home will be located. i.e. `/dydxprotocol/chain/.alice`
+MONIKERS=(
+	"alice"
+	"bob"
+	"carl"
+	"dave"
+)
+
+# Define all test accounts for the chain.
+TEST_ACCOUNTS=(
+	"dydx199tqg4wdlnu4qjlxchpd7seg454937hjrknju4" # alice
+	"dydx10fx7sy6ywd5senxae9dwytf8jxek3t2gcen2vs" # bob
+	"dydx1fjg6zp6vv8t9wvy4lps03r5l4g7tkjw9wvmh70" # carl
+	"dydx1wau5mja7j7zdavtfq9lu7ejef05hm6ffenlcsn" # dave
+)
+
+FAUCET_ACCOUNTS=(
+	"dydx1nzuttarf5k2j0nug5yzhr6p74t9avehn9hlh8m" # main faucet
+)
+
+# Define dependencies for this script.
+# `jq` and `dasel` are used to manipulate json and yaml files respectively.
+install_prerequisites() {
+	apk add dasel jq
+}
+
+# Create all validators for the chain including a full-node.
+# Initialize their genesis files and home directories.
+create_validators() {
+	# Create temporary directory for all gentx files.
+	mkdir /tmp/gentx
+
+	# Iterate over all validators and set up their home directories, as well as generate `gentx` transaction for each.
+	for i in "${!MONIKERS[@]}"; do
+		VAL_HOME_DIR="$HOME/chain/.${MONIKERS[$i]}"
+		VAL_CONFIG_DIR="$VAL_HOME_DIR/config"
+
+		# Initialize the chain and validator files.
+		dydxprotocold init "${MONIKERS[$i]}" -o --chain-id=$CHAIN_ID --home "$VAL_HOME_DIR"
+
+		# Overwrite the randomly generated `priv_validator_key.json` with a key generated deterministically from the mnemonic.
+		dydxprotocold tendermint gen-priv-key --home "$VAL_HOME_DIR" --mnemonic "${MNEMONICS[$i]}"
+
+		# Note: `dydxprotocold init` non-deterministically creates `node_id.json` for each validator.
+		# This is inconvenient for persistent peering during testing in Terraform configuration as the `node_id`
+		# would change with every build of this container.
+		#
+		# For that reason we overwrite the non-deterministically generated one with a deterministic key defined in this file here.
+		new_file=$(jq ".priv_key.value = \"${NODE_KEYS[$i]}\"" "$VAL_CONFIG_DIR"/node_key.json)
+		cat <<<"$new_file" >"$VAL_CONFIG_DIR"/node_key.json
+
+		edit_config "$VAL_CONFIG_DIR"
+
+		# Using "*" as a subscript results in a single arg: "dydx1... dydx1... dydx1..."
+		# Using "@" as a subscript results in separate args: "dydx1..." "dydx1..." "dydx1..."
+		# Note: `edit_genesis` must be called before `add-genesis-account`.
+		edit_genesis "$VAL_CONFIG_DIR" "${TEST_ACCOUNTS[*]}" "${FAUCET_ACCOUNTS[*]}" "" "" "" ""
+		update_genesis_use_test_volatile_market "$VAL_CONFIG_DIR"
+		update_all_markets_with_fixed_price_exchange "$VAL_CONFIG_DIR"
+		update_genesis_complete_bridge_delay "$VAL_CONFIG_DIR" "30"
+
+		echo "${MNEMONICS[$i]}" | dydxprotocold keys add "${MONIKERS[$i]}" --recover --keyring-backend=test --home "$VAL_HOME_DIR"
+
+		for acct in "${TEST_ACCOUNTS[@]}"; do
+			dydxprotocold add-genesis-account "$acct" 100000000000000000$USDC_DENOM,$TESTNET_VALIDATOR_NATIVE_TOKEN_BALANCE$NATIVE_TOKEN --home "$VAL_HOME_DIR"
+		done
+		for acct in "${FAUCET_ACCOUNTS[@]}"; do
+			dydxprotocold add-genesis-account "$acct" 900000000000000000$USDC_DENOM,$TESTNET_VALIDATOR_NATIVE_TOKEN_BALANCE$NATIVE_TOKEN --home "$VAL_HOME_DIR"
+		done
+
+		dydxprotocold gentx "${MONIKERS[$i]}" $TESTNET_VALIDATOR_SELF_DELEGATE_AMOUNT$NATIVE_TOKEN --moniker="${MONIKERS[$i]}" --keyring-backend=test --chain-id=$CHAIN_ID --home "$VAL_HOME_DIR"
+
+		# Copy the gentx to a shared directory.
+		cp -a "$VAL_CONFIG_DIR/gentx/." /tmp/gentx
+	done
+
+	# Copy gentxs to the first validator's home directory to build the genesis json file
+	FIRST_VAL_HOME_DIR="$HOME/chain/.${MONIKERS[0]}"
+	FIRST_VAL_CONFIG_DIR="$FIRST_VAL_HOME_DIR/config"
+
+	rm -rf "$FIRST_VAL_CONFIG_DIR/gentx"
+	mkdir "$FIRST_VAL_CONFIG_DIR/gentx"
+	cp -r /tmp/gentx "$FIRST_VAL_CONFIG_DIR"
+
+	# Build the final genesis.json file that all validators and the full-nodes will use.
+	dydxprotocold collect-gentxs --home "$FIRST_VAL_HOME_DIR"
+
+	# Copy this genesis file to each of the other validators
+	for i in "${!MONIKERS[@]}"; do
+		if [[ "$i" == 0 ]]; then
+			# Skip first moniker as it already has the correct genesis file.
+			continue
+		fi
+
+		VAL_HOME_DIR="$HOME/chain/.${MONIKERS[$i]}"
+		VAL_CONFIG_DIR="$VAL_HOME_DIR/config"
+		rm -rf "$VAL_CONFIG_DIR/genesis.json"
+		cp "$FIRST_VAL_CONFIG_DIR/genesis.json" "$VAL_CONFIG_DIR/genesis.json"
+	done
+}
+
+setup_cosmovisor() {
+	for i in "${!MONIKERS[@]}"; do
+		VAL_HOME_DIR="$HOME/chain/.${MONIKERS[$i]}"
+		export DAEMON_NAME=dydxprotocold
+		export DAEMON_HOME="$HOME/chain/.${MONIKERS[$i]}"
+
+		cosmovisor init /bin/dydxprotocold
+	done
+}
+
+# TODO(DEC-1894): remove this function once we migrate off of persistent peers.
+# Note: DO NOT add more config modifications in this method. Use `cmd/config.go` to configure
+# the default config values.
+edit_config() {
+	CONFIG_FOLDER=$1
+
+	# Disable pex
+	dasel put -t bool -f "$CONFIG_FOLDER"/config.toml '.p2p.pex' -v 'false'
+
+	# Default `timeout_commit` is 999ms. For local testnet, use a larger value to make 
+	# block time longer for easier troubleshooting.
+	dasel put -t string -f "$CONFIG_FOLDER"/config.toml '.consensus.timeout_commit' -v '5s'
+}
+
+install_prerequisites
+create_validators
+setup_cosmovisor
diff --git a/protocol/testing/genesis.sh b/protocol/testing/genesis.sh
index da3c29f904..6efae9d71f 100755
--- a/protocol/testing/genesis.sh
+++ b/protocol/testing/genesis.sh
@@ -1572,6 +1572,37 @@ function update_genesis_use_test_volatile_market() {
 	dasel put -t int -f "$GENESIS" '.app_state.clob.clob_pairs.last().quantum_conversion_exponent' -v '-8'
 }
 
+# Modify the genesis file to only use fixed price exchange.
+function update_all_markets_with_fixed_price_exchange() {
+    GENESIS=$1/genesis.json
+
+    # Read the number of markets
+    NUM_MARKETS=$(jq -c '.app_state.prices.market_params | length' < "${GENESIS}")
+
+    # Loop through each market and update the parameters
+    for ((j = 0; j < NUM_MARKETS; j++)); do
+        # Get the current ticker
+        TICKER=$(jq -r ".app_state.prices.market_params[$j].pair" < "${GENESIS}")
+
+        # Update the exchange_config_json using the EOF syntax
+        exchange_config_json=$(cat <<-EOF
+{
+    "exchanges": [
+        {
+            "exchangeName": "TestFixedPriceExchange",
+            "ticker": "${TICKER}"
+        }
+    ]
+}
+EOF
+        )
+        dasel put -t string -f "$GENESIS" ".app_state.prices.market_params.[$j].exchange_config_json" -v "$exchange_config_json"
+
+        # Update the min_exchanges
+        dasel put -t int -f "$GENESIS" ".app_state.prices.market_params.[$j].min_exchanges" -v "1"
+    done
+}
+
 # Modify the genesis file with reduced complete bridge delay (for testing in non-prod envs).
 update_genesis_complete_bridge_delay() {
 	GENESIS=$1/genesis.json