Sanity Test for nodes running in k8s (BFT-394)

.gitignore

@@ -9,3 +9,6 @@ logs/
 .terraform
 .ssh
 **/*terraform.tfstate*
+
+# Binaries generated in Docker
+node/tools/docker_binaries

Dockerfile

@@ -1,23 +1,43 @@
 # Build Stage
-FROM rust:latest as build
-COPY /node/ /node/
-COPY Makefile .
-WORKDIR /node
+FROM rust:latest as builder
+COPY /node/ /app/
+WORKDIR /app
 RUN apt-get update && apt-get install -y libclang-dev
 RUN cargo build --release
-RUN cd .. && make docker_node_configs
 
-# Runtime Stage
-FROM debian:stable-slim as runtime
+# Binary copy stage
+FROM scratch as executor-binary
+COPY --from=builder /app/target/release/executor .
+
+# Binary copy stage
+FROM scratch as tester-binary
+COPY --from=builder /app/target/release/tester .
 
-COPY --from=build /node/target/release/executor /node/
-COPY --from=build /node/tools/docker-config/nodes-config /node/
+# Executor runtime Stage
+FROM debian:stable-slim as executor-runtime
+
+COPY /node/tools/docker_binaries/executor /node/
+COPY /node/tools/k8s_configs/ /node/k8s_config
+COPY /node/tools/docker-config/ /node/docker_config
 COPY docker-entrypoint.sh /node/
+COPY k8s_entrypoint.sh /node/
 
 WORKDIR /node
 RUN chmod +x docker-entrypoint.sh
+RUN chmod +x k8s_entrypoint.sh
 
 ENTRYPOINT ["./docker-entrypoint.sh"]
 
 EXPOSE 3054
 EXPOSE 3051
+
+# Tester runtime Stage
+FROM debian:stable-slim as tester-runtime
+COPY node/tools/docker_binaries/tester /test/
+COPY node/tests/tester_entrypoint.sh /test/
+COPY node/tests/config.txt /test/
+
+WORKDIR /test
+
+RUN chmod +x tester_entrypoint.sh
+

Makefile

@@ -1,44 +1,77 @@
-.PHONY: node nodes_config docker_node_configs node_docker consensus_docker_example clean clean_docker
-IP?=127.0.0.1:3054
+.PHONY: node nodes_config docker_nodes_config node_docker consensus_docker_example clean clean_docker addresses_file blank_configs
+NODE?=0
 DOCKER_IP=172.12.0.10
 EXECUTABLE_NODE_DIR=node/tools
+NODES=4
+SEED_NODES=1
 
 # Locally run commands
 
 node:
-	export RUST_LOG=INFO && cd ${EXECUTABLE_NODE_DIR}/nodes-config/${IP} && cargo run -- --database ../../database/${IP}
+	export RUST_LOG=INFO && cd ${EXECUTABLE_NODE_DIR}/nodes-config/node_${NODE} && cargo run -- --database ../../database/node_${NODE}
 
 nodes_config:
 	cd ${EXECUTABLE_NODE_DIR} && cargo run --bin localnet_config -- --input-addrs addresses.txt --output-dir nodes-config
 
 # Docker commands
 
-# This command will run inside the Dockerfile and it's not necessary to use it outside there.
-docker_node_configs:
-	cd ${EXECUTABLE_NODE_DIR} && cargo run --release --bin localnet_config -- --input-addrs docker-config/addresses.txt --output-dir docker-config/nodes-config
+docker_build_executor:
+	docker build --output=node/tools/docker_binaries --target=executor-binary .
 
-node_docker:
-	mkdir -p ${EXECUTABLE_NODE_DIR}/docker-config
-	cd ${EXECUTABLE_NODE_DIR}/docker-config && rm -rf addresses.txt && echo ${DOCKER_IP}:3054 >> addresses.txt
-	docker-compose up -d node-1
+docker_node_image:
+	docker build -t consensus-node --target=executor-runtime .
+
+docker_nodes_config:
+	cd ${EXECUTABLE_NODE_DIR} && cargo run  --bin localnet_config -- --input-addrs docker-config/addresses.txt --output-dir docker-config
+
+docker_node:
+	$(MAKE) docker_node_image
+	docker run -d --name consensus-node-${NODE} --env NODE_ID="consensus-node-${NODE}" consensus-node
 
 consensus_docker_example:
 	mkdir -p ${EXECUTABLE_NODE_DIR}/docker-config
-	cd ${EXECUTABLE_NODE_DIR}/docker-config && rm -rf addresses.txt && touch addresses.txt && echo 172.12.0.10:3054 >> addresses.txt &&  echo 172.12.0.11:3054 >> addresses.txt
+	cd ${EXECUTABLE_NODE_DIR}/docker-config && rm -rf addresses.txt && echo 172.12.0.10:3054 >> addresses.txt &&  echo 172.12.0.11:3054 >> addresses.txt
+	$(MAKE) docker_nodes_config
+	$(MAKE) docker_node_image
 	docker-compose up -d
 
 stop_docker_nodes:
 	docker stop consensus-node-1 consensus-node-2
 
+start_k8s_nodes:
+	cd ${EXECUTABLE_NODE_DIR} && cargo run  --bin deployer generate-config --nodes ${NODES}
+	$(MAKE) docker_node_image
+	minikube image load consensus-node:latest
+	cd ${EXECUTABLE_NODE_DIR} && cargo run --release --bin deployer deploy --nodes ${NODES} --seed-nodes ${SEED_NODES}
+
 # Clean commands
 
-clean: clean_docker
+clean: clean_docker clean_k8s
 	rm -rf ${EXECUTABLE_NODE_DIR}/nodes-config
 	rm -rf ${EXECUTABLE_NODE_DIR}/database
 
+clean_k8s:
+	rm -rf ${EXECUTABLE_NODE_DIR}/k8s_configs
+	kubectl delete deployments --all
+	kubectl delete pods --all
+
 clean_docker:
-	rm -rf ${EXECUTABLE_NODE_DIR}/docker-config
 	docker rm -f consensus-node-1
 	docker rm -f consensus-node-2
 	docker network rm -f node-net
 	docker image rm -f consensus-node
+	docker image rm -f test-suite
+
+addresses_file:
+	mkdir -p ${EXECUTABLE_NODE_DIR}/docker-config
+	cd ${EXECUTABLE_NODE_DIR}/docker-config && \
+	rm -rf addresses.txt && \
+	touch addresses.txt && \
+	for n in $$(seq 0 $$((${NODES} - 1))); do echo 0.0.0.$$n:3054 >> addresses.txt; done
+
+blank_configs: addresses_file docker_node_configs
+	for n in $$(seq 0 $$((${NODES} - 1))); do \
+	   jq '.publicAddr = "0.0.0.0:3054"' node/tools/docker-config/nodes-config/node_$$n/config.json | \
+	   jq '.gossipStaticOutbound = "[]"' > node/tools/docker-config/nodes-config/node_$$n/config.tmp && \
+	   mv -f node/tools/docker-config/nodes-config/node_$$n/config.tmp node/tools/docker-config/nodes-config/node_$$n/config.json; \
+	done

compose.yaml

@@ -2,7 +2,9 @@ version: "3.9"
 
 services:
   node-1:
-    build: .
+    build:
+      context: .
+      target: runtime
     image: consensus-node
     container_name: consensus-node-1
     ports:

docker-entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # This file works as an entrypoint of the docker container running the node binary copied inside of it.
 
-cd $(hostname -i):3054
+cd docker_config/${NODE_ID}
 export RUST_LOG=INFO
-../executor
+../../executor $@

k8s_entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# This file works as an entrypoint of the kubernetes cluster running the node binary copied inside of it.
+
+cd k8s_config/${NODE_ID}
+export RUST_LOG=INFO
+../../executor $@