Skip to content

Commit

Permalink
Hetzner module (#303)
Browse files Browse the repository at this point in the history
* restore the script files deleted in #263

* change scripts path for testing module

* create hetzner module for resources

* make provisioners reusable by main module and testing module

* change backend

* Fixes based on PR agent
  • Loading branch information
DaMandal0rian authored Jun 11, 2024
1 parent 51c00c9 commit 6bdc29a
Show file tree
Hide file tree
Showing 15 changed files with 810 additions and 50 deletions.
9 changes: 9 additions & 0 deletions resources/hetzner/backend.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
terraform {
cloud {
organization = "subspace-sre"

workspaces {
name = var.workspace_name
}
}
}
84 changes: 84 additions & 0 deletions resources/hetzner/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
module "network" {
source = "../../templates/terraform/hetzner"
path_to_scripts = "../../templates/scripts"
network_name = var.network_name

bootstrap-node-config = {
deployment-version = 1
instance-count = var.instance_count["bootstrap"]
repo-org = "subspace"
node-tag = "bootstrap-node"
additional-node-ips = var.additional_node_ips["bootstrap"]
reserved-only = true
prune = false
genesis-hash = var.genesis_hash
dsn-listen-port = 30533
node-dsn-port = 30433
}

bootstrap-node-evm-config = {
deployment-version = 1
instance-count = var.instance_count["bootstrap"]
repo-org = "subspace"
node-tag = "bootstrap-node"
additional-node-ips = var.additional_node_ips["bootstrap"]
reserved-only = true
prune = false
genesis-hash = var.genesis_hash
dsn-listen-port = 30533
node-dsn-port = 30433
operator-port = 30334
}

node-config = {
deployment-version = 1
instance-count = var.instance_count["node"]
repo-org = "subspace"
node-tag = "subspace-node"
additional-node-ips = var.additional_node_ips["node"]
reserved-only = true
prune = false
node-dsn-port = 30433
}

domain-node-config = {
deployment-version = 1
instance-count = var.instance_count["domain"]
repo-org = "subspace"
node-tag = "subspace-node"
additional-node-ips = var.additional_node_ips["domain"]
domain-prefix = "domain"
reserved-only = true
prune = false
node-dsn-port = 30433
enable-domains = true
domain-id = var.domain_id
domain-labels = var.domain_labels
}

farmer-node-config = {
deployment-version = 1
instance-count = var.instance_count["farmer"]
repo-org = "subspace"
node-tag = "farmer-node"
additional-node-ips = var.additional_node_ips["farmer"]
reserved-only = true
prune = false
plot-size = "10G"
reward-address = var.farmer_reward_address
force-block-production = true
node-dsn-port = 30433

}

tf_token = var.tf_token
private_key_path = var.private_key_path
branch_name = var.branch_name
ssh_user = var.ssh_user
genesis_hash = var.genesis_hash
}

# External data source to run the shell command and extract the value of the operator bootnode connection parameter
data "external" "operator_peer_multiaddr" {
program = ["bash", "-c", "echo '{\"OPERATOR_MULTI_ADDR\": \"'$(sed -nr 's/^NODE_0_OPERATOR_MULTI_ADDR=(.*)/\\1/p' ./bootstrap_node_evm_keys.txt)'\"}'"]
}
30 changes: 30 additions & 0 deletions resources/hetzner/outputs.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
output "bootstrap-node-ipv4-addresses" {
value = module.network.bootstrap-node-ipv4-addresses
description = "Bootstrap node IPv4 Addresses"
}

output "bootstrap-node-evm-ipv4-addresses" {
value = module.network.bootstrap-node-evm-ipv4-addresses
description = "Bootstrap node EVM IPv4 Addresses"
}

output "node-ipv4-addresses" {
value = module.network.node-ipv4-addresses
description = "subspace node IPv4 Addresses"
}

output "domain-node-ipv4-addresses" {
value = module.network.domain-node-ipv4-addresses
description = "domain node IPv4 Addresses"
}


output "farmer-nodes-ipv4-addresses" {
value = module.network.farmer-node-ipv4-addresses
description = "Farmer node IPv4 Addresses"
}

# Output the operator_peer_multiaddr value
output "operator_peer_multiaddr" {
value = data.external.operator_peer_multiaddr.result["operator_peer_multiaddr"]
}
76 changes: 76 additions & 0 deletions resources/hetzner/variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
variable "farmer_reward_address" {
description = "Farmer's reward address"
type = string
}

variable "network_name" {
description = "Network name"
type = string
}

//todo change this to a map
variable "domain_id" {
description = "Domain ID"
type = list(number)
default = [0]
}

//todo change this to a map
variable "domain_labels" {
description = "Tag of the domain to run"
type = list(string)
default = ["evm"]
}

variable "instance_count" {
type = map(number)
default = {
bootstrap = 2
node = 1
farmer = 1
domain = 2
evm_bootstrap = 1
}
}

variable "additional_node_ips" {
type = map(list(string))
default = {
bootstrap = [""]
node = [""]
farmer = [""]
domain = [""]
evm_bootstrap = [""]
}
}

variable "ssh_user" {
type = string
default = "root"
}

variable "private_key_path" {
type = string
default = "~/.ssh/hetzner"
}

variable "tf_token" {
type = string
sensitive = true
}

variable "branch_name" {
description = "name of testing branch"
type = string
default = "main"
}

variable "genesis_hash" {
description = "Genesis hash"
type = string
}

variable "workspace_name" {
description = "Name of the workspace"
type = string
}
25 changes: 21 additions & 4 deletions templates/terraform/hetzner/bootstrap_node_evm_provisioner.tf
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,19 @@ resource "null_resource" "setup-bootstrap-nodes-evm" {

}

resource "null_resource" "clone_branch" {
count = var.branch_name != "main" ? 1 : 0

provisioner "remote-exec" {
inline = [
"cd /root/subspace/",
"git clone https://github.com/subspace/subspace.git",
"cd subspace",
"git checkout ${var.branch_name}"
]
}
}

resource "null_resource" "prune-bootstrap-nodes-evm" {
count = var.bootstrap-node-evm-config.prune ? length(local.bootstrap_nodes_evm_ip_v4) : 0
depends_on = [null_resource.setup-bootstrap-nodes-evm]
Expand Down Expand Up @@ -135,8 +148,8 @@ resource "null_resource" "start-bootstrap-nodes-evm" {
"sudo hostnamectl set-hostname ${var.network_name}-bootstrap-node-evm-${count.index}",

# create .env file
"echo NODE_ORG=${var.bootstrap-node-evm-config.docker-org} > /root/subspace/.env",
"echo NODE_TAG=${var.bootstrap-node-evm-config.docker-tag} >> /root/subspace/.env",
"echo NODE_ORG=${var.bootstrap-node-evm-config.repo-org} > /root/subspace/.env",
"echo NODE_TAG=${var.bootstrap-node-evm-config.node-tag} >> /root/subspace/.env",
"echo NETWORK_NAME=${var.network_name} >> /root/subspace/.env",
"echo NODE_ID=${count.index} >> /root/subspace/.env",
"echo NODE_KEY=$(sed -nr 's/NODE_${count.index}_KEY=//p' /root/subspace/node_keys.txt) >> /root/subspace/.env",
Expand All @@ -151,12 +164,16 @@ resource "null_resource" "start-bootstrap-nodes-evm" {
"echo NODE_DSN_PORT=${var.bootstrap-node-evm-config.node-dsn-port} >> /root/subspace/.env",
"echo OPERATOR_PORT=${var.bootstrap-node-evm-config.operator-port} >> /root/subspace/.env",
"echo GENESIS_HASH=${var.bootstrap-node-evm-config.genesis-hash} >> /root/subspace/.env",
"echo BRANCH_NAME=${var.branch_name} >> /root/subspace/.env",

# create docker compose file
"bash /root/subspace/create_compose_file.sh ${var.bootstrap-node-evm-config.reserved-only} ${length(local.bootstrap_nodes_evm_ip_v4)} ${count.index} ${length(local.bootstrap_nodes_ip_v4)} ${var.domain-node-config.enable-domains} ",
"bash /root/subspace/create_compose_file.sh ${var.bootstrap-node-evm-config.reserved-only} ${length(local.bootstrap_nodes_evm_ip_v4)} ${count.index} ${length(local.bootstrap_nodes_ip_v4)} ${var.domain-node-config.enable-domains}",

# start subspace node
"sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d",
var.branch_name != "main" ? join(" && ", [
"cp -f /root/subspace/.env /root/subspace/subspace/.env",
"sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d"
]) : "sudo docker compose -f /root/subspace/docker-compose.yml up -d"
]
}
}
14 changes: 9 additions & 5 deletions templates/terraform/hetzner/bootstrap_node_provisioner.tf
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,11 @@ resource "null_resource" "setup-bootstrap-nodes" {
"sudo bash /root/subspace/installer.sh",
]
}
}

resource "null_resource" "clone_branch" {
count = var.branch_name != "main" ? 1 : 0

# clone testing branch
provisioner "remote-exec" {
inline = [
"cd /root/subspace/",
Expand All @@ -53,7 +56,6 @@ resource "null_resource" "setup-bootstrap-nodes" {
"git checkout ${var.branch_name}"
]
}

}

resource "null_resource" "prune-bootstrap-nodes" {
Expand Down Expand Up @@ -146,14 +148,16 @@ resource "null_resource" "start-boostrap-nodes" {
"echo DSN_LISTEN_PORT=${var.bootstrap-node-config.dsn-listen-port} >> /root/subspace/.env",
"echo NODE_DSN_PORT=${var.bootstrap-node-config.node-dsn-port} >> /root/subspace/.env",
"echo GENESIS_HASH=${var.bootstrap-node-config.genesis-hash} >> /root/subspace/.env",
"echo BRANCH_NAME=${var.branch_name} >> /root/subspace/.env",

# create docker compose file
"chmod +x /root/subspace/create_compose_file.sh",
"bash /root/subspace/create_compose_file.sh ${var.bootstrap-node-config.reserved-only} ${length(local.bootstrap_nodes_ip_v4)} ${count.index}",

# start subspace node
"cp -f /root/subspace/.env /root/subspace/subspace/.env",
"sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d",
var.branch_name != "main" ? join(" && ", [
"cp -f /root/subspace/.env /root/subspace/subspace/.env",
"sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d"
]) : "sudo docker compose -f /root/subspace/docker-compose.yml up -d"
]
}
}
39 changes: 10 additions & 29 deletions templates/terraform/hetzner/domain_node_provisioner.tf
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,11 @@ resource "null_resource" "setup-domain-nodes" {
"sudo bash /root/subspace/installer.sh",
]
}
# clone testing branch
}

resource "null_resource" "clone_branch" {
count = var.branch_name != "main" ? 1 : 0

provisioner "remote-exec" {
inline = [
"cd /root/subspace/",
Expand All @@ -51,7 +55,6 @@ resource "null_resource" "setup-domain-nodes" {
"git checkout ${var.branch_name}"
]
}

}

resource "null_resource" "prune-domain-nodes" {
Expand Down Expand Up @@ -163,38 +166,16 @@ resource "null_resource" "start-domain-nodes" {
"echo RELAYER_DOMAIN_ID=$(sed -nr 's/NODE_${count.index}_RELAYER_DOMAIN_ID=//p' /root/subspace/relayer_ids.txt) >> /root/subspace/.env",
"echo PIECE_CACHE_SIZE=${var.piece_cache_size} >> /root/subspace/.env",
"echo NODE_DSN_PORT=${var.domain-node-config.node-dsn-port} >> /root/subspace/.env",
"echo BRANCH_NAME=${var.branch_name} >> /root/subspace/.env",

# create docker compose file
"bash /root/subspace/create_compose_file.sh ${var.bootstrap-node-config.reserved-only} ${length(local.domain_node_ip_v4)} ${count.index} ${length(local.bootstrap_nodes_ip_v4)} ${var.domain-node-config.enable-domains} ${var.domain-node-config.domain-id[0]}",

# start subspace node
"cp -f /root/subspace/.env /root/subspace/subspace/.env",
"sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d",
]
}
}

resource "null_resource" "inject-domain-keystore" {
# for now we have one executor running. Should change here when multiple executors are expected.
count = length(local.domain_node_ip_v4) > 0 ? 1 : 0
depends_on = [null_resource.start-domain-nodes]
# trigger on node deployment version change
triggers = {
deployment_version = var.domain-node-config.deployment-version
}

connection {
host = local.domain_node_ip_v4[0]
user = var.ssh_user
type = "ssh"
agent = true
private_key = file("${var.private_key_path}")
timeout = "300s"
}

provisioner "remote-exec" {
inline = [
"sudo docker cp /root/subspace/keystore/. subspace-archival-node-1:/var/subspace/keystore/"
var.branch_name != "main" ? join(" && ", [
"cp -f /root/subspace/.env /root/subspace/subspace/.env",
"sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d"
]) : "sudo docker compose -f /root/subspace/docker-compose.yml up -d"
]
}
}
Loading

0 comments on commit 6bdc29a

Please sign in to comment.