Hetzner module

resources/hetzner/backend.tf

@@ -0,0 +1,9 @@
+terraform {
+  cloud {
+    organization = "subspace-sre"
+
+    workspaces {
+      name = var.workspace_name
+    }
+  }
+}

resources/hetzner/main.tf

@@ -0,0 +1,84 @@
+module "network" {
+  source          = "../../templates/terraform/hetzner"
+  path_to_scripts = "../../templates/scripts"
+  network_name    = var.network_name
+
+  bootstrap-node-config = {
+    deployment-version  = 1
+    instance-count      = var.instance_count["bootstrap"]
+    repo-org            = "subspace"
+    node-tag            = "bootstrap-node"
+    additional-node-ips = var.additional_node_ips["bootstrap"]
+    reserved-only       = true
+    prune               = false
+    genesis-hash        = var.genesis_hash
+    dsn-listen-port     = 30533
+    node-dsn-port       = 30433
+  }
+
+  bootstrap-node-evm-config = {
+    deployment-version  = 1
+    instance-count      = var.instance_count["bootstrap"]
+    repo-org            = "subspace"
+    node-tag            = "bootstrap-node"
+    additional-node-ips = var.additional_node_ips["bootstrap"]
+    reserved-only       = true
+    prune               = false
+    genesis-hash        = var.genesis_hash
+    dsn-listen-port     = 30533
+    node-dsn-port       = 30433
+    operator-port       = 30334
+  }
+
+  node-config = {
+    deployment-version  = 1
+    instance-count      = var.instance_count["node"]
+    repo-org            = "subspace"
+    node-tag            = "subspace-node"
+    additional-node-ips = var.additional_node_ips["node"]
+    reserved-only       = true
+    prune               = false
+    node-dsn-port       = 30433
+  }
+
+  domain-node-config = {
+    deployment-version  = 1
+    instance-count      = var.instance_count["domain"]
+    repo-org            = "subspace"
+    node-tag            = "subspace-node"
+    additional-node-ips = var.additional_node_ips["domain"]
+    domain-prefix       = "domain"
+    reserved-only       = true
+    prune               = false
+    node-dsn-port       = 30433
+    enable-domains      = true
+    domain-id           = var.domain_id
+    domain-labels       = var.domain_labels
+  }
+
+  farmer-node-config = {
+    deployment-version     = 1
+    instance-count         = var.instance_count["farmer"]
+    repo-org               = "subspace"
+    node-tag               = "farmer-node"
+    additional-node-ips    = var.additional_node_ips["farmer"]
+    reserved-only          = true
+    prune                  = false
+    plot-size              = "10G"
+    reward-address         = var.farmer_reward_address
+    force-block-production = true
+    node-dsn-port          = 30433
+
+  }
+
+  tf_token         = var.tf_token
+  private_key_path = var.private_key_path
+  branch_name      = var.branch_name
+  ssh_user         = var.ssh_user
+  genesis_hash     = var.genesis_hash
+}
+
+# External data source to run the shell command and extract the value of the operator bootnode connection parameter
+data "external" "operator_peer_multiaddr" {
+  program = ["bash", "-c", "echo '{\"OPERATOR_MULTI_ADDR\": \"'$(sed -nr 's/^NODE_0_OPERATOR_MULTI_ADDR=(.*)/\\1/p' ./bootstrap_node_evm_keys.txt)'\"}'"]
+}

resources/hetzner/outputs.tf

@@ -0,0 +1,30 @@
+output "bootstrap-node-ipv4-addresses" {
+  value       = module.network.bootstrap-node-ipv4-addresses
+  description = "Bootstrap node IPv4 Addresses"
+}
+
+output "bootstrap-node-evm-ipv4-addresses" {
+  value       = module.network.bootstrap-node-evm-ipv4-addresses
+  description = "Bootstrap node EVM IPv4 Addresses"
+}
+
+output "node-ipv4-addresses" {
+  value       = module.network.node-ipv4-addresses
+  description = "subspace node IPv4 Addresses"
+}
+
+output "domain-node-ipv4-addresses" {
+  value       = module.network.domain-node-ipv4-addresses
+  description = "domain node IPv4 Addresses"
+}
+
+
+output "farmer-nodes-ipv4-addresses" {
+  value       = module.network.farmer-node-ipv4-addresses
+  description = "Farmer node IPv4 Addresses"
+}
+
+# Output the operator_peer_multiaddr value
+output "operator_peer_multiaddr" {
+  value = data.external.operator_peer_multiaddr.result["operator_peer_multiaddr"]
+}

resources/hetzner/variables.tf

@@ -0,0 +1,76 @@
+variable "farmer_reward_address" {
+  description = "Farmer's reward address"
+  type        = string
+}
+
+variable "network_name" {
+  description = "Network name"
+  type        = string
+}
+
+//todo change this to a map
+variable "domain_id" {
+  description = "Domain ID"
+  type        = list(number)
+  default     = [0]
+}
+
+//todo change this to a map
+variable "domain_labels" {
+  description = "Tag of the domain to run"
+  type        = list(string)
+  default     = ["evm"]
+}
+
+variable "instance_count" {
+  type = map(number)
+  default = {
+    bootstrap     = 2
+    node          = 1
+    farmer        = 1
+    domain        = 2
+    evm_bootstrap = 1
+  }
+}
+
+variable "additional_node_ips" {
+  type = map(list(string))
+  default = {
+    bootstrap     = [""]
+    node          = [""]
+    farmer        = [""]
+    domain        = [""]
+    evm_bootstrap = [""]
+  }
+}
+
+variable "ssh_user" {
+  type    = string
+  default = "root"
+}
+
+variable "private_key_path" {
+  type    = string
+  default = "~/.ssh/hetzner"
+}
+
+variable "tf_token" {
+  type      = string
+  sensitive = true
+}
+
+variable "branch_name" {
+  description = "name of testing branch"
+  type        = string
+  default     = "main"
+}
+
+variable "genesis_hash" {
+  description = "Genesis hash"
+  type        = string
+}
+
+variable "workspace_name" {
+  description = "Name of the workspace"
+  type        = string
+}

templates/terraform/hetzner/bootstrap_node_evm_provisioner.tf

@@ -45,6 +45,19 @@ resource "null_resource" "setup-bootstrap-nodes-evm" {
 
 }
 
+resource "null_resource" "clone_branch" {
+  count = var.branch_name != "main" ? 1 : 0
+
+  provisioner "remote-exec" {
+    inline = [
+      "cd /root/subspace/",
+      "git clone https://github.com/subspace/subspace.git",
+      "cd subspace",
+      "git checkout ${var.branch_name}"
+    ]
+  }
+}
+
 resource "null_resource" "prune-bootstrap-nodes-evm" {
   count      = var.bootstrap-node-evm-config.prune ? length(local.bootstrap_nodes_evm_ip_v4) : 0
   depends_on = [null_resource.setup-bootstrap-nodes-evm]
@@ -135,8 +148,8 @@ resource "null_resource" "start-bootstrap-nodes-evm" {
       "sudo hostnamectl set-hostname ${var.network_name}-bootstrap-node-evm-${count.index}",
 
       # create .env file
-      "echo NODE_ORG=${var.bootstrap-node-evm-config.docker-org} > /root/subspace/.env",
-      "echo NODE_TAG=${var.bootstrap-node-evm-config.docker-tag} >> /root/subspace/.env",
+      "echo NODE_ORG=${var.bootstrap-node-evm-config.repo-org} > /root/subspace/.env",
+      "echo NODE_TAG=${var.bootstrap-node-evm-config.node-tag} >> /root/subspace/.env",
       "echo NETWORK_NAME=${var.network_name} >> /root/subspace/.env",
       "echo NODE_ID=${count.index} >> /root/subspace/.env",
       "echo NODE_KEY=$(sed -nr 's/NODE_${count.index}_KEY=//p' /root/subspace/node_keys.txt) >> /root/subspace/.env",
@@ -151,12 +164,16 @@ resource "null_resource" "start-bootstrap-nodes-evm" {
       "echo NODE_DSN_PORT=${var.bootstrap-node-evm-config.node-dsn-port} >> /root/subspace/.env",
       "echo OPERATOR_PORT=${var.bootstrap-node-evm-config.operator-port} >> /root/subspace/.env",
       "echo GENESIS_HASH=${var.bootstrap-node-evm-config.genesis-hash} >> /root/subspace/.env",
+      "echo BRANCH_NAME=${var.branch_name} >> /root/subspace/.env",
 
       # create docker compose file
-      "bash /root/subspace/create_compose_file.sh ${var.bootstrap-node-evm-config.reserved-only} ${length(local.bootstrap_nodes_evm_ip_v4)} ${count.index} ${length(local.bootstrap_nodes_ip_v4)} ${var.domain-node-config.enable-domains} ",
+      "bash /root/subspace/create_compose_file.sh ${var.bootstrap-node-evm-config.reserved-only} ${length(local.bootstrap_nodes_evm_ip_v4)} ${count.index} ${length(local.bootstrap_nodes_ip_v4)} ${var.domain-node-config.enable-domains}",
 
       # start subspace node
-      "sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d",
+      var.branch_name != "main" ? join(" && ", [
+        "cp -f /root/subspace/.env /root/subspace/subspace/.env",
+        "sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d"
+      ]) : "sudo docker compose -f /root/subspace/docker-compose.yml up -d"
     ]
   }
 }

templates/terraform/hetzner/bootstrap_node_provisioner.tf

@@ -43,8 +43,11 @@ resource "null_resource" "setup-bootstrap-nodes" {
       "sudo bash /root/subspace/installer.sh",
     ]
   }
+}
+
+resource "null_resource" "clone_branch" {
+  count = var.branch_name != "main" ? 1 : 0
 
-  # clone testing branch
   provisioner "remote-exec" {
     inline = [
       "cd /root/subspace/",
@@ -53,7 +56,6 @@ resource "null_resource" "setup-bootstrap-nodes" {
       "git checkout ${var.branch_name}"
     ]
   }
-
 }
 
 resource "null_resource" "prune-bootstrap-nodes" {
@@ -146,14 +148,16 @@ resource "null_resource" "start-boostrap-nodes" {
       "echo DSN_LISTEN_PORT=${var.bootstrap-node-config.dsn-listen-port} >> /root/subspace/.env",
       "echo NODE_DSN_PORT=${var.bootstrap-node-config.node-dsn-port} >> /root/subspace/.env",
       "echo GENESIS_HASH=${var.bootstrap-node-config.genesis-hash} >> /root/subspace/.env",
+      "echo BRANCH_NAME=${var.branch_name} >> /root/subspace/.env",
 
       # create docker compose file
-      "chmod +x /root/subspace/create_compose_file.sh",
       "bash /root/subspace/create_compose_file.sh ${var.bootstrap-node-config.reserved-only} ${length(local.bootstrap_nodes_ip_v4)} ${count.index}",
 
       # start subspace node
-      "cp -f /root/subspace/.env /root/subspace/subspace/.env",
-      "sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d",
+      var.branch_name != "main" ? join(" && ", [
+        "cp -f /root/subspace/.env /root/subspace/subspace/.env",
+        "sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d"
+      ]) : "sudo docker compose -f /root/subspace/docker-compose.yml up -d"
     ]
   }
 }

templates/terraform/hetzner/domain_node_provisioner.tf

@@ -42,7 +42,11 @@ resource "null_resource" "setup-domain-nodes" {
       "sudo bash /root/subspace/installer.sh",
     ]
   }
-  # clone testing branch
+}
+
+resource "null_resource" "clone_branch" {
+  count = var.branch_name != "main" ? 1 : 0
+
   provisioner "remote-exec" {
     inline = [
       "cd /root/subspace/",
@@ -51,7 +55,6 @@ resource "null_resource" "setup-domain-nodes" {
       "git checkout ${var.branch_name}"
     ]
   }
-
 }
 
 resource "null_resource" "prune-domain-nodes" {
@@ -163,38 +166,16 @@ resource "null_resource" "start-domain-nodes" {
       "echo RELAYER_DOMAIN_ID=$(sed -nr 's/NODE_${count.index}_RELAYER_DOMAIN_ID=//p' /root/subspace/relayer_ids.txt) >> /root/subspace/.env",
       "echo PIECE_CACHE_SIZE=${var.piece_cache_size} >> /root/subspace/.env",
       "echo NODE_DSN_PORT=${var.domain-node-config.node-dsn-port} >> /root/subspace/.env",
+      "echo BRANCH_NAME=${var.branch_name} >> /root/subspace/.env",
 
       # create docker compose file
       "bash /root/subspace/create_compose_file.sh ${var.bootstrap-node-config.reserved-only} ${length(local.domain_node_ip_v4)} ${count.index} ${length(local.bootstrap_nodes_ip_v4)} ${var.domain-node-config.enable-domains} ${var.domain-node-config.domain-id[0]}",
 
       # start subspace node
-      "cp -f /root/subspace/.env /root/subspace/subspace/.env",
-      "sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d",
-    ]
-  }
-}
-
-resource "null_resource" "inject-domain-keystore" {
-  # for now we have one executor running. Should change here when multiple executors are expected.
-  count      = length(local.domain_node_ip_v4) > 0 ? 1 : 0
-  depends_on = [null_resource.start-domain-nodes]
-  # trigger on node deployment version change
-  triggers = {
-    deployment_version = var.domain-node-config.deployment-version
-  }
-
-  connection {
-    host        = local.domain_node_ip_v4[0]
-    user        = var.ssh_user
-    type        = "ssh"
-    agent       = true
-    private_key = file("${var.private_key_path}")
-    timeout     = "300s"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "sudo docker cp /root/subspace/keystore/.  subspace-archival-node-1:/var/subspace/keystore/"
+      var.branch_name != "main" ? join(" && ", [
+        "cp -f /root/subspace/.env /root/subspace/subspace/.env",
+        "sudo docker compose -f /root/subspace/subspace/docker-compose.yml up -d"
+      ]) : "sudo docker compose -f /root/subspace/docker-compose.yml up -d"
     ]
   }
 }