tweaked readme

agents.tf

@@ -1,9 +1,9 @@
 module "agents" {
   source = "./modules/host"
 
-  count = var.agents_num
+  for_each = local.agent_nodepools
-  name  = "k3s-agent-${count.index}"
 
+  name                   = each.key
   ssh_keys               = [hcloud_ssh_key.k3s.id]
   public_key             = var.public_key
   private_key            = var.private_key
@@ -11,41 +11,44 @@ module "agents" {
   firewall_ids           = [hcloud_firewall.k3s.id]
   placement_group_id     = hcloud_placement_group.k3s.id
   location               = var.location
-  network_id             = hcloud_network.k3s.id
+  server_type            = each.value.server_type
-  ip                     = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index)
+  ipv4_subnet_id         = hcloud_network_subnet.subnet[each.value.subnet].id
-  server_type            = var.agent_server_type
+  private_ipv4           = cidrhost(var.network_ipv4_subnets[each.value.subnet], each.value.index + 1)
-
   labels = {
     "provisioner" = "terraform",
     "engine"      = "k3s"
   }
 
   hcloud_token = var.hcloud_token
+
+  depends_on = [
+    hcloud_network_subnet.subnet
+  ]
 }
 
 resource "null_resource" "agents" {
-  count = var.agents_num
+  for_each = local.agent_nodepools
 
   triggers = {
-    agent_id = module.agents[count.index].id
+    agent_id = module.agents[each.key].id
   }
 
   connection {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = module.agents[count.index].ipv4_address
+    host           = module.agents[each.key].ipv4_address
   }
 
   # Generating k3s agent config file
   provisioner "file" {
     content = yamlencode({
-      node-name     = module.agents[count.index].name
+      node-name     = module.agents[each.key].name
-      server        = "https://${local.first_control_plane_network_ip}:6443"
+      server        = "https://${local.first_control_plane_network_ipv4}:6443"
       token         = random_password.k3s_token.result
       kubelet-arg   = "cloud-provider=external"
       flannel-iface = "eth1"
-      node-ip       = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index)
+      node-ip       = module.agents[each.key].ipv4_address
       node-label    = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
     })
     destination = "/tmp/config.yaml"
@@ -74,6 +77,6 @@ resource "null_resource" "agents" {
 
   depends_on = [
     null_resource.first_control_plane,
-    hcloud_network_subnet.k3s
+    hcloud_network_subnet.subnet
   ]
 }

control_planes.tf

@@ -1,7 +1,7 @@
 module "control_planes" {
   source = "./modules/host"
 
-  count = var.servers_num
+  count = var.control_plane_count
   name  = "k3s-control-plane-${count.index}"
 
   ssh_keys               = [hcloud_ssh_key.k3s.id]
@@ -11,9 +11,9 @@ module "control_planes" {
   firewall_ids           = [hcloud_firewall.k3s.id]
   placement_group_id     = hcloud_placement_group.k3s.id
   location               = var.location
-  network_id             = hcloud_network.k3s.id
-  ip                     = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index)
   server_type            = var.control_plane_server_type
+  ipv4_subnet_id         = hcloud_network_subnet.subnet["control_plane"].id
+  private_ipv4           = cidrhost(var.network_ipv4_subnets["control_plane"], count.index + 1)
 
   labels = {
     "provisioner" = "terraform",
@@ -21,10 +21,14 @@ module "control_planes" {
   }
 
   hcloud_token = var.hcloud_token
+
+  depends_on = [
+    hcloud_network_subnet.subnet
+  ]
 }
 
 resource "null_resource" "control_planes" {
-  count = var.servers_num
+  count = var.control_plane_count
 
   triggers = {
     control_plane_id = module.control_planes[count.index].id
@@ -79,6 +83,6 @@ resource "null_resource" "control_planes" {
 
   depends_on = [
     null_resource.first_control_plane,
-    hcloud_network_subnet.k3s
+    hcloud_network_subnet.subnet
   ]
 }

init.tf

@@ -58,7 +58,7 @@ resource "null_resource" "first_control_plane" {
   }
 
   depends_on = [
-    hcloud_network_subnet.k3s
+    hcloud_network_subnet.subnet["control_plane"]
   ]
 }
 
@@ -96,11 +96,11 @@ resource "null_resource" "kustomization" {
     content = templatefile(
       "${path.module}/templates/traefik_config.yaml.tpl",
       {
-        lb_disable_ipv6    = var.lb_disable_ipv6
+        load_balancer_disable_ipv6 = var.load_balancer_disable_ipv6
-        lb_server_type     = var.lb_server_type
+        load_balancer_type         = var.load_balancer_type
-        location           = var.location
+        location                   = var.location
-        traefik_acme_tls   = var.traefik_acme_tls
+        traefik_acme_tls           = var.traefik_acme_tls
-        traefik_acme_email = var.traefik_acme_email
+        traefik_acme_email         = var.traefik_acme_email
     })
     destination = "/tmp/post_install/traefik.yaml"
   }

locals.tf

@@ -1,5 +1,5 @@
 locals {
-  first_control_plane_network_ip = module.control_planes[0].private_ipv4_address
+  first_control_plane_network_ipv4 = module.control_planes[0].private_ipv4_address
 
   ssh_public_key = trimspace(file(var.public_key))
   # ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
@@ -30,4 +30,15 @@ locals {
   install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"])
 
   install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"])
+
+  agent_nodepools = merge([
+    for nodepool_name, nodepool_obj in var.agent_nodepools : {
+      for index in range(nodepool_obj.count) :
+      format("%s-%s", nodepool_name, index) => {
+        server_type : nodepool_obj.server_type,
+        subnet : lookup(nodepool_obj, "subnet", "default"),
+        index : index
+      }
+    }
+  ]...)
 }

main.tf

@@ -10,14 +10,15 @@ resource "hcloud_ssh_key" "k3s" {
 
 resource "hcloud_network" "k3s" {
   name     = "k3s"
-  ip_range = "10.0.0.0/8"
+  ip_range = var.network_ipv4_range
 }
 
-resource "hcloud_network_subnet" "k3s" {
+resource "hcloud_network_subnet" "subnet" {
+  for_each     = var.network_ipv4_subnets
   network_id   = hcloud_network.k3s.id
   type         = "cloud"
   network_zone = var.network_region
-  ip_range     = "10.0.0.0/16"
+  ip_range     = each.value
 }
 
 resource "hcloud_firewall" "k3s" {
@@ -29,8 +30,8 @@ resource "hcloud_firewall" "k3s" {
     protocol  = "tcp"
     port      = "any"
     source_ips = [
+      var.network_ipv4_range,
       "127.0.0.1/32",
-      "10.0.0.0/8",
       "169.254.169.254/32",
       "213.239.246.1/32"
     ]
@@ -40,8 +41,8 @@ resource "hcloud_firewall" "k3s" {
     protocol  = "udp"
     port      = "any"
     source_ips = [
+      var.network_ipv4_range,
       "127.0.0.1/32",
-      "10.0.0.0/8",
       "169.254.169.254/32",
       "213.239.246.1/32"
     ]
@@ -50,8 +51,8 @@ resource "hcloud_firewall" "k3s" {
     direction = "in"
     protocol  = "icmp"
     source_ips = [
+      var.network_ipv4_range,
       "127.0.0.1/32",
-      "10.0.0.0/8",
       "169.254.169.254/32",
       "213.239.246.1/32"
     ]
@@ -177,7 +178,7 @@ resource "null_resource" "destroy_traefik_loadbalancer" {
   depends_on = [
     local_file.kubeconfig,
     null_resource.control_planes[0],
-    hcloud_network_subnet.k3s,
+    hcloud_network_subnet.subnet,
     hcloud_network.k3s,
     hcloud_firewall.k3s,
     hcloud_placement_group.k3s,

modules/host/main.tf

@@ -12,11 +12,6 @@ resource "hcloud_server" "server" {
 
   labels = var.labels
 
-  network {
-    network_id = var.network_id
-    ip         = var.ip
-  }
-
   connection {
     user           = "root"
     private_key    = local.ssh_private_key
@@ -67,3 +62,9 @@ resource "hcloud_server" "server" {
     ]
   }
 }
+
+resource "hcloud_server_network" "server" {
+  ip        = var.private_ipv4
+  server_id = hcloud_server.server.id
+  subnet_id = var.ipv4_subnet_id
+}

modules/host/out.tf

@@ -3,7 +3,7 @@ output "ipv4_address" {
 }
 
 output "private_ipv4_address" {
-  value = var.ip
+  value = hcloud_server_network.server.ip
 }
 
 output "name" {

modules/host/variables.tf

@@ -54,15 +54,14 @@ variable "location" {
   type        = string
 }
 
-variable "network_id" {
+variable "ipv4_subnet_id" {
-  description = "The network or subnet id"
+  description = "The subnet id"
-  type        = number
+  type        = string
 }
 
-variable "ip" {
+variable "private_ipv4" {
-  description = "The IP"
+  description = "Private IP for the server"
   type        = string
-  nullable    = true
 }
 
 variable "server_type" {

output.tf

@@ -1,14 +1,16 @@
-output "controlplanes_public_ip" {
+output "control_planes_public_ipv4" {
   value       = module.control_planes.*.ipv4_address
-  description = "The public IP addresses of the controlplane server."
+  description = "The public IPv4 addresses of the controlplane server."
 }
 
-output "agents_public_ip" {
+output "agents_public_ipv4" {
-  value       = module.agents.*.ipv4_address
+  value = [
-  description = "The public IP addresses of the agent server."
+    for obj in module.agents : obj.ipv4_address
+  ]
+  description = "The public IPv4 addresses of the agent server."
 }
 
-output "load_balancer_public_ip" {
+output "load_balancer_public_ipv4" {
   description = "The public IPv4 address of the Hetzner load balancer"
   value       = data.hcloud_load_balancer.traefik.ipv4
 }

templates/traefik_config.yaml.tpl

@@ -15,9 +15,9 @@ spec:
         # keep hetzner-ccm from exposing our private ingress ip, which in general isn't routeable from the public internet
         "load-balancer.hetzner.cloud/disable-private-ingress": "true"
         # disable ipv6 by default, because external-dns doesn't support AAAA for hcloud yet https://github.com/kubernetes-sigs/external-dns/issues/2044
-        "load-balancer.hetzner.cloud/ipv6-disabled": "${lb_disable_ipv6}"
+        "load-balancer.hetzner.cloud/ipv6-disabled": "${load_balancer_disable_ipv6}"
         "load-balancer.hetzner.cloud/location": "${location}"
-        "load-balancer.hetzner.cloud/type": "${lb_server_type}"
+        "load-balancer.hetzner.cloud/type": "${load_balancer_type}"
         "load-balancer.hetzner.cloud/uses-proxyprotocol": "true"
     additionalArguments:
       - "--entryPoints.web.proxyProtocol.trustedIPs=127.0.0.1/32,10.0.0.0/8"

terraform.tfvars.example

@@ -7,17 +7,34 @@ private_key  = "/home/username/.ssh/id_ed25519"
 # These can be customized, or left with the default values
 # For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
 # For Hetzner server types see https://www.hetzner.com/cloud
-location                  = "fsn1" # change to `ash` for us-east Ashburn, Virginia location
+location         = "fsn1"       # change to `ash` for us-east Ashburn, Virginia location
-network_region            = "eu-central" # change to `us-east` if location is ash
+network_region   = "eu-central" # change to `us-east` if location is ash
-agent_server_type         = "cpx21"
+network_ipv4_range = "10.0.0.0/8"
+network_ipv4_subnets = {
+  control_plane = "10.1.0.0/16"
+  subnet1       = "10.2.0.0/16"
+  subnet2       = "10.3.0.0/16"
+}
+
 control_plane_server_type = "cpx11"
-lb_server_type            = "lb11"
+load_balancer_type            = "lb11"
 
 # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe).
-servers_num               = 3
+control_plane_count = 3
 
-# For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades.
+
-agents_num                = 2
+agent_nodepools = {
+  big = {
+    server_type = "cpx31",
+    count       = 1,
+    subnet      = "subnet1",
+  }
+  small = {
+    server_type = "cpx21",
+    count       = 2,
+    subnet      = "subnet2",
+  }
+}
 
 # If you want to use a specific Hetzner CCM and CSI version, set them below, otherwise leave as is for the latest versions
 # hetzner_ccm_version = ""

variables.tf

@@ -30,35 +30,40 @@ variable "network_region" {
   type        = string
 }
 
+variable "network_ipv4_range" {
+  description = "Default IPv4 range for network"
+  type        = string
+}
+
+variable "network_ipv4_subnets" {
+  description = "Subnets definition for default network"
+  type        = map(string)
+}
+
 variable "control_plane_server_type" {
   description = "Default control plane server type"
   type        = string
 }
 
-variable "agent_server_type" {
+variable "control_plane_count" {
-  description = "Default agent server type"
+  description = "Number of control plane nodes."
-  type        = string
+  type        = number
 }
 
-variable "lb_server_type" {
+variable "load_balancer_type" {
   description = "Default load balancer server type"
   type        = string
 }
 
-variable "lb_disable_ipv6" {
+variable "load_balancer_disable_ipv6" {
   description = "Disable ipv6 for the load balancer"
   type        = bool
   default     = false
 }
 
-variable "servers_num" {
+variable "agent_nodepools" {
-  description = "Number of control plane nodes."
-  type        = number
-}
-
-variable "agents_num" {
   description = "Number of agent nodes."
-  type        = number
+  type        = map(any)
 }
 
 variable "hetzner_ccm_version" {