Merge pull request #96 from jodhi/master

Add nodepools feature

agents.tf

@@ -1,9 +1,9 @@
 module "agents" {
   source = "./modules/host"
 
-  count = var.agents_num
+  for_each = local.agent_nodepools
-  name  = "k3s-agent-${count.index}"
 
+  name                   = each.key
   ssh_keys               = [hcloud_ssh_key.k3s.id]
   public_key             = var.public_key
   private_key            = var.private_key
@@ -11,41 +11,44 @@ module "agents" {
   firewall_ids           = [hcloud_firewall.k3s.id]
   placement_group_id     = hcloud_placement_group.k3s.id
   location               = var.location
-  network_id             = hcloud_network.k3s.id
+  server_type            = each.value.server_type
-  ip                     = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index)
+  subnet_id              = hcloud_network_subnet.subnet[each.value.subnet].id
-  server_type            = var.agent_server_type
+  private_ip             = cidrhost(var.network_subnets[each.value.subnet], each.value.index + 1)
-
   labels = {
     "provisioner" = "terraform",
     "engine"      = "k3s"
   }
 
   hcloud_token = var.hcloud_token
+
+  depends_on = [
+    hcloud_network_subnet.subnet
+  ]
 }
 
 resource "null_resource" "agents" {
-  count = var.agents_num
+  for_each = local.agent_nodepools
 
   triggers = {
-    agent_id = module.agents[count.index].id
+    agent_id = module.agents[each.key].id
   }
 
   connection {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = module.agents[count.index].ipv4_address
+    host           = module.agents[each.key].ipv4_address
   }
 
   # Generating k3s agent config file
   provisioner "file" {
     content = yamlencode({
-      node-name     = module.agents[count.index].name
+      node-name     = module.agents[each.key].name
       server        = "https://${local.first_control_plane_network_ip}:6443"
       token         = random_password.k3s_token.result
       kubelet-arg   = "cloud-provider=external"
       flannel-iface = "eth1"
-      node-ip       = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index)
+      node-ip       = module.agents[each.key].ipv4_address
       node-label    = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
     })
     destination = "/tmp/config.yaml"
@@ -74,6 +77,6 @@ resource "null_resource" "agents" {
 
   depends_on = [
     null_resource.first_control_plane,
-    hcloud_network_subnet.k3s
+    hcloud_network_subnet.subnet
   ]
 }

init.tf

@@ -58,7 +58,7 @@ resource "null_resource" "first_control_plane" {
   }
 
   depends_on = [
-    hcloud_network_subnet.k3s
+    hcloud_network_subnet.subnet["control_plane"]
   ]
 }
 

locals.tf

@@ -30,4 +30,15 @@ locals {
   install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"])
 
   install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"])
+
+  agent_nodepools = merge([
+    for nodepool_name, nodepool_obj in var.agent_nodepools : {
+      for index in range(nodepool_obj.count) :
+      format("%s-%s", nodepool_name, index) => {
+        server_type : nodepool_obj.server_type,
+        subnet : lookup(nodepool_obj, "subnet", "default"),
+        index : index
+      }
+    }
+  ]...)
 }

main.tf

@@ -10,14 +10,15 @@ resource "hcloud_ssh_key" "k3s" {
 
 resource "hcloud_network" "k3s" {
   name     = "k3s"
-  ip_range = "10.0.0.0/8"
+  ip_range = var.network_ip_range
 }
 
-resource "hcloud_network_subnet" "k3s" {
+resource "hcloud_network_subnet" "subnet" {
+  for_each     = var.network_subnets
   network_id   = hcloud_network.k3s.id
   type         = "cloud"
   network_zone = var.network_region
-  ip_range     = "10.0.0.0/16"
+  ip_range     = each.value
 }
 
 resource "hcloud_firewall" "k3s" {
@@ -29,8 +30,8 @@ resource "hcloud_firewall" "k3s" {
     protocol  = "tcp"
     port      = "any"
     source_ips = [
+      var.network_ip_range,
       "127.0.0.1/32",
-      "10.0.0.0/8",
       "169.254.169.254/32",
       "213.239.246.1/32"
     ]
@@ -40,8 +41,8 @@ resource "hcloud_firewall" "k3s" {
     protocol  = "udp"
     port      = "any"
     source_ips = [
+      var.network_ip_range,
       "127.0.0.1/32",
-      "10.0.0.0/8",
       "169.254.169.254/32",
       "213.239.246.1/32"
     ]
@@ -50,8 +51,8 @@ resource "hcloud_firewall" "k3s" {
     direction = "in"
     protocol  = "icmp"
     source_ips = [
+      var.network_ip_range,
       "127.0.0.1/32",
-      "10.0.0.0/8",
       "169.254.169.254/32",
       "213.239.246.1/32"
     ]
@@ -177,7 +178,7 @@ resource "null_resource" "destroy_traefik_loadbalancer" {
   depends_on = [
     local_file.kubeconfig,
     null_resource.control_planes[0],
-    hcloud_network_subnet.k3s,
+    hcloud_network_subnet.subnet,
     hcloud_network.k3s,
     hcloud_firewall.k3s,
     hcloud_placement_group.k3s,

modules/host/main.tf

@@ -12,11 +12,6 @@ resource "hcloud_server" "server" {
 
   labels = var.labels
 
-  network {
-    network_id = var.network_id
-    ip         = var.ip
-  }
-
   connection {
     user           = "root"
     private_key    = local.ssh_private_key
@@ -67,3 +62,9 @@ resource "hcloud_server" "server" {
     ]
   }
 }
+
+resource "hcloud_server_network" "server" {
+  ip        = var.private_ip
+  server_id = hcloud_server.server.id
+  subnet_id = var.subnet_id
+}

modules/host/out.tf

@@ -3,7 +3,7 @@ output "ipv4_address" {
 }
 
 output "private_ipv4_address" {
-  value = var.ip
+  value = hcloud_server_network.server.ip
 }
 
 output "name" {

modules/host/variables.tf

@@ -54,15 +54,14 @@ variable "location" {
   type        = string
 }
 
-variable "network_id" {
+variable "subnet_id" {
-  description = "The network or subnet id"
+  description = "The subnet id"
-  type        = number
+  type        = string
 }
 
-variable "ip" {
+variable "private_ip" {
-  description = "The IP"
+  description = "Private IP for the server"
   type        = string
-  nullable    = true
 }
 
 variable "server_type" {

output.tf

@@ -4,7 +4,9 @@ output "controlplanes_public_ip" {
 }
 
 output "agents_public_ip" {
-  value       = module.agents.*.ipv4_address
+  value = [
+    for obj in module.agents : obj.ipv4_address
+  ]
   description = "The public IP addresses of the agent server."
 }
 

servers.tf

@@ -11,9 +11,9 @@ module "control_planes" {
   firewall_ids           = [hcloud_firewall.k3s.id]
   placement_group_id     = hcloud_placement_group.k3s.id
   location               = var.location
-  network_id             = hcloud_network.k3s.id
-  ip                     = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index)
   server_type            = var.control_plane_server_type
+  subnet_id              = hcloud_network_subnet.subnet["control_plane"].id
+  private_ip             = cidrhost(var.network_subnets["control_plane"], count.index + 1)
 
   labels = {
     "provisioner" = "terraform",
@@ -21,6 +21,10 @@ module "control_planes" {
   }
 
   hcloud_token = var.hcloud_token
+
+  depends_on = [
+    hcloud_network_subnet.subnet
+  ]
 }
 
 resource "null_resource" "control_planes" {
@@ -79,6 +83,6 @@ resource "null_resource" "control_planes" {
 
   depends_on = [
     null_resource.first_control_plane,
-    hcloud_network_subnet.k3s
+    hcloud_network_subnet.subnet
   ]
 }

terraform.tfvars.example

@@ -9,15 +9,32 @@ private_key  = "/home/username/.ssh/id_ed25519"
 # For Hetzner server types see https://www.hetzner.com/cloud
 location         = "fsn1"       # change to `ash` for us-east Ashburn, Virginia location
 network_region   = "eu-central" # change to `us-east` if location is ash
-agent_server_type         = "cpx21"
+network_ip_range = "10.0.0.0/8"
+network_subnets = {
+  control_plane = "10.1.0.0/16"
+  subnet1       = "10.2.0.0/16"
+  subnet2       = "10.3.0.0/16"
+}
+
 control_plane_server_type = "cpx11"
 lb_server_type            = "lb11"
 
 # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe).
 servers_num = 3
 
-# For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades.
+
-agents_num                = 2
+agent_nodepools = {
+  big = {
+    server_type = "cpx31",
+    count       = 1,
+    subnet      = "subnet1",
+  }
+  small = {
+    server_type = "cpx21",
+    count       = 2,
+    subnet      = "subnet2",
+  }
+}
 
 # If you want to use a specific Hetzner CCM and CSI version, set them below, otherwise leave as is for the latest versions
 # hetzner_ccm_version = ""

variables.tf

@@ -30,15 +30,21 @@ variable "network_region" {
   type        = string
 }
 
+variable "network_ip_range" {
+  description = "Default IP range for network"
+  type        = string
+}
+
+variable "network_subnets" {
+  description = "Subnets definition for default network"
+  type        = map(string)
+}
+
 variable "control_plane_server_type" {
   description = "Default control plane server type"
   type        = string
 }
 
-variable "agent_server_type" {
-  description = "Default agent server type"
-  type        = string
-}
 
 variable "lb_server_type" {
   description = "Default load balancer server type"
@@ -56,9 +62,9 @@ variable "servers_num" {
   type        = number
 }
 
-variable "agents_num" {
+variable "agent_nodepools" {
   description = "Number of agent nodes."
-  type        = number
+  type        = map(any)
 }
 
 variable "hetzner_ccm_version" {