From 5e4d82fd95d2e15cc361e2cc2195cb99bc07d13c Mon Sep 17 00:00:00 2001
From: Henk van Maanen <36051232+HenkVanMaanen@users.noreply.github.com>
Date: Wed, 6 Apr 2022 20:38:24 +0200
Subject: [PATCH 01/10] wip

---
 control_planes.tf | 27 ++++++++++++++-------------
 init.tf           | 12 ++++++------
 kubeconfig.tf     |  4 ++--
 locals.tf         | 34 +++++++++++++++++++++++++---------
 output.tf         |  4 +++-
 variables.tf      | 20 ++++++++------------
 6 files changed, 58 insertions(+), 43 deletions(-)

diff --git a/control_planes.tf b/control_planes.tf
index d39047f..c24590a 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -1,7 +1,8 @@
 module "control_planes" {
   source = "./modules/host"
 
-  count                  = var.control_plane_count
+  for_each = local.control_plane_nodepools
+
   name                   = "${var.use_cluster_name_in_node_name ? "${var.cluster_name}-" : ""}control-plane"
   ssh_keys               = [hcloud_ssh_key.k3s.id]
   public_key             = var.public_key
@@ -9,13 +10,13 @@ module "control_planes" {
   additional_public_keys = var.additional_public_keys
   firewall_ids           = [hcloud_firewall.k3s.id]
   placement_group_id     = hcloud_placement_group.k3s.id
-  location               = var.location
-  server_type            = var.control_plane_server_type
+  location               = each.value.location
+  server_type            = each.value.server_type
   ipv4_subnet_id         = hcloud_network_subnet.subnet[1].id
 
   # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely
   # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough.
-  private_ipv4 = cidrhost(local.network_ipv4_subnets[1], count.index + 101)
+  private_ipv4 = cidrhost(local.network_ipv4_subnets[1], length([for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name]) + each.value.index + 101)
 
   labels = {
     "provisioner" = "terraform",
@@ -28,33 +29,33 @@ module "control_planes" {
 }
 
 resource "null_resource" "control_planes" {
-  count = var.control_plane_count
+  for_each = local.control_plane_nodepools
 
   triggers = {
-    control_plane_id = module.control_planes[count.index].id
+    control_plane_id = module.control_planes[each.key].id
   }
 
   connection {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = module.control_planes[count.index].ipv4_address
+    host           = module.control_planes[each.key].ipv4_address
   }
 
   # Generating k3s server config file
   provisioner "file" {
     content = yamlencode({
-      node-name                = module.control_planes[count.index].name
-      server                   = "https://${element(module.control_planes.*.private_ipv4_address, count.index > 0 ? 0 : 1)}:6443"
+      node-name                = module.control_planes[each.key].name
+      server                   = "https://${local.first_control_plane.private_ipv4_address}:6443"
       token                    = random_password.k3s_token.result
       disable-cloud-controller = true
       disable                  = local.disable_extras
       flannel-iface            = "eth1"
       kubelet-arg              = "cloud-provider=external"
-      node-ip                  = module.control_planes[count.index].private_ipv4_address
-      advertise-address        = module.control_planes[count.index].private_ipv4_address
-      node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
-      node-label               = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
+      node-ip                  = module.control_planes[each.key].private_ipv4_address
+      advertise-address        = module.control_planes[each.key].private_ipv4_address
+      node-label               = each.value.labels
+      node-taint               = each.value.taints
     })
     destination = "/tmp/config.yaml"
   }
diff --git a/init.tf b/init.tf
index 1e8eb7e..884fbe6 100644
--- a/init.tf
+++ b/init.tf
@@ -3,21 +3,21 @@ resource "null_resource" "first_control_plane" {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = module.control_planes[0].ipv4_address
+    host           = local.first_control_plane.ipv4_address
   }
 
   # Generating k3s master config file
   provisioner "file" {
     content = yamlencode({
-      node-name                = module.control_planes[0].name
+      node-name                = local.first_control_plane.name
       token                    = random_password.k3s_token.result
       cluster-init             = true
       disable-cloud-controller = true
       disable                  = local.disable_extras
       flannel-iface            = "eth1"
       kubelet-arg              = "cloud-provider=external"
-      node-ip                  = module.control_planes[0].private_ipv4_address
-      advertise-address        = module.control_planes[0].private_ipv4_address
+      node-ip                  = local.first_control_plane.private_ipv4_address
+      advertise-address        = local.first_control_plane.private_ipv4_address
       node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
       node-label               = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
     })
@@ -66,7 +66,7 @@ resource "null_resource" "kustomization" {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = module.control_planes[0].ipv4_address
+    host           = local.first_control_plane.ipv4_address
   }
 
   # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured.
@@ -97,7 +97,7 @@ resource "null_resource" "kustomization" {
         name                       = "${var.cluster_name}-traefik"
         load_balancer_disable_ipv6 = var.load_balancer_disable_ipv6
         load_balancer_type         = var.load_balancer_type
-        location                   = var.location
+        location                   = var.load_balancer_location
         traefik_acme_tls           = var.traefik_acme_tls
         traefik_acme_email         = var.traefik_acme_email
         traefik_additional_options = var.traefik_additional_options
diff --git a/kubeconfig.tf b/kubeconfig.tf
index 1c5cee5..b4969b5 100644
--- a/kubeconfig.tf
+++ b/kubeconfig.tf
@@ -1,7 +1,7 @@
 
 data "remote_file" "kubeconfig" {
   conn {
-    host        = module.control_planes[0].ipv4_address
+    host        = local.first_control_plane.ipv4_address
     port        = 22
     user        = "root"
     private_key = local.ssh_private_key
@@ -13,7 +13,7 @@ data "remote_file" "kubeconfig" {
 }
 
 locals {
-  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", module.control_planes[0].ipv4_address)
+  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", local.first_control_plane.ipv4_address)
   kubeconfig_parsed   = yamldecode(local.kubeconfig_external)
   kubeconfig_data = {
     host                   = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"]
diff --git a/locals.tf b/locals.tf
index 439b1b1..101e9a6 100644
--- a/locals.tf
+++ b/locals.tf
@@ -1,6 +1,6 @@
 locals {
   # if we are in a single cluster config, we use the default klipper lb instead of Hetzner LB
-  is_single_node_cluster = var.control_plane_count + sum(concat([for v in var.agent_nodepools : v.count], [0])) == 1
+  is_single_node_cluster = sum(concat([for v in var.control_plane_nodepools : v.count], [0])) + sum(concat([for v in var.agent_nodepools : v.count], [0])) == 1
   ssh_public_key         = trimspace(file(var.public_key))
   # ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
   ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key))
@@ -169,16 +169,30 @@ locals {
   install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"], local.apply_k3s_selinux)
   install_k3s_agent  = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"], local.apply_k3s_selinux)
 
-  agent_nodepools = merge([
-    for nodepool_obj in var.agent_nodepools : {
-      for index in range(nodepool_obj.count) :
-      format("%s-%s", nodepool_obj.name, index) => {
+  control_plane_nodepools = merge([
+    for pool_index, nodepool_obj in var.control_plane_nodepools : {
+      for node_index in range(nodepool_obj.count) :
+      format("%s-%s-%s", pool_index, node_index, nodepool_obj.name) => {
         nodepool_name : nodepool_obj.name,
         server_type : nodepool_obj.server_type,
         location : nodepool_obj.location,
-        labels : concat(local.default_labels, nodepool_obj.labels),
+        labels : concat(local.default_control_plane_labels, nodepool_obj.labels),
         taints : nodepool_obj.taints,
-        index : index
+        index : node_index
+      }
+    }
+  ]...)
+
+  agent_nodepools = merge([
+    for pool_index, nodepool_obj in var.agent_nodepools : {
+      for node_index in range(nodepool_obj.count) :
+      format("%s-%s-%s", pool_index, node_index, nodepool_obj.name) => {
+        nodepool_name : nodepool_obj.name,
+        server_type : nodepool_obj.server_type,
+        location : nodepool_obj.location,
+        labels : concat(local.default_agent_labels, nodepool_obj.labels),
+        taints : nodepool_obj.taints,
+        index : node_index
       }
     }
   ]...)
@@ -193,7 +207,9 @@ locals {
   # disable k3s extras
   disable_extras = concat(["local-storage"], local.is_single_node_cluster ? [] : ["servicelb"], var.traefik_enabled ? [] : ["traefik"], var.metrics_server_enabled ? [] : ["metrics-server"])
 
-
   # Default k3s node labels
-  default_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
+  default_agent_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
+  default_control_plane_labels = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
+
+  first_control_plane = module.control_planes[keys(module.control_planes)[0]]
 }
diff --git a/output.tf b/output.tf
index 89514ef..798f81f 100644
--- a/output.tf
+++ b/output.tf
@@ -4,7 +4,9 @@ output "cluster_name" {
 }
 
 output "control_planes_public_ipv4" {
-  value       = module.control_planes.*.ipv4_address
+  value = [
+    for obj in module.control_planes : obj.ipv4_address
+  ]
   description = "The public IPv4 addresses of the controlplane server."
 }
 
diff --git a/variables.tf b/variables.tf
index 2b5a694..df7aaf8 100644
--- a/variables.tf
+++ b/variables.tf
@@ -20,26 +20,16 @@ variable "additional_public_keys" {
   default     = []
 }
 
-variable "location" {
-  description = "Default server location"
-  type        = string
-}
-
 variable "network_region" {
   description = "Default region for network"
   type        = string
 }
 
-variable "control_plane_server_type" {
-  description = "Default control plane server type"
+variable "load_balancer_location" {
+  description = "Default load balancer location"
   type        = string
 }
 
-variable "control_plane_count" {
-  description = "Number of control plane nodes."
-  type        = number
-}
-
 variable "load_balancer_type" {
   description = "Default load balancer server type"
   type        = string
@@ -51,6 +41,12 @@ variable "load_balancer_disable_ipv6" {
   default     = false
 }
 
+variable "control_plane_nodepools" {
+  description = "Number of control plane nodes."
+  type        = list(any)
+  default     = []
+}
+
 variable "agent_nodepools" {
   description = "Number of agent nodes."
   type        = list(any)

From 65297f7ded7cce3a05e2f31b091c4f5a80938544 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Sat, 9 Apr 2022 08:40:36 +0200
Subject: [PATCH 02/10] initial success for one control plane nodepool

---
 agents.tf                |  2 +-
 locals.tf                |  4 ++--
 terraform.tfvars.example | 38 +++++++++++++++++++++++++++-----------
 3 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/agents.tf b/agents.tf
index 003fc82..82aa17c 100644
--- a/agents.tf
+++ b/agents.tf
@@ -46,7 +46,7 @@ resource "null_resource" "agents" {
   provisioner "file" {
     content = yamlencode({
       node-name     = module.agents[each.key].name
-      server        = "https://${module.control_planes[0].private_ipv4_address}:6443"
+      server        = "https://${local.first_control_plane.private_ipv4_address}:6443"
       token         = random_password.k3s_token.result
       kubelet-arg   = "cloud-provider=external"
       flannel-iface = "eth1"
diff --git a/locals.tf b/locals.tf
index 101e9a6..84cc38b 100644
--- a/locals.tf
+++ b/locals.tf
@@ -202,13 +202,13 @@ locals {
 
   # The first two subnets are respectively the default subnet 10.0.0.0/16 use for potientially anything and 10.1.0.0/16 used for control plane nodes.
   # the rest of the subnets are for agent nodes in each nodepools.
-  network_ipv4_subnets = [for index in range(length(var.agent_nodepools) + 2) : cidrsubnet(local.network_ipv4_cidr, 8, index)]
+  network_ipv4_subnets = [for index in range(length(var.control_plane_nodepools) + length(var.agent_nodepools) + 1) : cidrsubnet(local.network_ipv4_cidr, 8, index)]
 
   # disable k3s extras
   disable_extras = concat(["local-storage"], local.is_single_node_cluster ? [] : ["servicelb"], var.traefik_enabled ? [] : ["traefik"], var.metrics_server_enabled ? [] : ["metrics-server"])
 
   # Default k3s node labels
-  default_agent_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
+  default_agent_labels         = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
   default_control_plane_labels = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
 
   first_control_plane = module.control_planes[keys(module.control_planes)[0]]
diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 8641bd3..99199f0 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -14,23 +14,38 @@ private_key  = "/home/username/.ssh/id_ed25519"
 
 # These can be customized, or left with the default values
 # For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
-location         = "fsn1"       # change to `ash` for us-east Ashburn, Virginia location
 network_region   = "eu-central" # change to `us-east` if location is ash
 
 # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe).
 # As per rancher docs, it must be always an odd number, never even! See https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/
 # For instance, 1 is ok (non-HA), 2 not ok, 3 is ok (becomes HA).
-control_plane_count = 3
+# Of course, you can choose any number of node pools you want, with the location you want. The only contraint on the location is that you need to stay in the same network region, basically Europe or US, see above.
+# For the server type, # The type of control plane nodes, the minimum instance supported is cpx11 (just a few cents more than cx11), see https://www.hetzner.com/cloud.
+control_plane_nodepools = [
+  {
+    name        = "control-plane-fsn1",
+    server_type = "cpx11",
+    location    = "fsn1",
+    labels      = [],
+    taints      = [],
+    count       = 2
+  },
+  {
+    name        = "control-plane-nbg1",
+    server_type = "cpx11",
+    location    = "nbg1",
+    labels      = [],
+    taints      = [],
+    count       = 1
+  }
+]
 
-# The type of control plane nodes, see https://www.hetzner.com/cloud, the minimum instance supported is cpx11 (just a few cents more than cx11)
-control_plane_server_type = "cpx11"
 
 # As for the agent nodepools, below is just an example, if you do not want nodepools, just use one,
 # and change the name to what you want, it need not be "agent-big" or "agent-small", also give them the subnet prefer.
 # For single node clusters set this equal to [] or just set the counts to 0.
 # IMPORTANT: Once the cluster is created, you can change the count, and even set it to 0, but do not remove a nodepool from the list. 
-# You can add others at the end of the list if you want.
-# For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
+# You can add others at the end of the list if you want. For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
 # For Hetzner server types see https://www.hetzner.com/cloud
 agent_nodepools = [
   {
@@ -44,7 +59,7 @@ agent_nodepools = [
   {
     name        = "agent-large",
     server_type = "cpx21",
-    location    = "fsn1",
+    location    = "nbg1",
     labels      = [],
     taints      = [],
     count       = 1
@@ -53,18 +68,19 @@ agent_nodepools = [
     name        = "storage",
     server_type = "cpx21",
     location    = "fsn1",
-    labels      = [
+    labels = [
       "node.kubernetes.io/server-usage=storage"
     ],
-    taints      = [
+    taints = [
       "server-usage=storage:NoSchedule"
     ],
-    count       = 1
+    count = 1
   }
 ]
 
 # That will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer
-load_balancer_type        = "lb11"
+load_balancer_type     = "lb11"
+load_balancer_location = "fsn1"
 
 ### The following values are fully optional
 

From 41177769942921336eaac9a2fae2711af347ac45 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Sat, 9 Apr 2022 09:51:11 +0200
Subject: [PATCH 03/10] ip problem solved

---
 agents.tf         |  6 +++---
 control_planes.tf |  6 +++---
 init.tf           | 10 +++++-----
 kubeconfig.tf     |  4 ++--
 locals.tf         |  2 --
 5 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/agents.tf b/agents.tf
index 82aa17c..538f1e5 100644
--- a/agents.tf
+++ b/agents.tf
@@ -12,11 +12,11 @@ module "agents" {
   placement_group_id     = hcloud_placement_group.k3s.id
   location               = each.value.location
   server_type            = each.value.server_type
-  ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + 2].id
+  ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + length(var.control_plane_nodepools) + 1].id
 
   # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely
   # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough.
-  private_ipv4 = cidrhost(local.network_ipv4_subnets[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + 2], each.value.index + 101)
+  private_ipv4 = cidrhost(local.network_ipv4_subnets[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + length(var.control_plane_nodepools) + 1], each.value.index + 101)
 
   labels = {
     "provisioner" = "terraform",
@@ -46,7 +46,7 @@ resource "null_resource" "agents" {
   provisioner "file" {
     content = yamlencode({
       node-name     = module.agents[each.key].name
-      server        = "https://${local.first_control_plane.private_ipv4_address}:6443"
+      server        = "https://${module.control_planes[keys(module.control_planes)[0]].private_ipv4_address}:6443"
       token         = random_password.k3s_token.result
       kubelet-arg   = "cloud-provider=external"
       flannel-iface = "eth1"
diff --git a/control_planes.tf b/control_planes.tf
index c24590a..e2e9349 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -12,11 +12,11 @@ module "control_planes" {
   placement_group_id     = hcloud_placement_group.k3s.id
   location               = each.value.location
   server_type            = each.value.server_type
-  ipv4_subnet_id         = hcloud_network_subnet.subnet[1].id
+  ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0] + 1].id
 
   # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely
   # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough.
-  private_ipv4 = cidrhost(local.network_ipv4_subnets[1], length([for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name]) + each.value.index + 101)
+  private_ipv4 = cidrhost(local.network_ipv4_subnets[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0] + 1], each.value.index + 101)
 
   labels = {
     "provisioner" = "terraform",
@@ -46,7 +46,7 @@ resource "null_resource" "control_planes" {
   provisioner "file" {
     content = yamlencode({
       node-name                = module.control_planes[each.key].name
-      server                   = "https://${local.first_control_plane.private_ipv4_address}:6443"
+      server                   = "https://${module.control_planes[each.key].private_ipv4_address == "10.1.0.101" ? "10.1.0.102" : "10.1.0.101"}:6443"
       token                    = random_password.k3s_token.result
       disable-cloud-controller = true
       disable                  = local.disable_extras
diff --git a/init.tf b/init.tf
index 884fbe6..279f837 100644
--- a/init.tf
+++ b/init.tf
@@ -3,21 +3,21 @@ resource "null_resource" "first_control_plane" {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = local.first_control_plane.ipv4_address
+    host           = module.control_planes[keys(module.control_planes)[0]].ipv4_address
   }
 
   # Generating k3s master config file
   provisioner "file" {
     content = yamlencode({
-      node-name                = local.first_control_plane.name
+      node-name                = module.control_planes[keys(module.control_planes)[0]].name
       token                    = random_password.k3s_token.result
       cluster-init             = true
       disable-cloud-controller = true
       disable                  = local.disable_extras
       flannel-iface            = "eth1"
       kubelet-arg              = "cloud-provider=external"
-      node-ip                  = local.first_control_plane.private_ipv4_address
-      advertise-address        = local.first_control_plane.private_ipv4_address
+      node-ip                  = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
+      advertise-address        = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
       node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
       node-label               = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
     })
@@ -66,7 +66,7 @@ resource "null_resource" "kustomization" {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = local.first_control_plane.ipv4_address
+    host           = module.control_planes[keys(module.control_planes)[0]].ipv4_address
   }
 
   # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured.
diff --git a/kubeconfig.tf b/kubeconfig.tf
index b4969b5..5089da6 100644
--- a/kubeconfig.tf
+++ b/kubeconfig.tf
@@ -1,7 +1,7 @@
 
 data "remote_file" "kubeconfig" {
   conn {
-    host        = local.first_control_plane.ipv4_address
+    host        = module.control_planes[keys(module.control_planes)[0]].ipv4_address
     port        = 22
     user        = "root"
     private_key = local.ssh_private_key
@@ -13,7 +13,7 @@ data "remote_file" "kubeconfig" {
 }
 
 locals {
-  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", local.first_control_plane.ipv4_address)
+  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", module.control_planes[keys(module.control_planes)[0]].ipv4_address)
   kubeconfig_parsed   = yamldecode(local.kubeconfig_external)
   kubeconfig_data = {
     host                   = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"]
diff --git a/locals.tf b/locals.tf
index 84cc38b..42ef5ee 100644
--- a/locals.tf
+++ b/locals.tf
@@ -210,6 +210,4 @@ locals {
   # Default k3s node labels
   default_agent_labels         = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
   default_control_plane_labels = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
-
-  first_control_plane = module.control_planes[keys(module.control_planes)[0]]
 }

From b142be2d8416b372a759fb528b596f37ba984870 Mon Sep 17 00:00:00 2001
From: Henk van Maanen <36051232+HenkVanMaanen@users.noreply.github.com>
Date: Mon, 11 Apr 2022 17:44:50 +0200
Subject: [PATCH 04/10] fix(control-plane): use correct taints and labels for
 first control plane node and fix existing default taints

---
 init.tf   | 4 ++--
 locals.tf | 7 +++++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/init.tf b/init.tf
index 279f837..fd6ff68 100644
--- a/init.tf
+++ b/init.tf
@@ -18,8 +18,8 @@ resource "null_resource" "first_control_plane" {
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
       advertise-address        = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
-      node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
-      node-label               = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
+      node-taint               = local.control_plane_nodepools[keys(module.control_planes)[0]].taints
+      node-label               = local.control_plane_nodepools[keys(module.control_planes)[0]].labels
     })
     destination = "/tmp/config.yaml"
   }
diff --git a/locals.tf b/locals.tf
index 42ef5ee..e3feeb0 100644
--- a/locals.tf
+++ b/locals.tf
@@ -177,7 +177,7 @@ locals {
         server_type : nodepool_obj.server_type,
         location : nodepool_obj.location,
         labels : concat(local.default_control_plane_labels, nodepool_obj.labels),
-        taints : nodepool_obj.taints,
+        taints : concat(local.default_control_plane_taints, nodepool_obj.taints),
         index : node_index
       }
     }
@@ -209,5 +209,8 @@ locals {
 
   # Default k3s node labels
   default_agent_labels         = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
-  default_control_plane_labels = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
+  default_control_plane_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
+
+  # Default k3s node taints
+  default_control_plane_taints = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
 }

From fa2d933e563a36ea0cbbf74a62ddc96f0ad6479f Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 07:40:54 +0200
Subject: [PATCH 05/10] tweaked readme

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index f7a9970..8cd1060 100644
--- a/README.md
+++ b/README.md
@@ -33,9 +33,10 @@ _Please note that we are not affiliated to Hetzner, this is just an open source
 
 - Maintenance free with auto-upgrade to the latest version of MicroOS and k3s.
 - Proper use of the Hetzner private network to minimize latency and remove the need for encryption.
-- Automatic HA with the default setting of three control-plane nodes and two agent nodepools.
+- Automatic HA with the default setting of three control-plane nodes and two agent nodes.
+- Node pools for both control-plane and agent nodes. Different locations possible (super-HA).
 - Ability to add or remove as many nodes as you want while the cluster stays running.
-- Automatic Traefik ingress controller attached to a Hetzner load balancer with proxy protocol turned on.
+- Traefik ingress controller attached to a Hetzner load balancer with proxy protocol turned on.
 - Tons of flexible configuration options to suits all needs.
 
 _It uses Terraform to deploy as it's easy to use, and Hetzner provides a great [Hetzner Terraform Provider](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs)._

From b45e77e4374af570750af7d4f5bd1f4d6f8fd743 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 14:48:43 +0200
Subject: [PATCH 06/10] fixed ips

---
 control_planes.tf | 5 +++--
 init.tf           | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/control_planes.tf b/control_planes.tf
index e2e9349..607f7aa 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -3,7 +3,7 @@ module "control_planes" {
 
   for_each = local.control_plane_nodepools
 
-  name                   = "${var.use_cluster_name_in_node_name ? "${var.cluster_name}-" : ""}control-plane"
+  name                   = "${var.use_cluster_name_in_node_name ? "${var.cluster_name}-" : ""}${each.value.nodepool_name}"
   ssh_keys               = [hcloud_ssh_key.k3s.id]
   public_key             = var.public_key
   private_key            = var.private_key
@@ -46,7 +46,7 @@ resource "null_resource" "control_planes" {
   provisioner "file" {
     content = yamlencode({
       node-name                = module.control_planes[each.key].name
-      server                   = "https://${module.control_planes[each.key].private_ipv4_address == "10.1.0.101" ? "10.1.0.102" : "10.1.0.101"}:6443"
+      server                   = "https://${module.control_planes[each.key].private_ipv4_address == module.control_planes[keys(module.control_planes)[0]].private_ipv4_address && length(module.control_planes) > 1 ? module.control_planes[keys(module.control_planes)[1]].private_ipv4_address : module.control_planes[keys(module.control_planes)[0]].private_ipv4_address}:6443"
       token                    = random_password.k3s_token.result
       disable-cloud-controller = true
       disable                  = local.disable_extras
@@ -54,6 +54,7 @@ resource "null_resource" "control_planes" {
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = module.control_planes[each.key].private_ipv4_address
       advertise-address        = module.control_planes[each.key].private_ipv4_address
+      tls-san                  = module.control_planes[each.key].ipv4_address
       node-label               = each.value.labels
       node-taint               = each.value.taints
     })
diff --git a/init.tf b/init.tf
index fd6ff68..3e83242 100644
--- a/init.tf
+++ b/init.tf
@@ -18,6 +18,7 @@ resource "null_resource" "first_control_plane" {
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
       advertise-address        = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
+      tls-san                  = module.control_planes[keys(module.control_planes)[0]].ipv4_address
       node-taint               = local.control_plane_nodepools[keys(module.control_planes)[0]].taints
       node-label               = local.control_plane_nodepools[keys(module.control_planes)[0]].labels
     })

From ef34f4fd7363ff08030a71c5e34ce9912bd02b06 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 15:23:59 +0200
Subject: [PATCH 07/10] tweaked tfvars.example

---
 terraform.tfvars.example | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 99199f0..4815efb 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -16,11 +16,16 @@ private_key  = "/home/username/.ssh/id_ed25519"
 # For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
 network_region   = "eu-central" # change to `us-east` if location is ash
 
-# At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe).
+# For the control-planes, at least 3 nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe).
 # As per rancher docs, it must be always an odd number, never even! See https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/
-# For instance, 1 is ok (non-HA), 2 not ok, 3 is ok (becomes HA).
-# Of course, you can choose any number of node pools you want, with the location you want. The only contraint on the location is that you need to stay in the same network region, basically Europe or US, see above.
+# For instance, 1 is ok (non-HA), 2 not ok, 3 is ok (becomes HA). It does not matter if they are in the same nodepool or not! So they can be in different locations, and of different types.
+
+# Of course, you can choose any number of nodepools you want, with the location you want. The only contraint on the location is that you need to stay in the same network region, basically Europe or US, see above.
 # For the server type, # The type of control plane nodes, the minimum instance supported is cpx11 (just a few cents more than cx11), see https://www.hetzner.com/cloud.
+
+# IMPORTANT: Once the cluster is created, you can a nodepool count, and even set it to 0, even change its name (if the count is taken to 0), but do not remove a nodepool from the list. 
+# You can add others at the end of the list if you want. In the case of the first nodepool, it must have a minimum count of 1 always!
+# Also, before removing any nodes, it's important to drain and cordon it, otherwise it will leave your cluster in a bad state.
 control_plane_nodepools = [
   {
     name        = "control-plane-fsn1",
@@ -44,9 +49,10 @@ control_plane_nodepools = [
 # As for the agent nodepools, below is just an example, if you do not want nodepools, just use one,
 # and change the name to what you want, it need not be "agent-big" or "agent-small", also give them the subnet prefer.
 # For single node clusters set this equal to [] or just set the counts to 0.
+
 # IMPORTANT: Once the cluster is created, you can change the count, and even set it to 0, but do not remove a nodepool from the list. 
-# You can add others at the end of the list if you want. For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/
-# For Hetzner server types see https://www.hetzner.com/cloud
+# You can add others at the end of the list if you want.
+# Also, before removing any nodes, it's important to drain and cordon it, otherwise it will leave your cluster in a bad state.
 agent_nodepools = [
   {
     name        = "agent-small",

From d3bb8b260537f0e0bb04917d523e15da1f565925 Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 15:37:50 +0200
Subject: [PATCH 08/10] tweaked tfvars.example

---
 terraform.tfvars.example | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 4815efb..9cc16a2 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -23,9 +23,16 @@ network_region   = "eu-central" # change to `us-east` if location is ash
 # Of course, you can choose any number of nodepools you want, with the location you want. The only contraint on the location is that you need to stay in the same network region, basically Europe or US, see above.
 # For the server type, # The type of control plane nodes, the minimum instance supported is cpx11 (just a few cents more than cx11), see https://www.hetzner.com/cloud.
 
-# IMPORTANT: Once the cluster is created, you can a nodepool count, and even set it to 0, even change its name (if the count is taken to 0), but do not remove a nodepool from the list. 
-# You can add others at the end of the list if you want. In the case of the first nodepool, it must have a minimum count of 1 always!
-# Also, before removing any nodes, it's important to drain and cordon it, otherwise it will leave your cluster in a bad state.
+# IMPORTANT: Once the cluster is created, you can change nodepool count, and even set it to 0 (in the case of the first control-plane nodepool, the minimum is 1),
+# You can also rename it (if the count is taken to 0), but do not remove a nodepool from the list after the cluster is created. This is due to how IPs are allocated.
+# You can freely add others nodepools the end of the list if you want, and increase the count of any.
+# Also, before decreasing the count of any nodepools to 0, it's important to drain and cordon it the nodes in question, otherwise it will leave your cluster in a bad state.
+
+# Before initializing the cluster, you can change all parameters and add or remove any nodepools. 
+# If you want to have a single node cluster, just have 1 control plane nodepools with a count of 1.
+
+# Example below:
+
 control_plane_nodepools = [
   {
     name        = "control-plane-fsn1",
@@ -45,14 +52,6 @@ control_plane_nodepools = [
   }
 ]
 
-
-# As for the agent nodepools, below is just an example, if you do not want nodepools, just use one,
-# and change the name to what you want, it need not be "agent-big" or "agent-small", also give them the subnet prefer.
-# For single node clusters set this equal to [] or just set the counts to 0.
-
-# IMPORTANT: Once the cluster is created, you can change the count, and even set it to 0, but do not remove a nodepool from the list. 
-# You can add others at the end of the list if you want.
-# Also, before removing any nodes, it's important to drain and cordon it, otherwise it will leave your cluster in a bad state.
 agent_nodepools = [
   {
     name        = "agent-small",
@@ -86,6 +85,7 @@ agent_nodepools = [
 
 # That will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer
 load_balancer_type     = "lb11"
+
 load_balancer_location = "fsn1"
 
 ### The following values are fully optional

From f1a98c72902ccc7885b859dd2025989bf867ed3d Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 17:35:09 +0200
Subject: [PATCH 09/10] fixed single node cluster

---
 README.md                | 9 ++-------
 control_planes.tf        | 3 +--
 init.tf                  | 1 -
 locals.tf                | 4 +++-
 output.tf                | 4 +++-
 terraform.tfvars.example | 9 +++++----
 6 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 8cd1060..a40325f 100644
--- a/README.md
+++ b/README.md
@@ -170,14 +170,9 @@ spec:
 
 <summary>Single-node cluster</summary>
 
-Running a development cluster on a single node, without any high-availability is possible as well.
-In this case, we don't deploy an external load-balancer, but use [k3s service load balancer](https://rancher.com/docs/k3s/latest/en/networking/#service-load-balancer) on the host itself and open up port 80 & 443 in the firewall.
+Running a development cluster on a single node, without any high-availability is possible as well. You need one control plane nodepool with a count of 1, and one agent nodepool with a count of 0.
 
-``` terraform
-control_plane_count = 1
-allow_scheduling_on_control_plane = true
-agent_nodepools = []
-```
+In this case, we don't deploy an external load-balancer, but use the default [k3s service load balancer](https://rancher.com/docs/k3s/latest/en/networking/#service-load-balancer) on the host itself and open up port 80 & 443 in the firewall (done automatically).
 
 </details>
 
diff --git a/control_planes.tf b/control_planes.tf
index 607f7aa..37655a7 100644
--- a/control_planes.tf
+++ b/control_planes.tf
@@ -46,7 +46,7 @@ resource "null_resource" "control_planes" {
   provisioner "file" {
     content = yamlencode({
       node-name                = module.control_planes[each.key].name
-      server                   = "https://${module.control_planes[each.key].private_ipv4_address == module.control_planes[keys(module.control_planes)[0]].private_ipv4_address && length(module.control_planes) > 1 ? module.control_planes[keys(module.control_planes)[1]].private_ipv4_address : module.control_planes[keys(module.control_planes)[0]].private_ipv4_address}:6443"
+      server                   = length(module.control_planes) == 1 ? null : "https://${module.control_planes[each.key].private_ipv4_address == module.control_planes[keys(module.control_planes)[0]].private_ipv4_address ? module.control_planes[keys(module.control_planes)[1]].private_ipv4_address : module.control_planes[keys(module.control_planes)[0]].private_ipv4_address}:6443"
       token                    = random_password.k3s_token.result
       disable-cloud-controller = true
       disable                  = local.disable_extras
@@ -54,7 +54,6 @@ resource "null_resource" "control_planes" {
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = module.control_planes[each.key].private_ipv4_address
       advertise-address        = module.control_planes[each.key].private_ipv4_address
-      tls-san                  = module.control_planes[each.key].ipv4_address
       node-label               = each.value.labels
       node-taint               = each.value.taints
     })
diff --git a/init.tf b/init.tf
index 3e83242..fd6ff68 100644
--- a/init.tf
+++ b/init.tf
@@ -18,7 +18,6 @@ resource "null_resource" "first_control_plane" {
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
       advertise-address        = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
-      tls-san                  = module.control_planes[keys(module.control_planes)[0]].ipv4_address
       node-taint               = local.control_plane_nodepools[keys(module.control_planes)[0]].taints
       node-label               = local.control_plane_nodepools[keys(module.control_planes)[0]].labels
     })
diff --git a/locals.tf b/locals.tf
index e3feeb0..bda11db 100644
--- a/locals.tf
+++ b/locals.tf
@@ -211,6 +211,8 @@ locals {
   default_agent_labels         = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
   default_control_plane_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [])
 
+  allow_scheduling_on_control_plane = local.is_single_node_cluster ? true : var.allow_scheduling_on_control_plane
+
   # Default k3s node taints
-  default_control_plane_taints = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
+  default_control_plane_taints = concat([], local.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"])
 }
diff --git a/output.tf b/output.tf
index 798f81f..1ea34fd 100644
--- a/output.tf
+++ b/output.tf
@@ -19,7 +19,9 @@ output "agents_public_ipv4" {
 
 output "load_balancer_public_ipv4" {
   description = "The public IPv4 address of the Hetzner load balancer"
-  value       = local.is_single_node_cluster ? module.control_planes[0].ipv4_address : var.traefik_enabled == false ? null : data.hcloud_load_balancer.traefik[0].ipv4
+  value = local.is_single_node_cluster ? [
+    for obj in module.control_planes : obj.ipv4_address
+  ][0] : var.traefik_enabled == false ? null : data.hcloud_load_balancer.traefik[0].ipv4
 }
 
 output "kubeconfig_file" {
diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index 9cc16a2..e62200b 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -28,8 +28,9 @@ network_region   = "eu-central" # change to `us-east` if location is ash
 # You can freely add others nodepools the end of the list if you want, and increase the count of any.
 # Also, before decreasing the count of any nodepools to 0, it's important to drain and cordon it the nodes in question, otherwise it will leave your cluster in a bad state.
 
-# Before initializing the cluster, you can change all parameters and add or remove any nodepools. 
-# If you want to have a single node cluster, just have 1 control plane nodepools with a count of 1.
+# Before initializing the cluster, you can change all parameters and add or remove any nodepools.
+
+# If you want to have a single node cluster, just have 1 control plane nodepools with a count of 1, and one agent nodepool with a count of 0.
 
 # Example below:
 
@@ -59,7 +60,7 @@ agent_nodepools = [
     location    = "fsn1",
     labels      = [],
     taints      = [],
-    count       = 2
+    count       = 1
   },
   {
     name        = "agent-large",
@@ -106,7 +107,7 @@ load_balancer_location = "fsn1"
 # metrics_server_enabled = false
 
 # If you want to allow non-control-plane workloads to run on the control-plane nodes set "true" below. The default is "false".
-# Also good for single node clusters.
+# True by default for single node clusters.
 # allow_scheduling_on_control_plane = true
 
 # If you want to disable automatic upgrade of k3s, you can set this to false, default is "true".

From 9e1814cd9bd3f096f82eeccab9ca104472a33cfd Mon Sep 17 00:00:00 2001
From: Karim Naufal <rimkashox@gmail.com>
Date: Tue, 12 Apr 2022 18:00:37 +0200
Subject: [PATCH 10/10] tweaked tfvars.example

---
 terraform.tfvars.example | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform.tfvars.example b/terraform.tfvars.example
index e62200b..988785c 100644
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -25,7 +25,7 @@ network_region   = "eu-central" # change to `us-east` if location is ash
 
 # IMPORTANT: Once the cluster is created, you can change nodepool count, and even set it to 0 (in the case of the first control-plane nodepool, the minimum is 1),
 # You can also rename it (if the count is taken to 0), but do not remove a nodepool from the list after the cluster is created. This is due to how IPs are allocated.
-# You can freely add others nodepools the end of the list if you want, and increase the count of any.
+# Once the cluster is initialized, you cannot add more control plane nodepools. You can freely add others agent nodepools the end of the list if you want! 
 # Also, before decreasing the count of any nodepools to 0, it's important to drain and cordon it the nodes in question, otherwise it will leave your cluster in a bad state.
 
 # Before initializing the cluster, you can change all parameters and add or remove any nodepools.