Merge pull request #151 from kube-hetzner/placement-group-fix

Placement group fix

agents.tf

@@ -9,7 +9,7 @@ module "agents" {
   private_key            = var.private_key
   additional_public_keys = var.additional_public_keys
   firewall_ids           = [hcloud_firewall.k3s.id]
-  placement_group_id     = hcloud_placement_group.k3s.id
+  placement_group_id     = var.placement_group_disable ? 0 : element(hcloud_placement_group.agent.*.id, ceil(each.value.index / 10))
   location               = each.value.location
   server_type            = each.value.server_type
   ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + length(var.control_plane_nodepools) + 1].id

control_planes.tf

@@ -9,7 +9,7 @@ module "control_planes" {
   private_key            = var.private_key
   additional_public_keys = var.additional_public_keys
   firewall_ids           = [hcloud_firewall.k3s.id]
-  placement_group_id     = hcloud_placement_group.k3s.id
+  placement_group_id     = var.placement_group_disable ? 0 : element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10))
   location               = each.value.location
   server_type            = each.value.server_type
   ipv4_subnet_id         = hcloud_network_subnet.subnet[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0] + 1].id

locals.tf

@@ -1,6 +1,9 @@
 locals {
   # if we are in a single cluster config, we use the default klipper lb instead of Hetzner LB
-  is_single_node_cluster = sum(concat([for v in var.control_plane_nodepools : v.count], [0])) + sum(concat([for v in var.agent_nodepools : v.count], [0])) == 1
+  total_node_count       = sum(concat([for v in var.control_plane_nodepools : v.count], [0])) + sum(concat([for v in var.agent_nodepools : v.count], [0]))
+  control_plane_count    = sum(concat([for v in var.control_plane_nodepools : v.count], [0]))
+  agent_count            = sum(concat([for v in var.agent_nodepools : v.count], [0]))
+  is_single_node_cluster = local.total_node_count == 1
   ssh_public_key         = trimspace(file(var.public_key))
   # ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
   ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key))

main.tf

@@ -36,13 +36,16 @@ resource "hcloud_firewall" "k3s" {
   }
 }
 
-resource "hcloud_placement_group" "k3s" {
+resource "hcloud_placement_group" "control_plane" {
-  name = var.cluster_name
+  count = ceil(local.control_plane_count / 10)
-  type = "spread"
+  name  = "${var.cluster_name}-control-plane-${count.index + 1}"
-  labels = {
+  type  = "spread"
-    "provisioner" = "terraform",
+}
-    "engine"      = "k3s"
+
-  }
+resource "hcloud_placement_group" "agent" {
+  count = ceil(local.agent_count / 10)
+  name  = "${var.cluster_name}-agent-${count.index + 1}"
+  type  = "spread"
 }
 
 data "hcloud_load_balancer" "traefik" {
@@ -71,9 +74,10 @@ resource "null_resource" "destroy_traefik_loadbalancer" {
     local_sensitive_file.kubeconfig,
     null_resource.control_planes[0],
     hcloud_network_subnet.subnet,
+    hcloud_placement_group.control_plane,
+    hcloud_placement_group.agent,
     hcloud_network.k3s,
     hcloud_firewall.k3s,
-    hcloud_placement_group.k3s,
     hcloud_ssh_key.k3s
   ]
 }

terraform.tfvars.example

@@ -84,9 +84,8 @@ agent_nodepools = [
   }
 ]
 
-# That will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer
+# LB location and type, the latter will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer
 load_balancer_type     = "lb11"
-
 load_balancer_location = "fsn1"
 
 ### The following values are fully optional
@@ -147,3 +146,7 @@ load_balancer_location = "fsn1"
 # If you want to configure additional Arguments for traefik, enter them here as a list and in the form of traefik CLI arguments; see https://doc.traefik.io/traefik/reference/static-configuration/cli/
 # Example: traefik_additional_options = ["--log.level=DEBUG", "--tracing=true"]
 # traefik_additional_options = []
+
+# If you want to disable the automatic use of placement group "spread". See https://docs.hetzner.com/cloud/placement-groups/overview/
+# That may be useful if you need to deploy more than 500 nodes! The default is "false".
+# placement_group_disable = true

variables.tf

@@ -138,5 +138,10 @@ variable "cluster_name" {
 variable "traefik_additional_options" {
   type    = list(string)
   default = []
-
+}
+
+variable "placement_group_disable" {
+  type        = bool
+  default     = false
+  description = "Whether to disable placement groups"
 }