From 6d33e69b697a9e45e63e270e4d6ff0c7259b8017 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 13 Apr 2022 07:17:01 +0200 Subject: [PATCH 1/2] placement group fix attempt --- agents.tf | 2 +- control_planes.tf | 2 +- locals.tf | 5 ++++- main.tf | 17 ++++++++++------- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/agents.tf b/agents.tf index 538f1e5..38000be 100644 --- a/agents.tf +++ b/agents.tf @@ -9,7 +9,7 @@ module "agents" { private_key = var.private_key additional_public_keys = var.additional_public_keys firewall_ids = [hcloud_firewall.k3s.id] - placement_group_id = hcloud_placement_group.k3s.id + placement_group_id = element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10)) location = each.value.location server_type = each.value.server_type ipv4_subnet_id = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + length(var.control_plane_nodepools) + 1].id diff --git a/control_planes.tf b/control_planes.tf index 37655a7..b637d93 100644 --- a/control_planes.tf +++ b/control_planes.tf @@ -9,7 +9,7 @@ module "control_planes" { private_key = var.private_key additional_public_keys = var.additional_public_keys firewall_ids = [hcloud_firewall.k3s.id] - placement_group_id = hcloud_placement_group.k3s.id + placement_group_id = element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10)) location = each.value.location server_type = each.value.server_type ipv4_subnet_id = hcloud_network_subnet.subnet[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0] + 1].id diff --git a/locals.tf b/locals.tf index bda11db..addbbcb 100644 --- a/locals.tf +++ b/locals.tf @@ -1,6 +1,9 @@ locals { # if we are in a single cluster config, we use the default klipper lb instead of Hetzner LB - is_single_node_cluster = sum(concat([for v in var.control_plane_nodepools : v.count], [0])) + sum(concat([for v in var.agent_nodepools : v.count], [0])) == 1 + total_node_count = sum(concat([for v in var.control_plane_nodepools : v.count], [0])) + sum(concat([for v in var.agent_nodepools : v.count], [0])) + control_plane_count = sum(concat([for v in var.control_plane_nodepools : v.count], [0])) + agent_count = sum(concat([for v in var.agent_nodepools : v.count], [0])) + is_single_node_cluster = local.total_node_count == 1 ssh_public_key = trimspace(file(var.public_key)) # ssh_private_key is either the contents of var.private_key or null to use a ssh agent. ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key)) diff --git a/main.tf b/main.tf index e074930..6bdabfd 100644 --- a/main.tf +++ b/main.tf @@ -36,13 +36,16 @@ resource "hcloud_firewall" "k3s" { } } -resource "hcloud_placement_group" "k3s" { - name = var.cluster_name - type = "spread" - labels = { - "provisioner" = "terraform", - "engine" = "k3s" - } +resource "hcloud_placement_group" "control_plane" { + count = ceil(local.control_plane_count / 10) + name = "${var.cluster_name}-${count.index + 1}" + type = "spread" +} + +resource "hcloud_placement_group" "agent" { + count = ceil(local.agent_count / 10) + name = "${var.cluster_name}-${count.index + 1}" + type = "spread" } data "hcloud_load_balancer" "traefik" { From b17f1569f2297f3fb2832daa2090a1636212836d Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 13 Apr 2022 09:29:29 +0200 Subject: [PATCH 2/2] placement group fix --- agents.tf | 2 +- control_planes.tf | 2 +- main.tf | 7 ++++--- terraform.tfvars.example | 7 +++++-- variables.tf | 7 ++++++- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/agents.tf b/agents.tf index 38000be..1cda0ea 100644 --- a/agents.tf +++ b/agents.tf @@ -9,7 +9,7 @@ module "agents" { private_key = var.private_key additional_public_keys = var.additional_public_keys firewall_ids = [hcloud_firewall.k3s.id] - placement_group_id = element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10)) + placement_group_id = var.placement_group_disable ? 0 : element(hcloud_placement_group.agent.*.id, ceil(each.value.index / 10)) location = each.value.location server_type = each.value.server_type ipv4_subnet_id = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + length(var.control_plane_nodepools) + 1].id diff --git a/control_planes.tf b/control_planes.tf index b637d93..d73d6c8 100644 --- a/control_planes.tf +++ b/control_planes.tf @@ -9,7 +9,7 @@ module "control_planes" { private_key = var.private_key additional_public_keys = var.additional_public_keys firewall_ids = [hcloud_firewall.k3s.id] - placement_group_id = element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10)) + placement_group_id = var.placement_group_disable ? 0 : element(hcloud_placement_group.control_plane.*.id, ceil(each.value.index / 10)) location = each.value.location server_type = each.value.server_type ipv4_subnet_id = hcloud_network_subnet.subnet[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0] + 1].id diff --git a/main.tf b/main.tf index 6bdabfd..c09a324 100644 --- a/main.tf +++ b/main.tf @@ -38,13 +38,13 @@ resource "hcloud_firewall" "k3s" { resource "hcloud_placement_group" "control_plane" { count = ceil(local.control_plane_count / 10) - name = "${var.cluster_name}-${count.index + 1}" + name = "${var.cluster_name}-control-plane-${count.index + 1}" type = "spread" } resource "hcloud_placement_group" "agent" { count = ceil(local.agent_count / 10) - name = "${var.cluster_name}-${count.index + 1}" + name = "${var.cluster_name}-agent-${count.index + 1}" type = "spread" } @@ -74,9 +74,10 @@ resource "null_resource" "destroy_traefik_loadbalancer" { local_sensitive_file.kubeconfig, null_resource.control_planes[0], hcloud_network_subnet.subnet, + hcloud_placement_group.control_plane, + hcloud_placement_group.agent, hcloud_network.k3s, hcloud_firewall.k3s, - hcloud_placement_group.k3s, hcloud_ssh_key.k3s ] } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 988785c..60ce5b5 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -84,9 +84,8 @@ agent_nodepools = [ } ] -# That will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer +# LB location and type, the latter will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer load_balancer_type = "lb11" - load_balancer_location = "fsn1" ### The following values are fully optional @@ -147,3 +146,7 @@ load_balancer_location = "fsn1" # If you want to configure additional Arguments for traefik, enter them here as a list and in the form of traefik CLI arguments; see https://doc.traefik.io/traefik/reference/static-configuration/cli/ # Example: traefik_additional_options = ["--log.level=DEBUG", "--tracing=true"] # traefik_additional_options = [] + +# If you want to disable the automatic use of placement group "spread". See https://docs.hetzner.com/cloud/placement-groups/overview/ +# That may be useful if you need to deploy more than 500 nodes! The default is "false". +# placement_group_disable = true diff --git a/variables.tf b/variables.tf index df7aaf8..365ad40 100644 --- a/variables.tf +++ b/variables.tf @@ -138,5 +138,10 @@ variable "cluster_name" { variable "traefik_additional_options" { type = list(string) default = [] - +} + +variable "placement_group_disable" { + type = bool + default = false + description = "Whether to disable placement groups" }