From a72facdd1a2e389294c1ea0ebd8cf31672b4a603 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 9 Mar 2022 02:07:24 +0100 Subject: [PATCH 1/6] automated the creation of the subnets --- agents.tf | 4 ++-- control_planes.tf | 4 ++-- locals.tf | 10 ++++++++-- main.tf | 16 +++------------- terraform.tfvars.example | 12 ------------ variables.tf | 11 ----------- 6 files changed, 15 insertions(+), 42 deletions(-) diff --git a/agents.tf b/agents.tf index a8e5423..ce75203 100644 --- a/agents.tf +++ b/agents.tf @@ -12,11 +12,11 @@ module "agents" { placement_group_id = hcloud_placement_group.k3s.id location = var.location server_type = each.value.server_type - ipv4_subnet_id = hcloud_network_subnet.subnet[each.value.subnet].id + ipv4_subnet_id = hcloud_network_subnet.subnet[index(keys(var.agent_nodepools), each.value.nodepool_name) + 2].id # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough. - private_ipv4 = cidrhost(var.network_ipv4_subnets[each.value.subnet], each.value.index + 101) + private_ipv4 = cidrhost(local.network_ipv4_subnets[index(keys(var.agent_nodepools), each.value.nodepool_name) + 2], each.value.index + 101) labels = { "provisioner" = "terraform", diff --git a/control_planes.tf b/control_planes.tf index e3130c4..a8957aa 100644 --- a/control_planes.tf +++ b/control_planes.tf @@ -11,11 +11,11 @@ module "control_planes" { placement_group_id = hcloud_placement_group.k3s.id location = var.location server_type = var.control_plane_server_type - ipv4_subnet_id = hcloud_network_subnet.subnet["control_plane"].id + ipv4_subnet_id = hcloud_network_subnet.subnet[1].id # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough. - private_ipv4 = cidrhost(var.network_ipv4_subnets["control_plane"], count.index + 101) + private_ipv4 = cidrhost(local.network_ipv4_subnets[1], count.index + 101) labels = { "provisioner" = "terraform", diff --git a/locals.tf b/locals.tf index a01b5cd..45116c6 100644 --- a/locals.tf +++ b/locals.tf @@ -23,7 +23,7 @@ locals { hetzner_cloud_api_ipv4 = "213.239.246.1/32" whitelisted_ips = [ - var.network_ipv4_range, + local.network_ipv4_cidr, local.hetzner_metadata_service_ipv4, local.hetzner_cloud_api_ipv4, "127.0.0.1/32", @@ -175,9 +175,15 @@ locals { format("%s-%s", nodepool_name, index) => { nodepool_name : nodepool_name, server_type : nodepool_obj.server_type, - subnet : nodepool_obj.subnet, index : index } } ]...) + + # The main network cidr that all subnets will be created upon + network_ipv4_cidr = "10.0.0.0/8" + + # The first two subnets are respectively the default subnet 10.0.0.0/16 use for potientially anything and 10.1.0.0/16 used for control plane nodes. + # the rest of the subnets are for agent nodes in each nodepools. + network_ipv4_subnets = [for index in range(length(var.agent_nodepools) + 2) : cidrsubnet(local.network_ipv4_cidr, 8, index)] } diff --git a/main.tf b/main.tf index b78a41d..ff61cbd 100644 --- a/main.tf +++ b/main.tf @@ -15,25 +15,15 @@ resource "hcloud_ssh_key" "k3s" { resource "hcloud_network" "k3s" { name = random_pet.cluster.id - ip_range = var.network_ipv4_range -} - -# This is the default subnet to be used by the load balancer. -resource "hcloud_network_subnet" "default" { - network_id = hcloud_network.k3s.id - type = "cloud" - network_zone = var.network_region - ip_range = "10.0.0.0/16" + ip_range = local.network_ipv4_cidr } resource "hcloud_network_subnet" "subnet" { - for_each = var.network_ipv4_subnets + count = length(local.network_ipv4_subnets) network_id = hcloud_network.k3s.id type = "cloud" network_zone = var.network_region - ip_range = each.value - - depends_on = [hcloud_network_subnet.default] + ip_range = local.network_ipv4_subnets[count.index] } resource "hcloud_firewall" "k3s" { diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 1688b31..cd3f5dd 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -18,15 +18,6 @@ private_key = "/home/username/.ssh/id_ed25519" location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location network_region = "eu-central" # change to `us-east` if location is ash -# You can have up to as many subnets as you want (preferably if the form of 10.X.0.0/16), -# their primary use is to logically separate the nodes. -# The control_plane network is mandatory. -network_ipv4_subnets = { - control_plane = "10.1.0.0/16" - agent_big = "10.2.0.0/16" - agent_small = "10.3.0.0/16" -} - # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe). # As per rancher docs, it must be always an odd number, never even! See https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/ # For instance, 1 is ok (non-HA), 2 not ok, 3 is ok (becomes HA). @@ -56,9 +47,6 @@ load_balancer_type = "lb11" ### The following values are fully optional -# It's best to leave the network range as is, unless you know what you are doing. The default is "10.0.0.0/8". -# network_ipv4_range = "10.0.0.0/8" - # If you want to use a specific Hetzner CCM and CSI version, set them below, otherwise leave as is for the latest versions # hetzner_ccm_version = "" # hetzner_csi_version = "" diff --git a/variables.tf b/variables.tf index 49d0e51..90e0955 100644 --- a/variables.tf +++ b/variables.tf @@ -30,17 +30,6 @@ variable "network_region" { type = string } -variable "network_ipv4_range" { - description = "Default IPv4 range for network" - type = string - default = "10.0.0.0/8" -} - -variable "network_ipv4_subnets" { - description = "Subnets definition for default network" - type = map(string) -} - variable "control_plane_server_type" { description = "Default control plane server type" type = string From f8251427206ab363383046860616261f15a9acc3 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 9 Mar 2022 02:17:00 +0100 Subject: [PATCH 2/6] tweaked tfvars.example --- terraform.tfvars.example | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/terraform.tfvars.example b/terraform.tfvars.example index cd3f5dd..9dd4a5e 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -32,13 +32,11 @@ control_plane_server_type = "cpx11" agent_nodepools = { agent-big = { server_type = "cpx21", - count = 1, - subnet = "agent_big", + count = 1 } agent-small = { server_type = "cpx11", - count = 2, - subnet = "agent_small", + count = 2 } } From 2b7d2722c55c901979da1197a2061bd1b2af69d3 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 9 Mar 2022 03:15:15 +0100 Subject: [PATCH 3/6] changed the agent_nodepools to be a list --- agents.tf | 4 ++-- locals.tf | 8 ++++---- terraform.tfvars.example | 20 ++++++++++++-------- variables.tf | 4 ++-- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/agents.tf b/agents.tf index ce75203..6732d30 100644 --- a/agents.tf +++ b/agents.tf @@ -12,11 +12,11 @@ module "agents" { placement_group_id = hcloud_placement_group.k3s.id location = var.location server_type = each.value.server_type - ipv4_subnet_id = hcloud_network_subnet.subnet[index(keys(var.agent_nodepools), each.value.nodepool_name) + 2].id + ipv4_subnet_id = hcloud_network_subnet.subnet[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + 2].id # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough. - private_ipv4 = cidrhost(local.network_ipv4_subnets[index(keys(var.agent_nodepools), each.value.nodepool_name) + 2], each.value.index + 101) + private_ipv4 = cidrhost(local.network_ipv4_subnets[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0] + 2], each.value.index + 101) labels = { "provisioner" = "terraform", diff --git a/locals.tf b/locals.tf index 45116c6..13ea3db 100644 --- a/locals.tf +++ b/locals.tf @@ -1,6 +1,6 @@ locals { # if we are in a single cluster config, we use the default klipper lb instead of Hetzner LB - is_single_node_cluster = var.control_plane_count + length(keys(var.agent_nodepools)) == 1 + is_single_node_cluster = var.control_plane_count + length(var.agent_nodepools) == 1 ssh_public_key = trimspace(file(var.public_key)) # ssh_private_key is either the contents of var.private_key or null to use a ssh agent. ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key)) @@ -170,10 +170,10 @@ locals { install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"], local.apply_k3s_selinux) agent_nodepools = merge([ - for nodepool_name, nodepool_obj in var.agent_nodepools : { + for nodepool_obj in var.agent_nodepools : { for index in range(nodepool_obj.count) : - format("%s-%s", nodepool_name, index) => { - nodepool_name : nodepool_name, + format("%s-%s", nodepool_obj.name, index) => { + nodepool_name : nodepool_obj.name, server_type : nodepool_obj.server_type, index : index } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 9dd4a5e..9484d24 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -28,17 +28,21 @@ control_plane_server_type = "cpx11" # As for the agent nodepools, below is just an example, if you do not want nodepools, just use one, # and change the name to what you want, it need not be "agent-big" or "agent-small", also give them the subnet prefer. -# For single node clusters set this equal to {} -agent_nodepools = { - agent-big = { +# For single node clusters set this equal to [] or just set the counts to 0 +# IMPORTANT: Once the cluster is created, you can change the count, and even set it to 0, but do not remove a nodepool from the list. +# You can add others at the end of the list if you want. +agent_nodepools = [ + { + name = "agent-small", + server_type = "cpx11", + count = 2 + }, + { + name = "agent-large", server_type = "cpx21", count = 1 } - agent-small = { - server_type = "cpx11", - count = 2 - } -} +] # That will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer load_balancer_type = "lb11" diff --git a/variables.tf b/variables.tf index 90e0955..3460697 100644 --- a/variables.tf +++ b/variables.tf @@ -53,8 +53,8 @@ variable "load_balancer_disable_ipv6" { variable "agent_nodepools" { description = "Number of agent nodes." - type = map(any) - default = {} + type = list(any) + default = [] } variable "hetzner_ccm_version" { From be1c4efefb9923358f2b4af45bc13dd1b62e9a61 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 9 Mar 2022 03:43:10 +0100 Subject: [PATCH 4/6] tweaked single node setup --- locals.tf | 2 +- terraform.tfvars.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/locals.tf b/locals.tf index 13ea3db..8b08485 100644 --- a/locals.tf +++ b/locals.tf @@ -1,6 +1,6 @@ locals { # if we are in a single cluster config, we use the default klipper lb instead of Hetzner LB - is_single_node_cluster = var.control_plane_count + length(var.agent_nodepools) == 1 + is_single_node_cluster = var.control_plane_count + sum(concat([for v in var.agent_nodepools : v.count], [0])) == 1 ssh_public_key = trimspace(file(var.public_key)) # ssh_private_key is either the contents of var.private_key or null to use a ssh agent. ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key)) diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 9484d24..b7310ad 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -28,7 +28,7 @@ control_plane_server_type = "cpx11" # As for the agent nodepools, below is just an example, if you do not want nodepools, just use one, # and change the name to what you want, it need not be "agent-big" or "agent-small", also give them the subnet prefer. -# For single node clusters set this equal to [] or just set the counts to 0 +# For single node clusters set this equal to [] or just set the counts to 0. # IMPORTANT: Once the cluster is created, you can change the count, and even set it to 0, but do not remove a nodepool from the list. # You can add others at the end of the list if you want. agent_nodepools = [ From 9cc302c6c9cf1b0e37eecbbcf5ff7fff137c71ee Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 9 Mar 2022 09:40:20 +0100 Subject: [PATCH 5/6] small fix post merging master --- main.tf | 8 -------- 1 file changed, 8 deletions(-) diff --git a/main.tf b/main.tf index 0db3b0d..f125959 100644 --- a/main.tf +++ b/main.tf @@ -13,14 +13,6 @@ resource "hcloud_network" "k3s" { ip_range = var.network_ipv4_range } -# This is the default subnet to be used by the load balancer. -resource "hcloud_network_subnet" "default" { - network_id = hcloud_network.k3s.id - type = "cloud" - network_zone = var.network_region - ip_range = "10.0.0.0/16" -} - resource "hcloud_network_subnet" "subnet" { count = length(local.network_ipv4_subnets) network_id = hcloud_network.k3s.id From a051480af5f03728c9f314ce5e915a68f2636368 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 9 Mar 2022 09:47:57 +0100 Subject: [PATCH 6/6] small fix post merging master --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index f125959..d84ec0a 100644 --- a/main.tf +++ b/main.tf @@ -10,7 +10,7 @@ resource "hcloud_ssh_key" "k3s" { resource "hcloud_network" "k3s" { name = var.cluster_name - ip_range = var.network_ipv4_range + ip_range = local.network_ipv4_cidr } resource "hcloud_network_subnet" "subnet" {