From 7d301f2c6d4cc616a8e7176ed850d536c906508a Mon Sep 17 00:00:00 2001 From: jodhi Date: Wed, 23 Feb 2022 22:46:46 +0700 Subject: [PATCH 1/7] [WIP] nodepools feature --- agents.tf | 18 +++++++++--------- locals.tf | 10 ++++++++++ output.tf | 4 +++- terraform.tfvars.example | 13 +++++++++++-- variables.tf | 11 +++++------ 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/agents.tf b/agents.tf index 7325773..be36867 100644 --- a/agents.tf +++ b/agents.tf @@ -1,9 +1,9 @@ module "agents" { source = "./modules/host" - count = var.agents_num - name = "k3s-agent-${count.index}" + for_each = local.agent_nodepools + name = each.key ssh_keys = [hcloud_ssh_key.k3s.id] public_key = var.public_key private_key = var.private_key @@ -12,8 +12,8 @@ module "agents" { placement_group_id = hcloud_placement_group.k3s.id location = var.location network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index) - server_type = var.agent_server_type + ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + each.value.index) + server_type = each.value.server_type labels = { "provisioner" = "terraform", @@ -24,28 +24,28 @@ module "agents" { } resource "null_resource" "agents" { - count = var.agents_num + for_each = local.agent_nodepools triggers = { - agent_id = module.agents[count.index].id + agent_id = module.agents[each.key].id } connection { user = "root" private_key = local.ssh_private_key agent_identity = local.ssh_identity - host = module.agents[count.index].ipv4_address + host = module.agents[each.key].ipv4_address } # Generating k3s agent config file provisioner "file" { content = yamlencode({ - node-name = module.agents[count.index].name + node-name = module.agents[each.key].name server = "https://${local.first_control_plane_network_ip}:6443" token = random_password.k3s_token.result kubelet-arg = "cloud-provider=external" flannel-iface = "eth1" - node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index) + node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + each.value.index) node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] }) destination = "/tmp/config.yaml" diff --git a/locals.tf b/locals.tf index 72c2e61..69b0bda 100644 --- a/locals.tf +++ b/locals.tf @@ -30,4 +30,14 @@ locals { install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_EXEC=server sh -"]) install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_EXEC=agent sh -"]) + + agent_nodepools = merge([ + for nodepool_name, nodepool_obj in var.agent_nodepools : { + for index in range(nodepool_obj.count) : + format("%s-%s", nodepool_name, index) => { + index : index, # just for the compatibility with previous structure + server_type : nodepool_obj.server_type + } + } + ]...) } diff --git a/output.tf b/output.tf index 62e6c6f..6710da1 100644 --- a/output.tf +++ b/output.tf @@ -4,7 +4,9 @@ output "controlplanes_public_ip" { } output "agents_public_ip" { - value = module.agents.*.ipv4_address + value = [ + for obj in module.agents : obj.ipv4_address + ] description = "The public IP addresses of the agent server." } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 323955c..57a85f5 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -9,7 +9,6 @@ private_key = "/home/username/.ssh/id_ed25519" # For Hetzner server types see https://www.hetzner.com/cloud location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location network_region = "eu-central" # change to `us-east` if location is ash -agent_server_type = "cpx21" control_plane_server_type = "cpx11" lb_server_type = "lb11" @@ -17,7 +16,17 @@ lb_server_type = "lb11" servers_num = 3 # For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades. -agents_num = 2 +agent_nodepools = { + big = { + server_type = "cpx31", + count = 3 + } + # Will fail because of the ip address collision (we use index number) + # small = { + # server_type = "cpx31", + # count = 3 + # } +} # If you want to use a specific Hetzner CCM and CSI version, set them below, otherwise leave as is for the latest versions # hetzner_ccm_version = "" diff --git a/variables.tf b/variables.tf index 9346e5d..ad2e053 100644 --- a/variables.tf +++ b/variables.tf @@ -35,10 +35,6 @@ variable "control_plane_server_type" { type = string } -variable "agent_server_type" { - description = "Default agent server type" - type = string -} variable "lb_server_type" { description = "Default load balancer server type" @@ -56,9 +52,12 @@ variable "servers_num" { type = number } -variable "agents_num" { +variable "agent_nodepools" { description = "Number of agent nodes." - type = number + type = map(object({ + server_type = string + count = number + })) } variable "hetzner_ccm_version" { From 58c630ba78a2f539e3dd0276e8a1c9efccb3e178 Mon Sep 17 00:00:00 2001 From: jodhi Date: Thu, 24 Feb 2022 22:49:36 +0700 Subject: [PATCH 2/7] not hardcode the ip allocation for agents --- agents.tf | 3 +-- locals.tf | 2 +- modules/host/main.tf | 2 +- modules/host/variables.tf | 1 + variables.tf | 10 ++++++---- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/agents.tf b/agents.tf index be36867..7647cc0 100644 --- a/agents.tf +++ b/agents.tf @@ -12,7 +12,6 @@ module "agents" { placement_group_id = hcloud_placement_group.k3s.id location = var.location network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + each.value.index) server_type = each.value.server_type labels = { @@ -45,7 +44,7 @@ resource "null_resource" "agents" { token = random_password.k3s_token.result kubelet-arg = "cloud-provider=external" flannel-iface = "eth1" - node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + each.value.index) + node-ip = module.agents[each.key].ipv4_address node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] }) destination = "/tmp/config.yaml" diff --git a/locals.tf b/locals.tf index 69b0bda..7932296 100644 --- a/locals.tf +++ b/locals.tf @@ -33,7 +33,7 @@ locals { agent_nodepools = merge([ for nodepool_name, nodepool_obj in var.agent_nodepools : { - for index in range(nodepool_obj.count) : + for index in range(lookup(nodepool_obj, "count", var.agents_num)) : format("%s-%s", nodepool_name, index) => { index : index, # just for the compatibility with previous structure server_type : nodepool_obj.server_type diff --git a/modules/host/main.tf b/modules/host/main.tf index 0aa14cb..56e794e 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -14,7 +14,7 @@ resource "hcloud_server" "server" { network { network_id = var.network_id - ip = var.ip + ip = try(var.ip, null) } connection { diff --git a/modules/host/variables.tf b/modules/host/variables.tf index b336fc5..9c328b1 100644 --- a/modules/host/variables.tf +++ b/modules/host/variables.tf @@ -60,6 +60,7 @@ variable "network_id" { } variable "ip" { + default = null description = "The IP" type = string nullable = true diff --git a/variables.tf b/variables.tf index ad2e053..73d7301 100644 --- a/variables.tf +++ b/variables.tf @@ -52,12 +52,14 @@ variable "servers_num" { type = number } +variable "agents_num" { + description = "Default agent server type" + type = number +} + variable "agent_nodepools" { description = "Number of agent nodes." - type = map(object({ - server_type = string - count = number - })) + type = map(any) } variable "hetzner_ccm_version" { From bb5ab6bd9a7e8ce68a47b45ccf0d4213b06784c6 Mon Sep 17 00:00:00 2001 From: jodhi Date: Thu, 24 Feb 2022 23:19:31 +0700 Subject: [PATCH 3/7] make sure control-plane ip dynamic --- servers.tf | 1 - terraform.tfvars.example | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/servers.tf b/servers.tf index a10c7d7..acfc3f2 100644 --- a/servers.tf +++ b/servers.tf @@ -12,7 +12,6 @@ module "control_planes" { placement_group_id = hcloud_placement_group.k3s.id location = var.location network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index) server_type = var.control_plane_server_type labels = { diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 57a85f5..93945b2 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -16,16 +16,16 @@ lb_server_type = "lb11" servers_num = 3 # For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades. +agents_num = 2 + agent_nodepools = { big = { server_type = "cpx31", - count = 3 + count = 1 + } + small = { + server_type = "cpx21", } - # Will fail because of the ip address collision (we use index number) - # small = { - # server_type = "cpx31", - # count = 3 - # } } # If you want to use a specific Hetzner CCM and CSI version, set them below, otherwise leave as is for the latest versions From 519cae04d33a0586c10d86cda1c5438f7f223703 Mon Sep 17 00:00:00 2001 From: jodhi Date: Thu, 24 Feb 2022 23:25:13 +0700 Subject: [PATCH 4/7] fix typo --- terraform.tfvars.example | 2 +- variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 93945b2..481de9f 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -16,7 +16,7 @@ lb_server_type = "lb11" servers_num = 3 # For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades. -agents_num = 2 +agents_num = 2 agent_nodepools = { big = { diff --git a/variables.tf b/variables.tf index 73d7301..0b1d8f5 100644 --- a/variables.tf +++ b/variables.tf @@ -53,7 +53,7 @@ variable "servers_num" { } variable "agents_num" { - description = "Default agent server type" + description = "Number of agent nodes." type = number } From b5039e2bc1705c3b58448dde6b8b164288d02947 Mon Sep 17 00:00:00 2001 From: jodhi Date: Thu, 24 Feb 2022 23:57:36 +0700 Subject: [PATCH 5/7] fix dynamic private ip --- locals.tf | 1 - modules/host/main.tf | 10 +++++----- modules/host/out.tf | 2 +- modules/host/variables.tf | 7 ------- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/locals.tf b/locals.tf index 7932296..3c78ad0 100644 --- a/locals.tf +++ b/locals.tf @@ -35,7 +35,6 @@ locals { for nodepool_name, nodepool_obj in var.agent_nodepools : { for index in range(lookup(nodepool_obj, "count", var.agents_num)) : format("%s-%s", nodepool_name, index) => { - index : index, # just for the compatibility with previous structure server_type : nodepool_obj.server_type } } diff --git a/modules/host/main.tf b/modules/host/main.tf index 56e794e..b8b8dd9 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -12,11 +12,6 @@ resource "hcloud_server" "server" { labels = var.labels - network { - network_id = var.network_id - ip = try(var.ip, null) - } - connection { user = "root" private_key = local.ssh_private_key @@ -68,3 +63,8 @@ resource "hcloud_server" "server" { ] } } + +resource "hcloud_server_network" "server" { + server_id = hcloud_server.server.id + network_id = var.network_id +} diff --git a/modules/host/out.tf b/modules/host/out.tf index d2997ba..1c373c1 100644 --- a/modules/host/out.tf +++ b/modules/host/out.tf @@ -3,7 +3,7 @@ output "ipv4_address" { } output "private_ipv4_address" { - value = var.ip + value = hcloud_server_network.server.ip } output "name" { diff --git a/modules/host/variables.tf b/modules/host/variables.tf index 9c328b1..77a8176 100644 --- a/modules/host/variables.tf +++ b/modules/host/variables.tf @@ -59,13 +59,6 @@ variable "network_id" { type = number } -variable "ip" { - default = null - description = "The IP" - type = string - nullable = true -} - variable "server_type" { description = "The server type" type = string From c9101824fca04d4a898c6bfa2d742f3351f1295c Mon Sep 17 00:00:00 2001 From: jodhi Date: Sat, 26 Feb 2022 01:16:38 +0700 Subject: [PATCH 6/7] add specific subnet for nodepool --- agents.tf | 10 +++++++--- data.tf | 2 +- init.tf | 2 +- locals.tf | 6 ++++-- main.tf | 13 +++++++------ modules/host/main.tf | 5 +++-- modules/host/variables.tf | 11 ++++++++--- servers.tf | 9 +++++++-- terraform.tfvars.example | 20 ++++++++++++++------ variables.tf | 15 ++++++++++----- 10 files changed, 62 insertions(+), 31 deletions(-) diff --git a/agents.tf b/agents.tf index 7647cc0..8c3f6c9 100644 --- a/agents.tf +++ b/agents.tf @@ -11,15 +11,19 @@ module "agents" { firewall_ids = [hcloud_firewall.k3s.id] placement_group_id = hcloud_placement_group.k3s.id location = var.location - network_id = hcloud_network.k3s.id server_type = each.value.server_type - + subnet_id = hcloud_network_subnet.subnet[each.value.subnet].id + private_ip = cidrhost(var.network_subnets[each.value.subnet], each.value.index + 1) labels = { "provisioner" = "terraform", "engine" = "k3s" } hcloud_token = var.hcloud_token + + depends_on = [ + hcloud_network_subnet.subnet + ] } resource "null_resource" "agents" { @@ -73,6 +77,6 @@ resource "null_resource" "agents" { depends_on = [ null_resource.first_control_plane, - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet ] } diff --git a/data.tf b/data.tf index 41d1cff..6b2c552 100644 --- a/data.tf +++ b/data.tf @@ -15,4 +15,4 @@ data "github_release" "kured" { repository = "kured" owner = "weaveworks" retrieve_by = "latest" -} \ No newline at end of file +} diff --git a/init.tf b/init.tf index cb3eec9..4f1d350 100644 --- a/init.tf +++ b/init.tf @@ -58,7 +58,7 @@ resource "null_resource" "first_control_plane" { } depends_on = [ - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet["control_plane"] ] } diff --git a/locals.tf b/locals.tf index 3c78ad0..785c1c3 100644 --- a/locals.tf +++ b/locals.tf @@ -33,9 +33,11 @@ locals { agent_nodepools = merge([ for nodepool_name, nodepool_obj in var.agent_nodepools : { - for index in range(lookup(nodepool_obj, "count", var.agents_num)) : + for index in range(nodepool_obj.count) : format("%s-%s", nodepool_name, index) => { - server_type : nodepool_obj.server_type + server_type : nodepool_obj.server_type, + subnet : lookup(nodepool_obj, "subnet", "default"), + index : index } } ]...) diff --git a/main.tf b/main.tf index 3e71a22..34ba1bd 100644 --- a/main.tf +++ b/main.tf @@ -10,14 +10,15 @@ resource "hcloud_ssh_key" "k3s" { resource "hcloud_network" "k3s" { name = "k3s" - ip_range = "10.0.0.0/8" + ip_range = var.network_ip_range } -resource "hcloud_network_subnet" "k3s" { +resource "hcloud_network_subnet" "subnet" { + for_each = var.network_subnets network_id = hcloud_network.k3s.id type = "cloud" network_zone = var.network_region - ip_range = "10.0.0.0/16" + ip_range = each.value } resource "hcloud_firewall" "k3s" { @@ -29,8 +30,8 @@ resource "hcloud_firewall" "k3s" { protocol = "tcp" port = "any" source_ips = [ + var.network_ip_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] @@ -40,8 +41,8 @@ resource "hcloud_firewall" "k3s" { protocol = "udp" port = "any" source_ips = [ + var.network_ip_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] @@ -50,8 +51,8 @@ resource "hcloud_firewall" "k3s" { direction = "in" protocol = "icmp" source_ips = [ + var.network_ip_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] diff --git a/modules/host/main.tf b/modules/host/main.tf index b8b8dd9..9ce3e30 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -65,6 +65,7 @@ resource "hcloud_server" "server" { } resource "hcloud_server_network" "server" { - server_id = hcloud_server.server.id - network_id = var.network_id + ip = var.private_ip + server_id = hcloud_server.server.id + subnet_id = var.subnet_id } diff --git a/modules/host/variables.tf b/modules/host/variables.tf index 77a8176..614317f 100644 --- a/modules/host/variables.tf +++ b/modules/host/variables.tf @@ -54,9 +54,14 @@ variable "location" { type = string } -variable "network_id" { - description = "The network or subnet id" - type = number +variable "subnet_id" { + description = "The subnet id" + type = string +} + +variable "private_ip" { + description = "Private IP for the server" + type = string } variable "server_type" { diff --git a/servers.tf b/servers.tf index acfc3f2..0263f15 100644 --- a/servers.tf +++ b/servers.tf @@ -11,8 +11,9 @@ module "control_planes" { firewall_ids = [hcloud_firewall.k3s.id] placement_group_id = hcloud_placement_group.k3s.id location = var.location - network_id = hcloud_network.k3s.id server_type = var.control_plane_server_type + subnet_id = hcloud_network_subnet.subnet["control_plane"].id + private_ip = cidrhost(var.network_subnets["control_plane"], count.index + 1) labels = { "provisioner" = "terraform", @@ -20,6 +21,10 @@ module "control_planes" { } hcloud_token = var.hcloud_token + + depends_on = [ + hcloud_network_subnet.subnet + ] } resource "null_resource" "control_planes" { @@ -78,6 +83,6 @@ resource "null_resource" "control_planes" { depends_on = [ null_resource.first_control_plane, - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet ] } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 481de9f..f97712b 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -7,24 +7,32 @@ private_key = "/home/username/.ssh/id_ed25519" # These can be customized, or left with the default values # For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/ # For Hetzner server types see https://www.hetzner.com/cloud -location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location -network_region = "eu-central" # change to `us-east` if location is ash +location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location +network_region = "eu-central" # change to `us-east` if location is ash +network_ip_range = "10.0.0.0/8" +network_subnets = { + control_plane = "10.1.0.0/16" + subnet1 = "10.2.0.0/16" + subnet2 = "10.3.0.0/16" +} + control_plane_server_type = "cpx11" lb_server_type = "lb11" # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe). -servers_num = 3 +servers_num = 3 -# For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades. -agents_num = 2 agent_nodepools = { big = { server_type = "cpx31", - count = 1 + count = 1, + subnet = "subnet1", } small = { server_type = "cpx21", + count = 2, + subnet = "subnet2", } } diff --git a/variables.tf b/variables.tf index 0b1d8f5..c83a294 100644 --- a/variables.tf +++ b/variables.tf @@ -30,6 +30,16 @@ variable "network_region" { type = string } +variable "network_ip_range" { + description = "Default IP range for network" + type = string +} + +variable "network_subnets" { + description = "Subnets definition for default network" + type = map(string) +} + variable "control_plane_server_type" { description = "Default control plane server type" type = string @@ -52,11 +62,6 @@ variable "servers_num" { type = number } -variable "agents_num" { - description = "Number of agent nodes." - type = number -} - variable "agent_nodepools" { description = "Number of agent nodes." type = map(any) From 38f6ef83ce9f99dd5147efd5a70e6c556870154e Mon Sep 17 00:00:00 2001 From: jodhi Date: Sat, 26 Feb 2022 02:36:20 +0700 Subject: [PATCH 7/7] fix missing reference --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index d86fea1..d135788 100644 --- a/main.tf +++ b/main.tf @@ -178,7 +178,7 @@ resource "null_resource" "destroy_traefik_loadbalancer" { depends_on = [ local_file.kubeconfig, null_resource.control_planes[0], - hcloud_network_subnet.k3s, + hcloud_network_subnet.subnet, hcloud_network.k3s, hcloud_firewall.k3s, hcloud_placement_group.k3s,