From 7d301f2c6d4cc616a8e7176ed850d536c906508a Mon Sep 17 00:00:00 2001 From: jodhi Date: Wed, 23 Feb 2022 22:46:46 +0700 Subject: [PATCH 01/11] [WIP] nodepools feature --- agents.tf | 18 +++++++++--------- locals.tf | 10 ++++++++++ output.tf | 4 +++- terraform.tfvars.example | 13 +++++++++++-- variables.tf | 11 +++++------ 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/agents.tf b/agents.tf index 7325773..be36867 100644 --- a/agents.tf +++ b/agents.tf @@ -1,9 +1,9 @@ module "agents" { source = "./modules/host" - count = var.agents_num - name = "k3s-agent-${count.index}" + for_each = local.agent_nodepools + name = each.key ssh_keys = [hcloud_ssh_key.k3s.id] public_key = var.public_key private_key = var.private_key @@ -12,8 +12,8 @@ module "agents" { placement_group_id = hcloud_placement_group.k3s.id location = var.location network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index) - server_type = var.agent_server_type + ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + each.value.index) + server_type = each.value.server_type labels = { "provisioner" = "terraform", @@ -24,28 +24,28 @@ module "agents" { } resource "null_resource" "agents" { - count = var.agents_num + for_each = local.agent_nodepools triggers = { - agent_id = module.agents[count.index].id + agent_id = module.agents[each.key].id } connection { user = "root" private_key = local.ssh_private_key agent_identity = local.ssh_identity - host = module.agents[count.index].ipv4_address + host = module.agents[each.key].ipv4_address } # Generating k3s agent config file provisioner "file" { content = yamlencode({ - node-name = module.agents[count.index].name + node-name = module.agents[each.key].name server = "https://${local.first_control_plane_network_ip}:6443" token = random_password.k3s_token.result kubelet-arg = "cloud-provider=external" flannel-iface = "eth1" - node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index) + node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + each.value.index) node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] }) destination = "/tmp/config.yaml" diff --git a/locals.tf b/locals.tf index 72c2e61..69b0bda 100644 --- a/locals.tf +++ b/locals.tf @@ -30,4 +30,14 @@ locals { install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_EXEC=server sh -"]) install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_EXEC=agent sh -"]) + + agent_nodepools = merge([ + for nodepool_name, nodepool_obj in var.agent_nodepools : { + for index in range(nodepool_obj.count) : + format("%s-%s", nodepool_name, index) => { + index : index, # just for the compatibility with previous structure + server_type : nodepool_obj.server_type + } + } + ]...) } diff --git a/output.tf b/output.tf index 62e6c6f..6710da1 100644 --- a/output.tf +++ b/output.tf @@ -4,7 +4,9 @@ output "controlplanes_public_ip" { } output "agents_public_ip" { - value = module.agents.*.ipv4_address + value = [ + for obj in module.agents : obj.ipv4_address + ] description = "The public IP addresses of the agent server." } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 323955c..57a85f5 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -9,7 +9,6 @@ private_key = "/home/username/.ssh/id_ed25519" # For Hetzner server types see https://www.hetzner.com/cloud location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location network_region = "eu-central" # change to `us-east` if location is ash -agent_server_type = "cpx21" control_plane_server_type = "cpx11" lb_server_type = "lb11" @@ -17,7 +16,17 @@ lb_server_type = "lb11" servers_num = 3 # For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades. -agents_num = 2 +agent_nodepools = { + big = { + server_type = "cpx31", + count = 3 + } + # Will fail because of the ip address collision (we use index number) + # small = { + # server_type = "cpx31", + # count = 3 + # } +} # If you want to use a specific Hetzner CCM and CSI version, set them below, otherwise leave as is for the latest versions # hetzner_ccm_version = "" diff --git a/variables.tf b/variables.tf index 9346e5d..ad2e053 100644 --- a/variables.tf +++ b/variables.tf @@ -35,10 +35,6 @@ variable "control_plane_server_type" { type = string } -variable "agent_server_type" { - description = "Default agent server type" - type = string -} variable "lb_server_type" { description = "Default load balancer server type" @@ -56,9 +52,12 @@ variable "servers_num" { type = number } -variable "agents_num" { +variable "agent_nodepools" { description = "Number of agent nodes." - type = number + type = map(object({ + server_type = string + count = number + })) } variable "hetzner_ccm_version" { From 58c630ba78a2f539e3dd0276e8a1c9efccb3e178 Mon Sep 17 00:00:00 2001 From: jodhi Date: Thu, 24 Feb 2022 22:49:36 +0700 Subject: [PATCH 02/11] not hardcode the ip allocation for agents --- agents.tf | 3 +-- locals.tf | 2 +- modules/host/main.tf | 2 +- modules/host/variables.tf | 1 + variables.tf | 10 ++++++---- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/agents.tf b/agents.tf index be36867..7647cc0 100644 --- a/agents.tf +++ b/agents.tf @@ -12,7 +12,6 @@ module "agents" { placement_group_id = hcloud_placement_group.k3s.id location = var.location network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + each.value.index) server_type = each.value.server_type labels = { @@ -45,7 +44,7 @@ resource "null_resource" "agents" { token = random_password.k3s_token.result kubelet-arg = "cloud-provider=external" flannel-iface = "eth1" - node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + each.value.index) + node-ip = module.agents[each.key].ipv4_address node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] }) destination = "/tmp/config.yaml" diff --git a/locals.tf b/locals.tf index 69b0bda..7932296 100644 --- a/locals.tf +++ b/locals.tf @@ -33,7 +33,7 @@ locals { agent_nodepools = merge([ for nodepool_name, nodepool_obj in var.agent_nodepools : { - for index in range(nodepool_obj.count) : + for index in range(lookup(nodepool_obj, "count", var.agents_num)) : format("%s-%s", nodepool_name, index) => { index : index, # just for the compatibility with previous structure server_type : nodepool_obj.server_type diff --git a/modules/host/main.tf b/modules/host/main.tf index 0aa14cb..56e794e 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -14,7 +14,7 @@ resource "hcloud_server" "server" { network { network_id = var.network_id - ip = var.ip + ip = try(var.ip, null) } connection { diff --git a/modules/host/variables.tf b/modules/host/variables.tf index b336fc5..9c328b1 100644 --- a/modules/host/variables.tf +++ b/modules/host/variables.tf @@ -60,6 +60,7 @@ variable "network_id" { } variable "ip" { + default = null description = "The IP" type = string nullable = true diff --git a/variables.tf b/variables.tf index ad2e053..73d7301 100644 --- a/variables.tf +++ b/variables.tf @@ -52,12 +52,14 @@ variable "servers_num" { type = number } +variable "agents_num" { + description = "Default agent server type" + type = number +} + variable "agent_nodepools" { description = "Number of agent nodes." - type = map(object({ - server_type = string - count = number - })) + type = map(any) } variable "hetzner_ccm_version" { From bb5ab6bd9a7e8ce68a47b45ccf0d4213b06784c6 Mon Sep 17 00:00:00 2001 From: jodhi Date: Thu, 24 Feb 2022 23:19:31 +0700 Subject: [PATCH 03/11] make sure control-plane ip dynamic --- servers.tf | 1 - terraform.tfvars.example | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/servers.tf b/servers.tf index a10c7d7..acfc3f2 100644 --- a/servers.tf +++ b/servers.tf @@ -12,7 +12,6 @@ module "control_planes" { placement_group_id = hcloud_placement_group.k3s.id location = var.location network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index) server_type = var.control_plane_server_type labels = { diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 57a85f5..93945b2 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -16,16 +16,16 @@ lb_server_type = "lb11" servers_num = 3 # For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades. +agents_num = 2 + agent_nodepools = { big = { server_type = "cpx31", - count = 3 + count = 1 + } + small = { + server_type = "cpx21", } - # Will fail because of the ip address collision (we use index number) - # small = { - # server_type = "cpx31", - # count = 3 - # } } # If you want to use a specific Hetzner CCM and CSI version, set them below, otherwise leave as is for the latest versions From 519cae04d33a0586c10d86cda1c5438f7f223703 Mon Sep 17 00:00:00 2001 From: jodhi Date: Thu, 24 Feb 2022 23:25:13 +0700 Subject: [PATCH 04/11] fix typo --- terraform.tfvars.example | 2 +- variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 93945b2..481de9f 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -16,7 +16,7 @@ lb_server_type = "lb11" servers_num = 3 # For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades. -agents_num = 2 +agents_num = 2 agent_nodepools = { big = { diff --git a/variables.tf b/variables.tf index 73d7301..0b1d8f5 100644 --- a/variables.tf +++ b/variables.tf @@ -53,7 +53,7 @@ variable "servers_num" { } variable "agents_num" { - description = "Default agent server type" + description = "Number of agent nodes." type = number } From b5039e2bc1705c3b58448dde6b8b164288d02947 Mon Sep 17 00:00:00 2001 From: jodhi Date: Thu, 24 Feb 2022 23:57:36 +0700 Subject: [PATCH 05/11] fix dynamic private ip --- locals.tf | 1 - modules/host/main.tf | 10 +++++----- modules/host/out.tf | 2 +- modules/host/variables.tf | 7 ------- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/locals.tf b/locals.tf index 7932296..3c78ad0 100644 --- a/locals.tf +++ b/locals.tf @@ -35,7 +35,6 @@ locals { for nodepool_name, nodepool_obj in var.agent_nodepools : { for index in range(lookup(nodepool_obj, "count", var.agents_num)) : format("%s-%s", nodepool_name, index) => { - index : index, # just for the compatibility with previous structure server_type : nodepool_obj.server_type } } diff --git a/modules/host/main.tf b/modules/host/main.tf index 56e794e..b8b8dd9 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -12,11 +12,6 @@ resource "hcloud_server" "server" { labels = var.labels - network { - network_id = var.network_id - ip = try(var.ip, null) - } - connection { user = "root" private_key = local.ssh_private_key @@ -68,3 +63,8 @@ resource "hcloud_server" "server" { ] } } + +resource "hcloud_server_network" "server" { + server_id = hcloud_server.server.id + network_id = var.network_id +} diff --git a/modules/host/out.tf b/modules/host/out.tf index d2997ba..1c373c1 100644 --- a/modules/host/out.tf +++ b/modules/host/out.tf @@ -3,7 +3,7 @@ output "ipv4_address" { } output "private_ipv4_address" { - value = var.ip + value = hcloud_server_network.server.ip } output "name" { diff --git a/modules/host/variables.tf b/modules/host/variables.tf index 9c328b1..77a8176 100644 --- a/modules/host/variables.tf +++ b/modules/host/variables.tf @@ -59,13 +59,6 @@ variable "network_id" { type = number } -variable "ip" { - default = null - description = "The IP" - type = string - nullable = true -} - variable "server_type" { description = "The server type" type = string From c9101824fca04d4a898c6bfa2d742f3351f1295c Mon Sep 17 00:00:00 2001 From: jodhi Date: Sat, 26 Feb 2022 01:16:38 +0700 Subject: [PATCH 06/11] add specific subnet for nodepool --- agents.tf | 10 +++++++--- data.tf | 2 +- init.tf | 2 +- locals.tf | 6 ++++-- main.tf | 13 +++++++------ modules/host/main.tf | 5 +++-- modules/host/variables.tf | 11 ++++++++--- servers.tf | 9 +++++++-- terraform.tfvars.example | 20 ++++++++++++++------ variables.tf | 15 ++++++++++----- 10 files changed, 62 insertions(+), 31 deletions(-) diff --git a/agents.tf b/agents.tf index 7647cc0..8c3f6c9 100644 --- a/agents.tf +++ b/agents.tf @@ -11,15 +11,19 @@ module "agents" { firewall_ids = [hcloud_firewall.k3s.id] placement_group_id = hcloud_placement_group.k3s.id location = var.location - network_id = hcloud_network.k3s.id server_type = each.value.server_type - + subnet_id = hcloud_network_subnet.subnet[each.value.subnet].id + private_ip = cidrhost(var.network_subnets[each.value.subnet], each.value.index + 1) labels = { "provisioner" = "terraform", "engine" = "k3s" } hcloud_token = var.hcloud_token + + depends_on = [ + hcloud_network_subnet.subnet + ] } resource "null_resource" "agents" { @@ -73,6 +77,6 @@ resource "null_resource" "agents" { depends_on = [ null_resource.first_control_plane, - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet ] } diff --git a/data.tf b/data.tf index 41d1cff..6b2c552 100644 --- a/data.tf +++ b/data.tf @@ -15,4 +15,4 @@ data "github_release" "kured" { repository = "kured" owner = "weaveworks" retrieve_by = "latest" -} \ No newline at end of file +} diff --git a/init.tf b/init.tf index cb3eec9..4f1d350 100644 --- a/init.tf +++ b/init.tf @@ -58,7 +58,7 @@ resource "null_resource" "first_control_plane" { } depends_on = [ - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet["control_plane"] ] } diff --git a/locals.tf b/locals.tf index 3c78ad0..785c1c3 100644 --- a/locals.tf +++ b/locals.tf @@ -33,9 +33,11 @@ locals { agent_nodepools = merge([ for nodepool_name, nodepool_obj in var.agent_nodepools : { - for index in range(lookup(nodepool_obj, "count", var.agents_num)) : + for index in range(nodepool_obj.count) : format("%s-%s", nodepool_name, index) => { - server_type : nodepool_obj.server_type + server_type : nodepool_obj.server_type, + subnet : lookup(nodepool_obj, "subnet", "default"), + index : index } } ]...) diff --git a/main.tf b/main.tf index 3e71a22..34ba1bd 100644 --- a/main.tf +++ b/main.tf @@ -10,14 +10,15 @@ resource "hcloud_ssh_key" "k3s" { resource "hcloud_network" "k3s" { name = "k3s" - ip_range = "10.0.0.0/8" + ip_range = var.network_ip_range } -resource "hcloud_network_subnet" "k3s" { +resource "hcloud_network_subnet" "subnet" { + for_each = var.network_subnets network_id = hcloud_network.k3s.id type = "cloud" network_zone = var.network_region - ip_range = "10.0.0.0/16" + ip_range = each.value } resource "hcloud_firewall" "k3s" { @@ -29,8 +30,8 @@ resource "hcloud_firewall" "k3s" { protocol = "tcp" port = "any" source_ips = [ + var.network_ip_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] @@ -40,8 +41,8 @@ resource "hcloud_firewall" "k3s" { protocol = "udp" port = "any" source_ips = [ + var.network_ip_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] @@ -50,8 +51,8 @@ resource "hcloud_firewall" "k3s" { direction = "in" protocol = "icmp" source_ips = [ + var.network_ip_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] diff --git a/modules/host/main.tf b/modules/host/main.tf index b8b8dd9..9ce3e30 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -65,6 +65,7 @@ resource "hcloud_server" "server" { } resource "hcloud_server_network" "server" { - server_id = hcloud_server.server.id - network_id = var.network_id + ip = var.private_ip + server_id = hcloud_server.server.id + subnet_id = var.subnet_id } diff --git a/modules/host/variables.tf b/modules/host/variables.tf index 77a8176..614317f 100644 --- a/modules/host/variables.tf +++ b/modules/host/variables.tf @@ -54,9 +54,14 @@ variable "location" { type = string } -variable "network_id" { - description = "The network or subnet id" - type = number +variable "subnet_id" { + description = "The subnet id" + type = string +} + +variable "private_ip" { + description = "Private IP for the server" + type = string } variable "server_type" { diff --git a/servers.tf b/servers.tf index acfc3f2..0263f15 100644 --- a/servers.tf +++ b/servers.tf @@ -11,8 +11,9 @@ module "control_planes" { firewall_ids = [hcloud_firewall.k3s.id] placement_group_id = hcloud_placement_group.k3s.id location = var.location - network_id = hcloud_network.k3s.id server_type = var.control_plane_server_type + subnet_id = hcloud_network_subnet.subnet["control_plane"].id + private_ip = cidrhost(var.network_subnets["control_plane"], count.index + 1) labels = { "provisioner" = "terraform", @@ -20,6 +21,10 @@ module "control_planes" { } hcloud_token = var.hcloud_token + + depends_on = [ + hcloud_network_subnet.subnet + ] } resource "null_resource" "control_planes" { @@ -78,6 +83,6 @@ resource "null_resource" "control_planes" { depends_on = [ null_resource.first_control_plane, - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet ] } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 481de9f..f97712b 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -7,24 +7,32 @@ private_key = "/home/username/.ssh/id_ed25519" # These can be customized, or left with the default values # For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/ # For Hetzner server types see https://www.hetzner.com/cloud -location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location -network_region = "eu-central" # change to `us-east` if location is ash +location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location +network_region = "eu-central" # change to `us-east` if location is ash +network_ip_range = "10.0.0.0/8" +network_subnets = { + control_plane = "10.1.0.0/16" + subnet1 = "10.2.0.0/16" + subnet2 = "10.3.0.0/16" +} + control_plane_server_type = "cpx11" lb_server_type = "lb11" # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe). -servers_num = 3 +servers_num = 3 -# For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades. -agents_num = 2 agent_nodepools = { big = { server_type = "cpx31", - count = 1 + count = 1, + subnet = "subnet1", } small = { server_type = "cpx21", + count = 2, + subnet = "subnet2", } } diff --git a/variables.tf b/variables.tf index 0b1d8f5..c83a294 100644 --- a/variables.tf +++ b/variables.tf @@ -30,6 +30,16 @@ variable "network_region" { type = string } +variable "network_ip_range" { + description = "Default IP range for network" + type = string +} + +variable "network_subnets" { + description = "Subnets definition for default network" + type = map(string) +} + variable "control_plane_server_type" { description = "Default control plane server type" type = string @@ -52,11 +62,6 @@ variable "servers_num" { type = number } -variable "agents_num" { - description = "Number of agent nodes." - type = number -} - variable "agent_nodepools" { description = "Number of agent nodes." type = map(any) From 38f6ef83ce9f99dd5147efd5a70e6c556870154e Mon Sep 17 00:00:00 2001 From: jodhi Date: Sat, 26 Feb 2022 02:36:20 +0700 Subject: [PATCH 07/11] fix missing reference --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index d86fea1..d135788 100644 --- a/main.tf +++ b/main.tf @@ -178,7 +178,7 @@ resource "null_resource" "destroy_traefik_loadbalancer" { depends_on = [ local_file.kubeconfig, null_resource.control_planes[0], - hcloud_network_subnet.k3s, + hcloud_network_subnet.subnet, hcloud_network.k3s, hcloud_firewall.k3s, hcloud_placement_group.k3s, From 22606926a14bb6614465d542d65c3f88e1a00961 Mon Sep 17 00:00:00 2001 From: phaer Date: Sat, 26 Feb 2022 12:04:45 +0100 Subject: [PATCH 08/11] servers_num -> control_plane_count --- README.md | 2 +- servers.tf | 4 ++-- terraform.tfvars.example | 2 +- variables.tf | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f5d0301..81cc5c7 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ To scale the number of nodes up or down, just make sure to properly `kubectl dra For instance: ```tfvars -servers_num = 4 +control_plane_count = 4 agents_num = 3 ``` diff --git a/servers.tf b/servers.tf index 0263f15..9f26677 100644 --- a/servers.tf +++ b/servers.tf @@ -1,7 +1,7 @@ module "control_planes" { source = "./modules/host" - count = var.servers_num + count = var.control_plane_count name = "k3s-control-plane-${count.index}" ssh_keys = [hcloud_ssh_key.k3s.id] @@ -28,7 +28,7 @@ module "control_planes" { } resource "null_resource" "control_planes" { - count = var.servers_num + count = var.control_plane_count triggers = { control_plane_id = module.control_planes[count.index].id diff --git a/terraform.tfvars.example b/terraform.tfvars.example index f3942ce..5ae9930 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -20,7 +20,7 @@ control_plane_server_type = "cpx11" lb_server_type = "lb11" # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe). -servers_num = 3 +control_plane_count = 3 agent_nodepools = { diff --git a/variables.tf b/variables.tf index 24bb8f1..a0ed062 100644 --- a/variables.tf +++ b/variables.tf @@ -45,6 +45,11 @@ variable "control_plane_server_type" { type = string } +variable "control_plane_count" { + description = "Number of control plane nodes." + type = number +} + variable "lb_server_type" { description = "Default load balancer server type" @@ -57,11 +62,6 @@ variable "lb_disable_ipv6" { default = false } -variable "servers_num" { - description = "Number of control plane nodes." - type = number -} - variable "agent_nodepools" { description = "Number of agent nodes." type = map(any) From 2045f4293bab38e0fa1ecd521c6445b215977183 Mon Sep 17 00:00:00 2001 From: phaer Date: Sat, 26 Feb 2022 12:07:42 +0100 Subject: [PATCH 09/11] servers.tf -> control_planes.tf --- servers.tf => control_planes.tf | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename servers.tf => control_planes.tf (100%) diff --git a/servers.tf b/control_planes.tf similarity index 100% rename from servers.tf rename to control_planes.tf From 52b3d9b82c514f363b8def5bd16b5b2d255e3faf Mon Sep 17 00:00:00 2001 From: phaer Date: Sat, 26 Feb 2022 12:10:50 +0100 Subject: [PATCH 10/11] lb -> load_balancer... ...to keep variables and output naming consistent --- init.tf | 4 ++-- templates/traefik_config.yaml.tpl | 4 ++-- terraform.tfvars.example | 2 +- variables.tf | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/init.tf b/init.tf index 0812470..68bfa1a 100644 --- a/init.tf +++ b/init.tf @@ -96,8 +96,8 @@ resource "null_resource" "kustomization" { content = templatefile( "${path.module}/templates/traefik_config.yaml.tpl", { - lb_disable_ipv6 = var.lb_disable_ipv6 - lb_server_type = var.lb_server_type + load_balancer_disable_ipv6 = var.load_balancer_disable_ipv6 + load_balancer_type = var.load_balancer_type location = var.location traefik_acme_tls = var.traefik_acme_tls traefik_acme_email = var.traefik_acme_email diff --git a/templates/traefik_config.yaml.tpl b/templates/traefik_config.yaml.tpl index f8156f8..75ce20f 100644 --- a/templates/traefik_config.yaml.tpl +++ b/templates/traefik_config.yaml.tpl @@ -15,9 +15,9 @@ spec: # keep hetzner-ccm from exposing our private ingress ip, which in general isn't routeable from the public internet "load-balancer.hetzner.cloud/disable-private-ingress": "true" # disable ipv6 by default, because external-dns doesn't support AAAA for hcloud yet https://github.com/kubernetes-sigs/external-dns/issues/2044 - "load-balancer.hetzner.cloud/ipv6-disabled": "${lb_disable_ipv6}" + "load-balancer.hetzner.cloud/ipv6-disabled": "${load_balancer_disable_ipv6}" "load-balancer.hetzner.cloud/location": "${location}" - "load-balancer.hetzner.cloud/type": "${lb_server_type}" + "load-balancer.hetzner.cloud/type": "${load_balancer_type}" "load-balancer.hetzner.cloud/uses-proxyprotocol": "true" additionalArguments: - "--entryPoints.web.proxyProtocol.trustedIPs=127.0.0.1/32,10.0.0.0/8" diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 5ae9930..0f7aa6c 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -17,7 +17,7 @@ network_subnets = { } control_plane_server_type = "cpx11" -lb_server_type = "lb11" +load_balancer_type = "lb11" # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe). control_plane_count = 3 diff --git a/variables.tf b/variables.tf index a0ed062..627a04e 100644 --- a/variables.tf +++ b/variables.tf @@ -51,12 +51,12 @@ variable "control_plane_count" { } -variable "lb_server_type" { +variable "load_balancer_type" { description = "Default load balancer server type" type = string } -variable "lb_disable_ipv6" { +variable "load_balancer_disable_ipv6" { description = "Disable ipv6 for the load balancer" type = bool default = false From 38fc145ad0cf13423f8d1103fc8bc3c96e29bc89 Mon Sep 17 00:00:00 2001 From: phaer Date: Sat, 26 Feb 2022 12:26:14 +0100 Subject: [PATCH 11/11] ip -> ipv4... ...explictly label current ip values as ipv4, in order to more easily add ipv6 support later on. --- agents.tf | 6 +++--- control_planes.tf | 4 ++-- init.tf | 10 +++++----- locals.tf | 2 +- main.tf | 10 +++++----- modules/host/main.tf | 4 ++-- modules/host/variables.tf | 4 ++-- output.tf | 10 +++++----- terraform.tfvars.example | 4 ++-- variables.tf | 7 +++---- 10 files changed, 30 insertions(+), 31 deletions(-) diff --git a/agents.tf b/agents.tf index 8c3f6c9..da4c4f5 100644 --- a/agents.tf +++ b/agents.tf @@ -12,8 +12,8 @@ module "agents" { placement_group_id = hcloud_placement_group.k3s.id location = var.location server_type = each.value.server_type - subnet_id = hcloud_network_subnet.subnet[each.value.subnet].id - private_ip = cidrhost(var.network_subnets[each.value.subnet], each.value.index + 1) + ipv4_subnet_id = hcloud_network_subnet.subnet[each.value.subnet].id + private_ipv4 = cidrhost(var.network_ipv4_subnets[each.value.subnet], each.value.index + 1) labels = { "provisioner" = "terraform", "engine" = "k3s" @@ -44,7 +44,7 @@ resource "null_resource" "agents" { provisioner "file" { content = yamlencode({ node-name = module.agents[each.key].name - server = "https://${local.first_control_plane_network_ip}:6443" + server = "https://${local.first_control_plane_network_ipv4}:6443" token = random_password.k3s_token.result kubelet-arg = "cloud-provider=external" flannel-iface = "eth1" diff --git a/control_planes.tf b/control_planes.tf index 9f26677..1f7c239 100644 --- a/control_planes.tf +++ b/control_planes.tf @@ -12,8 +12,8 @@ module "control_planes" { placement_group_id = hcloud_placement_group.k3s.id location = var.location server_type = var.control_plane_server_type - subnet_id = hcloud_network_subnet.subnet["control_plane"].id - private_ip = cidrhost(var.network_subnets["control_plane"], count.index + 1) + ipv4_subnet_id = hcloud_network_subnet.subnet["control_plane"].id + private_ipv4 = cidrhost(var.network_ipv4_subnets["control_plane"], count.index + 1) labels = { "provisioner" = "terraform", diff --git a/init.tf b/init.tf index 68bfa1a..7f7e021 100644 --- a/init.tf +++ b/init.tf @@ -96,11 +96,11 @@ resource "null_resource" "kustomization" { content = templatefile( "${path.module}/templates/traefik_config.yaml.tpl", { - load_balancer_disable_ipv6 = var.load_balancer_disable_ipv6 - load_balancer_type = var.load_balancer_type - location = var.location - traefik_acme_tls = var.traefik_acme_tls - traefik_acme_email = var.traefik_acme_email + load_balancer_disable_ipv6 = var.load_balancer_disable_ipv6 + load_balancer_type = var.load_balancer_type + location = var.location + traefik_acme_tls = var.traefik_acme_tls + traefik_acme_email = var.traefik_acme_email }) destination = "/tmp/post_install/traefik.yaml" } diff --git a/locals.tf b/locals.tf index 5141bc0..08d3fd7 100644 --- a/locals.tf +++ b/locals.tf @@ -1,5 +1,5 @@ locals { - first_control_plane_network_ip = module.control_planes[0].private_ipv4_address + first_control_plane_network_ipv4 = module.control_planes[0].private_ipv4_address ssh_public_key = trimspace(file(var.public_key)) # ssh_private_key is either the contents of var.private_key or null to use a ssh agent. diff --git a/main.tf b/main.tf index d135788..190c5ce 100644 --- a/main.tf +++ b/main.tf @@ -10,11 +10,11 @@ resource "hcloud_ssh_key" "k3s" { resource "hcloud_network" "k3s" { name = "k3s" - ip_range = var.network_ip_range + ip_range = var.network_ipv4_range } resource "hcloud_network_subnet" "subnet" { - for_each = var.network_subnets + for_each = var.network_ipv4_subnets network_id = hcloud_network.k3s.id type = "cloud" network_zone = var.network_region @@ -30,7 +30,7 @@ resource "hcloud_firewall" "k3s" { protocol = "tcp" port = "any" source_ips = [ - var.network_ip_range, + var.network_ipv4_range, "127.0.0.1/32", "169.254.169.254/32", "213.239.246.1/32" @@ -41,7 +41,7 @@ resource "hcloud_firewall" "k3s" { protocol = "udp" port = "any" source_ips = [ - var.network_ip_range, + var.network_ipv4_range, "127.0.0.1/32", "169.254.169.254/32", "213.239.246.1/32" @@ -51,7 +51,7 @@ resource "hcloud_firewall" "k3s" { direction = "in" protocol = "icmp" source_ips = [ - var.network_ip_range, + var.network_ipv4_range, "127.0.0.1/32", "169.254.169.254/32", "213.239.246.1/32" diff --git a/modules/host/main.tf b/modules/host/main.tf index fc0ee44..beb8e99 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -64,7 +64,7 @@ resource "hcloud_server" "server" { } resource "hcloud_server_network" "server" { - ip = var.private_ip + ip = var.private_ipv4 server_id = hcloud_server.server.id - subnet_id = var.subnet_id + subnet_id = var.ipv4_subnet_id } diff --git a/modules/host/variables.tf b/modules/host/variables.tf index 614317f..01abaa6 100644 --- a/modules/host/variables.tf +++ b/modules/host/variables.tf @@ -54,12 +54,12 @@ variable "location" { type = string } -variable "subnet_id" { +variable "ipv4_subnet_id" { description = "The subnet id" type = string } -variable "private_ip" { +variable "private_ipv4" { description = "Private IP for the server" type = string } diff --git a/output.tf b/output.tf index 6710da1..762290d 100644 --- a/output.tf +++ b/output.tf @@ -1,16 +1,16 @@ -output "controlplanes_public_ip" { +output "control_planes_public_ipv4" { value = module.control_planes.*.ipv4_address - description = "The public IP addresses of the controlplane server." + description = "The public IPv4 addresses of the controlplane server." } -output "agents_public_ip" { +output "agents_public_ipv4" { value = [ for obj in module.agents : obj.ipv4_address ] - description = "The public IP addresses of the agent server." + description = "The public IPv4 addresses of the agent server." } -output "load_balancer_public_ip" { +output "load_balancer_public_ipv4" { description = "The public IPv4 address of the Hetzner load balancer" value = data.hcloud_load_balancer.traefik.ipv4 } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 0f7aa6c..949ed81 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -9,8 +9,8 @@ private_key = "/home/username/.ssh/id_ed25519" # For Hetzner server types see https://www.hetzner.com/cloud location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location network_region = "eu-central" # change to `us-east` if location is ash -network_ip_range = "10.0.0.0/8" -network_subnets = { +network_ipv4_range = "10.0.0.0/8" +network_ipv4_subnets = { control_plane = "10.1.0.0/16" subnet1 = "10.2.0.0/16" subnet2 = "10.3.0.0/16" diff --git a/variables.tf b/variables.tf index 627a04e..ab836e0 100644 --- a/variables.tf +++ b/variables.tf @@ -30,12 +30,12 @@ variable "network_region" { type = string } -variable "network_ip_range" { - description = "Default IP range for network" +variable "network_ipv4_range" { + description = "Default IPv4 range for network" type = string } -variable "network_subnets" { +variable "network_ipv4_subnets" { description = "Subnets definition for default network" type = map(string) } @@ -50,7 +50,6 @@ variable "control_plane_count" { type = number } - variable "load_balancer_type" { description = "Default load balancer server type" type = string