diff --git a/agents.tf b/agents.tf index 7325773..da4c4f5 100644 --- a/agents.tf +++ b/agents.tf @@ -1,9 +1,9 @@ module "agents" { source = "./modules/host" - count = var.agents_num - name = "k3s-agent-${count.index}" + for_each = local.agent_nodepools + name = each.key ssh_keys = [hcloud_ssh_key.k3s.id] public_key = var.public_key private_key = var.private_key @@ -11,41 +11,44 @@ module "agents" { firewall_ids = [hcloud_firewall.k3s.id] placement_group_id = hcloud_placement_group.k3s.id location = var.location - network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index) - server_type = var.agent_server_type - + server_type = each.value.server_type + ipv4_subnet_id = hcloud_network_subnet.subnet[each.value.subnet].id + private_ipv4 = cidrhost(var.network_ipv4_subnets[each.value.subnet], each.value.index + 1) labels = { "provisioner" = "terraform", "engine" = "k3s" } hcloud_token = var.hcloud_token + + depends_on = [ + hcloud_network_subnet.subnet + ] } resource "null_resource" "agents" { - count = var.agents_num + for_each = local.agent_nodepools triggers = { - agent_id = module.agents[count.index].id + agent_id = module.agents[each.key].id } connection { user = "root" private_key = local.ssh_private_key agent_identity = local.ssh_identity - host = module.agents[count.index].ipv4_address + host = module.agents[each.key].ipv4_address } # Generating k3s agent config file provisioner "file" { content = yamlencode({ - node-name = module.agents[count.index].name - server = "https://${local.first_control_plane_network_ip}:6443" + node-name = module.agents[each.key].name + server = "https://${local.first_control_plane_network_ipv4}:6443" token = random_password.k3s_token.result kubelet-arg = "cloud-provider=external" flannel-iface = "eth1" - node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index) + node-ip = module.agents[each.key].ipv4_address node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] }) destination = "/tmp/config.yaml" @@ -74,6 +77,6 @@ resource "null_resource" "agents" { depends_on = [ null_resource.first_control_plane, - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet ] } diff --git a/servers.tf b/control_planes.tf similarity index 88% rename from servers.tf rename to control_planes.tf index a10c7d7..1f7c239 100644 --- a/servers.tf +++ b/control_planes.tf @@ -1,7 +1,7 @@ module "control_planes" { source = "./modules/host" - count = var.servers_num + count = var.control_plane_count name = "k3s-control-plane-${count.index}" ssh_keys = [hcloud_ssh_key.k3s.id] @@ -11,9 +11,9 @@ module "control_planes" { firewall_ids = [hcloud_firewall.k3s.id] placement_group_id = hcloud_placement_group.k3s.id location = var.location - network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index) server_type = var.control_plane_server_type + ipv4_subnet_id = hcloud_network_subnet.subnet["control_plane"].id + private_ipv4 = cidrhost(var.network_ipv4_subnets["control_plane"], count.index + 1) labels = { "provisioner" = "terraform", @@ -21,10 +21,14 @@ module "control_planes" { } hcloud_token = var.hcloud_token + + depends_on = [ + hcloud_network_subnet.subnet + ] } resource "null_resource" "control_planes" { - count = var.servers_num + count = var.control_plane_count triggers = { control_plane_id = module.control_planes[count.index].id @@ -79,6 +83,6 @@ resource "null_resource" "control_planes" { depends_on = [ null_resource.first_control_plane, - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet ] } diff --git a/data.tf b/data.tf index 41d1cff..6b2c552 100644 --- a/data.tf +++ b/data.tf @@ -15,4 +15,4 @@ data "github_release" "kured" { repository = "kured" owner = "weaveworks" retrieve_by = "latest" -} \ No newline at end of file +} diff --git a/init.tf b/init.tf index f2029be..7f7e021 100644 --- a/init.tf +++ b/init.tf @@ -58,7 +58,7 @@ resource "null_resource" "first_control_plane" { } depends_on = [ - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet["control_plane"] ] } @@ -96,11 +96,11 @@ resource "null_resource" "kustomization" { content = templatefile( "${path.module}/templates/traefik_config.yaml.tpl", { - lb_disable_ipv6 = var.lb_disable_ipv6 - lb_server_type = var.lb_server_type - location = var.location - traefik_acme_tls = var.traefik_acme_tls - traefik_acme_email = var.traefik_acme_email + load_balancer_disable_ipv6 = var.load_balancer_disable_ipv6 + load_balancer_type = var.load_balancer_type + location = var.location + traefik_acme_tls = var.traefik_acme_tls + traefik_acme_email = var.traefik_acme_email }) destination = "/tmp/post_install/traefik.yaml" } diff --git a/locals.tf b/locals.tf index 7cc21cb..08d3fd7 100644 --- a/locals.tf +++ b/locals.tf @@ -1,5 +1,5 @@ locals { - first_control_plane_network_ip = module.control_planes[0].private_ipv4_address + first_control_plane_network_ipv4 = module.control_planes[0].private_ipv4_address ssh_public_key = trimspace(file(var.public_key)) # ssh_private_key is either the contents of var.private_key or null to use a ssh agent. @@ -30,4 +30,15 @@ locals { install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"]) install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"]) + + agent_nodepools = merge([ + for nodepool_name, nodepool_obj in var.agent_nodepools : { + for index in range(nodepool_obj.count) : + format("%s-%s", nodepool_name, index) => { + server_type : nodepool_obj.server_type, + subnet : lookup(nodepool_obj, "subnet", "default"), + index : index + } + } + ]...) } diff --git a/main.tf b/main.tf index e66cedf..190c5ce 100644 --- a/main.tf +++ b/main.tf @@ -10,14 +10,15 @@ resource "hcloud_ssh_key" "k3s" { resource "hcloud_network" "k3s" { name = "k3s" - ip_range = "10.0.0.0/8" + ip_range = var.network_ipv4_range } -resource "hcloud_network_subnet" "k3s" { +resource "hcloud_network_subnet" "subnet" { + for_each = var.network_ipv4_subnets network_id = hcloud_network.k3s.id type = "cloud" network_zone = var.network_region - ip_range = "10.0.0.0/16" + ip_range = each.value } resource "hcloud_firewall" "k3s" { @@ -29,8 +30,8 @@ resource "hcloud_firewall" "k3s" { protocol = "tcp" port = "any" source_ips = [ + var.network_ipv4_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] @@ -40,8 +41,8 @@ resource "hcloud_firewall" "k3s" { protocol = "udp" port = "any" source_ips = [ + var.network_ipv4_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] @@ -50,8 +51,8 @@ resource "hcloud_firewall" "k3s" { direction = "in" protocol = "icmp" source_ips = [ + var.network_ipv4_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] @@ -177,7 +178,7 @@ resource "null_resource" "destroy_traefik_loadbalancer" { depends_on = [ local_file.kubeconfig, null_resource.control_planes[0], - hcloud_network_subnet.k3s, + hcloud_network_subnet.subnet, hcloud_network.k3s, hcloud_firewall.k3s, hcloud_placement_group.k3s, diff --git a/modules/host/main.tf b/modules/host/main.tf index b904279..beb8e99 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -12,11 +12,6 @@ resource "hcloud_server" "server" { labels = var.labels - network { - network_id = var.network_id - ip = var.ip - } - connection { user = "root" private_key = local.ssh_private_key @@ -67,3 +62,9 @@ resource "hcloud_server" "server" { ] } } + +resource "hcloud_server_network" "server" { + ip = var.private_ipv4 + server_id = hcloud_server.server.id + subnet_id = var.ipv4_subnet_id +} diff --git a/modules/host/out.tf b/modules/host/out.tf index d2997ba..1c373c1 100644 --- a/modules/host/out.tf +++ b/modules/host/out.tf @@ -3,7 +3,7 @@ output "ipv4_address" { } output "private_ipv4_address" { - value = var.ip + value = hcloud_server_network.server.ip } output "name" { diff --git a/modules/host/variables.tf b/modules/host/variables.tf index b336fc5..01abaa6 100644 --- a/modules/host/variables.tf +++ b/modules/host/variables.tf @@ -54,15 +54,14 @@ variable "location" { type = string } -variable "network_id" { - description = "The network or subnet id" - type = number +variable "ipv4_subnet_id" { + description = "The subnet id" + type = string } -variable "ip" { - description = "The IP" +variable "private_ipv4" { + description = "Private IP for the server" type = string - nullable = true } variable "server_type" { diff --git a/output.tf b/output.tf index 62e6c6f..762290d 100644 --- a/output.tf +++ b/output.tf @@ -1,14 +1,16 @@ -output "controlplanes_public_ip" { +output "control_planes_public_ipv4" { value = module.control_planes.*.ipv4_address - description = "The public IP addresses of the controlplane server." + description = "The public IPv4 addresses of the controlplane server." } -output "agents_public_ip" { - value = module.agents.*.ipv4_address - description = "The public IP addresses of the agent server." +output "agents_public_ipv4" { + value = [ + for obj in module.agents : obj.ipv4_address + ] + description = "The public IPv4 addresses of the agent server." } -output "load_balancer_public_ip" { +output "load_balancer_public_ipv4" { description = "The public IPv4 address of the Hetzner load balancer" value = data.hcloud_load_balancer.traefik.ipv4 } diff --git a/templates/traefik_config.yaml.tpl b/templates/traefik_config.yaml.tpl index f8156f8..75ce20f 100644 --- a/templates/traefik_config.yaml.tpl +++ b/templates/traefik_config.yaml.tpl @@ -15,9 +15,9 @@ spec: # keep hetzner-ccm from exposing our private ingress ip, which in general isn't routeable from the public internet "load-balancer.hetzner.cloud/disable-private-ingress": "true" # disable ipv6 by default, because external-dns doesn't support AAAA for hcloud yet https://github.com/kubernetes-sigs/external-dns/issues/2044 - "load-balancer.hetzner.cloud/ipv6-disabled": "${lb_disable_ipv6}" + "load-balancer.hetzner.cloud/ipv6-disabled": "${load_balancer_disable_ipv6}" "load-balancer.hetzner.cloud/location": "${location}" - "load-balancer.hetzner.cloud/type": "${lb_server_type}" + "load-balancer.hetzner.cloud/type": "${load_balancer_type}" "load-balancer.hetzner.cloud/uses-proxyprotocol": "true" additionalArguments: - "--entryPoints.web.proxyProtocol.trustedIPs=127.0.0.1/32,10.0.0.0/8" diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 89c9f26..949ed81 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -7,17 +7,34 @@ private_key = "/home/username/.ssh/id_ed25519" # These can be customized, or left with the default values # For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/ # For Hetzner server types see https://www.hetzner.com/cloud -location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location -network_region = "eu-central" # change to `us-east` if location is ash -agent_server_type = "cpx21" +location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location +network_region = "eu-central" # change to `us-east` if location is ash +network_ipv4_range = "10.0.0.0/8" +network_ipv4_subnets = { + control_plane = "10.1.0.0/16" + subnet1 = "10.2.0.0/16" + subnet2 = "10.3.0.0/16" +} + control_plane_server_type = "cpx11" -lb_server_type = "lb11" +load_balancer_type = "lb11" # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe). -servers_num = 3 +control_plane_count = 3 -# For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades. -agents_num = 2 + +agent_nodepools = { + big = { + server_type = "cpx31", + count = 1, + subnet = "subnet1", + } + small = { + server_type = "cpx21", + count = 2, + subnet = "subnet2", + } +} # If you want to use a specific Hetzner CCM and CSI version, set them below, otherwise leave as is for the latest versions # hetzner_ccm_version = "" diff --git a/variables.tf b/variables.tf index 7f0e2f3..ab836e0 100644 --- a/variables.tf +++ b/variables.tf @@ -30,35 +30,40 @@ variable "network_region" { type = string } +variable "network_ipv4_range" { + description = "Default IPv4 range for network" + type = string +} + +variable "network_ipv4_subnets" { + description = "Subnets definition for default network" + type = map(string) +} + variable "control_plane_server_type" { description = "Default control plane server type" type = string } -variable "agent_server_type" { - description = "Default agent server type" - type = string +variable "control_plane_count" { + description = "Number of control plane nodes." + type = number } -variable "lb_server_type" { +variable "load_balancer_type" { description = "Default load balancer server type" type = string } -variable "lb_disable_ipv6" { +variable "load_balancer_disable_ipv6" { description = "Disable ipv6 for the load balancer" type = bool default = false } -variable "servers_num" { - description = "Number of control plane nodes." - type = number -} - -variable "agents_num" { +variable "agent_nodepools" { description = "Number of agent nodes." - type = number + type = map(any) } variable "hetzner_ccm_version" {