diff --git a/agents.tf b/agents.tf index 7325773..8c3f6c9 100644 --- a/agents.tf +++ b/agents.tf @@ -1,9 +1,9 @@ module "agents" { source = "./modules/host" - count = var.agents_num - name = "k3s-agent-${count.index}" + for_each = local.agent_nodepools + name = each.key ssh_keys = [hcloud_ssh_key.k3s.id] public_key = var.public_key private_key = var.private_key @@ -11,41 +11,44 @@ module "agents" { firewall_ids = [hcloud_firewall.k3s.id] placement_group_id = hcloud_placement_group.k3s.id location = var.location - network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index) - server_type = var.agent_server_type - + server_type = each.value.server_type + subnet_id = hcloud_network_subnet.subnet[each.value.subnet].id + private_ip = cidrhost(var.network_subnets[each.value.subnet], each.value.index + 1) labels = { "provisioner" = "terraform", "engine" = "k3s" } hcloud_token = var.hcloud_token + + depends_on = [ + hcloud_network_subnet.subnet + ] } resource "null_resource" "agents" { - count = var.agents_num + for_each = local.agent_nodepools triggers = { - agent_id = module.agents[count.index].id + agent_id = module.agents[each.key].id } connection { user = "root" private_key = local.ssh_private_key agent_identity = local.ssh_identity - host = module.agents[count.index].ipv4_address + host = module.agents[each.key].ipv4_address } # Generating k3s agent config file provisioner "file" { content = yamlencode({ - node-name = module.agents[count.index].name + node-name = module.agents[each.key].name server = "https://${local.first_control_plane_network_ip}:6443" token = random_password.k3s_token.result kubelet-arg = "cloud-provider=external" flannel-iface = "eth1" - node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index) + node-ip = module.agents[each.key].ipv4_address node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] }) destination = "/tmp/config.yaml" @@ -74,6 +77,6 @@ resource "null_resource" "agents" { depends_on = [ null_resource.first_control_plane, - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet ] } diff --git a/data.tf b/data.tf index 41d1cff..6b2c552 100644 --- a/data.tf +++ b/data.tf @@ -15,4 +15,4 @@ data "github_release" "kured" { repository = "kured" owner = "weaveworks" retrieve_by = "latest" -} \ No newline at end of file +} diff --git a/init.tf b/init.tf index f2029be..0812470 100644 --- a/init.tf +++ b/init.tf @@ -58,7 +58,7 @@ resource "null_resource" "first_control_plane" { } depends_on = [ - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet["control_plane"] ] } diff --git a/locals.tf b/locals.tf index 7cc21cb..5141bc0 100644 --- a/locals.tf +++ b/locals.tf @@ -30,4 +30,15 @@ locals { install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"]) install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"]) + + agent_nodepools = merge([ + for nodepool_name, nodepool_obj in var.agent_nodepools : { + for index in range(nodepool_obj.count) : + format("%s-%s", nodepool_name, index) => { + server_type : nodepool_obj.server_type, + subnet : lookup(nodepool_obj, "subnet", "default"), + index : index + } + } + ]...) } diff --git a/main.tf b/main.tf index e66cedf..d135788 100644 --- a/main.tf +++ b/main.tf @@ -10,14 +10,15 @@ resource "hcloud_ssh_key" "k3s" { resource "hcloud_network" "k3s" { name = "k3s" - ip_range = "10.0.0.0/8" + ip_range = var.network_ip_range } -resource "hcloud_network_subnet" "k3s" { +resource "hcloud_network_subnet" "subnet" { + for_each = var.network_subnets network_id = hcloud_network.k3s.id type = "cloud" network_zone = var.network_region - ip_range = "10.0.0.0/16" + ip_range = each.value } resource "hcloud_firewall" "k3s" { @@ -29,8 +30,8 @@ resource "hcloud_firewall" "k3s" { protocol = "tcp" port = "any" source_ips = [ + var.network_ip_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] @@ -40,8 +41,8 @@ resource "hcloud_firewall" "k3s" { protocol = "udp" port = "any" source_ips = [ + var.network_ip_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] @@ -50,8 +51,8 @@ resource "hcloud_firewall" "k3s" { direction = "in" protocol = "icmp" source_ips = [ + var.network_ip_range, "127.0.0.1/32", - "10.0.0.0/8", "169.254.169.254/32", "213.239.246.1/32" ] @@ -177,7 +178,7 @@ resource "null_resource" "destroy_traefik_loadbalancer" { depends_on = [ local_file.kubeconfig, null_resource.control_planes[0], - hcloud_network_subnet.k3s, + hcloud_network_subnet.subnet, hcloud_network.k3s, hcloud_firewall.k3s, hcloud_placement_group.k3s, diff --git a/modules/host/main.tf b/modules/host/main.tf index b904279..fc0ee44 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -12,11 +12,6 @@ resource "hcloud_server" "server" { labels = var.labels - network { - network_id = var.network_id - ip = var.ip - } - connection { user = "root" private_key = local.ssh_private_key @@ -67,3 +62,9 @@ resource "hcloud_server" "server" { ] } } + +resource "hcloud_server_network" "server" { + ip = var.private_ip + server_id = hcloud_server.server.id + subnet_id = var.subnet_id +} diff --git a/modules/host/out.tf b/modules/host/out.tf index d2997ba..1c373c1 100644 --- a/modules/host/out.tf +++ b/modules/host/out.tf @@ -3,7 +3,7 @@ output "ipv4_address" { } output "private_ipv4_address" { - value = var.ip + value = hcloud_server_network.server.ip } output "name" { diff --git a/modules/host/variables.tf b/modules/host/variables.tf index b336fc5..614317f 100644 --- a/modules/host/variables.tf +++ b/modules/host/variables.tf @@ -54,15 +54,14 @@ variable "location" { type = string } -variable "network_id" { - description = "The network or subnet id" - type = number +variable "subnet_id" { + description = "The subnet id" + type = string } -variable "ip" { - description = "The IP" +variable "private_ip" { + description = "Private IP for the server" type = string - nullable = true } variable "server_type" { diff --git a/output.tf b/output.tf index 62e6c6f..6710da1 100644 --- a/output.tf +++ b/output.tf @@ -4,7 +4,9 @@ output "controlplanes_public_ip" { } output "agents_public_ip" { - value = module.agents.*.ipv4_address + value = [ + for obj in module.agents : obj.ipv4_address + ] description = "The public IP addresses of the agent server." } diff --git a/servers.tf b/servers.tf index a10c7d7..0263f15 100644 --- a/servers.tf +++ b/servers.tf @@ -11,9 +11,9 @@ module "control_planes" { firewall_ids = [hcloud_firewall.k3s.id] placement_group_id = hcloud_placement_group.k3s.id location = var.location - network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index) server_type = var.control_plane_server_type + subnet_id = hcloud_network_subnet.subnet["control_plane"].id + private_ip = cidrhost(var.network_subnets["control_plane"], count.index + 1) labels = { "provisioner" = "terraform", @@ -21,6 +21,10 @@ module "control_planes" { } hcloud_token = var.hcloud_token + + depends_on = [ + hcloud_network_subnet.subnet + ] } resource "null_resource" "control_planes" { @@ -79,6 +83,6 @@ resource "null_resource" "control_planes" { depends_on = [ null_resource.first_control_plane, - hcloud_network_subnet.k3s + hcloud_network_subnet.subnet ] } diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 89c9f26..f3942ce 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -7,17 +7,34 @@ private_key = "/home/username/.ssh/id_ed25519" # These can be customized, or left with the default values # For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/ # For Hetzner server types see https://www.hetzner.com/cloud -location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location -network_region = "eu-central" # change to `us-east` if location is ash -agent_server_type = "cpx21" +location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location +network_region = "eu-central" # change to `us-east` if location is ash +network_ip_range = "10.0.0.0/8" +network_subnets = { + control_plane = "10.1.0.0/16" + subnet1 = "10.2.0.0/16" + subnet2 = "10.3.0.0/16" +} + control_plane_server_type = "cpx11" lb_server_type = "lb11" # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe). -servers_num = 3 +servers_num = 3 -# For agent nodes, at least 2 is recommended for HA, but you can keep automatic upgrades. -agents_num = 2 + +agent_nodepools = { + big = { + server_type = "cpx31", + count = 1, + subnet = "subnet1", + } + small = { + server_type = "cpx21", + count = 2, + subnet = "subnet2", + } +} # If you want to use a specific Hetzner CCM and CSI version, set them below, otherwise leave as is for the latest versions # hetzner_ccm_version = "" diff --git a/variables.tf b/variables.tf index 7f0e2f3..24bb8f1 100644 --- a/variables.tf +++ b/variables.tf @@ -30,15 +30,21 @@ variable "network_region" { type = string } +variable "network_ip_range" { + description = "Default IP range for network" + type = string +} + +variable "network_subnets" { + description = "Subnets definition for default network" + type = map(string) +} + variable "control_plane_server_type" { description = "Default control plane server type" type = string } -variable "agent_server_type" { - description = "Default agent server type" - type = string -} variable "lb_server_type" { description = "Default load balancer server type" @@ -56,9 +62,9 @@ variable "servers_num" { type = number } -variable "agents_num" { +variable "agent_nodepools" { description = "Number of agent nodes." - type = number + type = map(any) } variable "hetzner_ccm_version" {