From 5e4d82fd95d2e15cc361e2cc2195cb99bc07d13c Mon Sep 17 00:00:00 2001 From: Henk van Maanen <36051232+HenkVanMaanen@users.noreply.github.com> Date: Wed, 6 Apr 2022 20:38:24 +0200 Subject: [PATCH] wip --- control_planes.tf | 27 ++++++++++++++------------- init.tf | 12 ++++++------ kubeconfig.tf | 4 ++-- locals.tf | 34 +++++++++++++++++++++++++--------- output.tf | 4 +++- variables.tf | 20 ++++++++------------ 6 files changed, 58 insertions(+), 43 deletions(-) diff --git a/control_planes.tf b/control_planes.tf index d39047f..c24590a 100644 --- a/control_planes.tf +++ b/control_planes.tf @@ -1,7 +1,8 @@ module "control_planes" { source = "./modules/host" - count = var.control_plane_count + for_each = local.control_plane_nodepools + name = "${var.use_cluster_name_in_node_name ? "${var.cluster_name}-" : ""}control-plane" ssh_keys = [hcloud_ssh_key.k3s.id] public_key = var.public_key @@ -9,13 +10,13 @@ module "control_planes" { additional_public_keys = var.additional_public_keys firewall_ids = [hcloud_firewall.k3s.id] placement_group_id = hcloud_placement_group.k3s.id - location = var.location - server_type = var.control_plane_server_type + location = each.value.location + server_type = each.value.server_type ipv4_subnet_id = hcloud_network_subnet.subnet[1].id # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough. - private_ipv4 = cidrhost(local.network_ipv4_subnets[1], count.index + 101) + private_ipv4 = cidrhost(local.network_ipv4_subnets[1], length([for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name]) + each.value.index + 101) labels = { "provisioner" = "terraform", @@ -28,33 +29,33 @@ module "control_planes" { } resource "null_resource" "control_planes" { - count = var.control_plane_count + for_each = local.control_plane_nodepools triggers = { - control_plane_id = module.control_planes[count.index].id + control_plane_id = module.control_planes[each.key].id } connection { user = "root" private_key = local.ssh_private_key agent_identity = local.ssh_identity - host = module.control_planes[count.index].ipv4_address + host = module.control_planes[each.key].ipv4_address } # Generating k3s server config file provisioner "file" { content = yamlencode({ - node-name = module.control_planes[count.index].name - server = "https://${element(module.control_planes.*.private_ipv4_address, count.index > 0 ? 0 : 1)}:6443" + node-name = module.control_planes[each.key].name + server = "https://${local.first_control_plane.private_ipv4_address}:6443" token = random_password.k3s_token.result disable-cloud-controller = true disable = local.disable_extras flannel-iface = "eth1" kubelet-arg = "cloud-provider=external" - node-ip = module.control_planes[count.index].private_ipv4_address - advertise-address = module.control_planes[count.index].private_ipv4_address - node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"] - node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] + node-ip = module.control_planes[each.key].private_ipv4_address + advertise-address = module.control_planes[each.key].private_ipv4_address + node-label = each.value.labels + node-taint = each.value.taints }) destination = "/tmp/config.yaml" } diff --git a/init.tf b/init.tf index 1e8eb7e..884fbe6 100644 --- a/init.tf +++ b/init.tf @@ -3,21 +3,21 @@ resource "null_resource" "first_control_plane" { user = "root" private_key = local.ssh_private_key agent_identity = local.ssh_identity - host = module.control_planes[0].ipv4_address + host = local.first_control_plane.ipv4_address } # Generating k3s master config file provisioner "file" { content = yamlencode({ - node-name = module.control_planes[0].name + node-name = local.first_control_plane.name token = random_password.k3s_token.result cluster-init = true disable-cloud-controller = true disable = local.disable_extras flannel-iface = "eth1" kubelet-arg = "cloud-provider=external" - node-ip = module.control_planes[0].private_ipv4_address - advertise-address = module.control_planes[0].private_ipv4_address + node-ip = local.first_control_plane.private_ipv4_address + advertise-address = local.first_control_plane.private_ipv4_address node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"] node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] }) @@ -66,7 +66,7 @@ resource "null_resource" "kustomization" { user = "root" private_key = local.ssh_private_key agent_identity = local.ssh_identity - host = module.control_planes[0].ipv4_address + host = local.first_control_plane.ipv4_address } # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured. @@ -97,7 +97,7 @@ resource "null_resource" "kustomization" { name = "${var.cluster_name}-traefik" load_balancer_disable_ipv6 = var.load_balancer_disable_ipv6 load_balancer_type = var.load_balancer_type - location = var.location + location = var.load_balancer_location traefik_acme_tls = var.traefik_acme_tls traefik_acme_email = var.traefik_acme_email traefik_additional_options = var.traefik_additional_options diff --git a/kubeconfig.tf b/kubeconfig.tf index 1c5cee5..b4969b5 100644 --- a/kubeconfig.tf +++ b/kubeconfig.tf @@ -1,7 +1,7 @@ data "remote_file" "kubeconfig" { conn { - host = module.control_planes[0].ipv4_address + host = local.first_control_plane.ipv4_address port = 22 user = "root" private_key = local.ssh_private_key @@ -13,7 +13,7 @@ data "remote_file" "kubeconfig" { } locals { - kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", module.control_planes[0].ipv4_address) + kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", local.first_control_plane.ipv4_address) kubeconfig_parsed = yamldecode(local.kubeconfig_external) kubeconfig_data = { host = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"] diff --git a/locals.tf b/locals.tf index 439b1b1..101e9a6 100644 --- a/locals.tf +++ b/locals.tf @@ -1,6 +1,6 @@ locals { # if we are in a single cluster config, we use the default klipper lb instead of Hetzner LB - is_single_node_cluster = var.control_plane_count + sum(concat([for v in var.agent_nodepools : v.count], [0])) == 1 + is_single_node_cluster = sum(concat([for v in var.control_plane_nodepools : v.count], [0])) + sum(concat([for v in var.agent_nodepools : v.count], [0])) == 1 ssh_public_key = trimspace(file(var.public_key)) # ssh_private_key is either the contents of var.private_key or null to use a ssh agent. ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key)) @@ -169,16 +169,30 @@ locals { install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"], local.apply_k3s_selinux) install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"], local.apply_k3s_selinux) - agent_nodepools = merge([ - for nodepool_obj in var.agent_nodepools : { - for index in range(nodepool_obj.count) : - format("%s-%s", nodepool_obj.name, index) => { + control_plane_nodepools = merge([ + for pool_index, nodepool_obj in var.control_plane_nodepools : { + for node_index in range(nodepool_obj.count) : + format("%s-%s-%s", pool_index, node_index, nodepool_obj.name) => { nodepool_name : nodepool_obj.name, server_type : nodepool_obj.server_type, location : nodepool_obj.location, - labels : concat(local.default_labels, nodepool_obj.labels), + labels : concat(local.default_control_plane_labels, nodepool_obj.labels), taints : nodepool_obj.taints, - index : index + index : node_index + } + } + ]...) + + agent_nodepools = merge([ + for pool_index, nodepool_obj in var.agent_nodepools : { + for node_index in range(nodepool_obj.count) : + format("%s-%s-%s", pool_index, node_index, nodepool_obj.name) => { + nodepool_name : nodepool_obj.name, + server_type : nodepool_obj.server_type, + location : nodepool_obj.location, + labels : concat(local.default_agent_labels, nodepool_obj.labels), + taints : nodepool_obj.taints, + index : node_index } } ]...) @@ -193,7 +207,9 @@ locals { # disable k3s extras disable_extras = concat(["local-storage"], local.is_single_node_cluster ? [] : ["servicelb"], var.traefik_enabled ? [] : ["traefik"], var.metrics_server_enabled ? [] : ["metrics-server"]) - # Default k3s node labels - default_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []) + default_agent_labels = concat([], var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []) + default_control_plane_labels = concat([], var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]) + + first_control_plane = module.control_planes[keys(module.control_planes)[0]] } diff --git a/output.tf b/output.tf index 89514ef..798f81f 100644 --- a/output.tf +++ b/output.tf @@ -4,7 +4,9 @@ output "cluster_name" { } output "control_planes_public_ipv4" { - value = module.control_planes.*.ipv4_address + value = [ + for obj in module.control_planes : obj.ipv4_address + ] description = "The public IPv4 addresses of the controlplane server." } diff --git a/variables.tf b/variables.tf index 2b5a694..df7aaf8 100644 --- a/variables.tf +++ b/variables.tf @@ -20,26 +20,16 @@ variable "additional_public_keys" { default = [] } -variable "location" { - description = "Default server location" - type = string -} - variable "network_region" { description = "Default region for network" type = string } -variable "control_plane_server_type" { - description = "Default control plane server type" +variable "load_balancer_location" { + description = "Default load balancer location" type = string } -variable "control_plane_count" { - description = "Number of control plane nodes." - type = number -} - variable "load_balancer_type" { description = "Default load balancer server type" type = string @@ -51,6 +41,12 @@ variable "load_balancer_disable_ipv6" { default = false } +variable "control_plane_nodepools" { + description = "Number of control plane nodes." + type = list(any) + default = [] +} + variable "agent_nodepools" { description = "Number of agent nodes." type = list(any)