From ac1b0b93a8ac88f37b8c465519a2affec7ceeb9b Mon Sep 17 00:00:00 2001 From: jonny Date: Fri, 11 Mar 2022 12:17:48 +0100 Subject: [PATCH 1/3] add toggle to disable traefik & metric server --- README.md | 2 +- init.tf | 8 ++++---- main.tf | 2 +- output.tf | 2 +- terraform.tfvars.example | 7 ++++++- variables.tf | 12 ++++++++++++ 6 files changed, 25 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 34581ac..6cc23fc 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ _Please note that we are not affiliated to Hetzner, this is just an open source - Proper use of the underlying Hetzner private network to remove the need for encryption and minimize latency. - Automatic HA with the default setting of three control-plane and two agents nodes. - Ability to add or remove as many nodes as you want while the cluster stays running. -- Automatic Traefik ingress controller attached to a Hetzner load balancer with proxy protocol turned on. +- (Optional) Traefik ingress controller attached to a Hetzner load balancer with proxy protocol turned on. - (Optional) Out of the box config of Traefik with SSL certficate auto-generation. _It uses Terraform to deploy as it's easy to use, and Hetzner provides a great [Hetzner Terraform Provider](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs)._ diff --git a/init.tf b/init.tf index 4b64e15..92ab81c 100644 --- a/init.tf +++ b/init.tf @@ -13,7 +13,7 @@ resource "null_resource" "first_control_plane" { token = random_password.k3s_token.result cluster-init = true disable-cloud-controller = true - disable = concat(["local-storage"], local.is_single_node_cluster ? [] : ["servicelb"]) + disable = concat(["local-storage"], local.is_single_node_cluster ? [] : ["servicelb"], var.traefik_enabled ? [] : ["traefik"], var.metric_server_enabled ? [] : ["metric-server"]) flannel-iface = "eth1" kubelet-arg = "cloud-provider=external" node-ip = module.control_planes[0].private_ipv4_address @@ -79,7 +79,7 @@ resource "null_resource" "kustomization" { "https://raw.githubusercontent.com/hetznercloud/csi-driver/${local.csi_version}/deploy/kubernetes/hcloud-csi.yml", "https://github.com/weaveworks/kured/releases/download/${local.kured_version}/kured-${local.kured_version}-dockerhub.yaml", "https://raw.githubusercontent.com/rancher/system-upgrade-controller/master/manifests/system-upgrade-controller.yaml", - ], local.is_single_node_cluster ? [] : ["traefik.yaml"]), + ], local.is_single_node_cluster ? [] : var.traefik_enabled ? ["traefik.yaml"] : []), patchesStrategicMerge = [ file("${path.module}/kustomize/kured.yaml"), file("${path.module}/kustomize/ccm.yaml"), @@ -91,7 +91,7 @@ resource "null_resource" "kustomization" { # Upload traefik config provisioner "file" { - content = local.is_single_node_cluster ? "" : templatefile( + content = local.is_single_node_cluster ? "" : var.traefik_enabled == false ? "" : templatefile( "${path.module}/templates/traefik_config.yaml.tpl", { name = "${var.cluster_name}-traefik" @@ -142,7 +142,7 @@ resource "null_resource" "kustomization" { "kubectl -n system-upgrade wait --for=condition=available --timeout=120s deployment/system-upgrade-controller", "kubectl -n system-upgrade apply -f /tmp/post_install/plans.yaml" ], - local.is_single_node_cluster ? [] : [<<-EOT + local.is_single_node_cluster ? [] : var.traefik_enabled == false ? [] : [<<-EOT timeout 120 bash < /dev/null)" ]; do echo "Waiting for load-balancer to get an IP..." diff --git a/main.tf b/main.tf index 0652399..3186534 100644 --- a/main.tf +++ b/main.tf @@ -56,7 +56,7 @@ resource "hcloud_placement_group" "k3s" { } data "hcloud_load_balancer" "traefik" { - count = local.is_single_node_cluster ? 0 : 1 + count = local.is_single_node_cluster ? 0 : var.traefik_enabled == false ? 0 : 1 name = "${var.cluster_name}-traefik" depends_on = [null_resource.kustomization] diff --git a/output.tf b/output.tf index e0089fd..b84c324 100644 --- a/output.tf +++ b/output.tf @@ -17,7 +17,7 @@ output "agents_public_ipv4" { output "load_balancer_public_ipv4" { description = "The public IPv4 address of the Hetzner load balancer" - value = local.is_single_node_cluster ? module.control_planes[0].ipv4_address : data.hcloud_load_balancer.traefik[0].ipv4 + value = local.is_single_node_cluster || var.traefik_enabled == false ? module.control_planes[0].ipv4_address : data.hcloud_load_balancer.traefik[0].ipv4 } output "kubeconfig_file" { diff --git a/terraform.tfvars.example b/terraform.tfvars.example index e69c3a4..26d5d70 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -5,7 +5,7 @@ # This is in order to keep terraform from re-provisioning all nodes at once which would loose data. If you want to update, # those, you should instead change the value here and then manually re-provision each node one-by-one. Grep for "lifecycle". -# * Your Hetzner project API token +# * Your Hetzner project API token hcloud_token = "xxxxxxxxxxxxxxxxxxYYYYYYYYYYYYYYYYYYYzzzzzzzzzzzzzzzzzzzzz" # * Your public key public_key = "/home/username/.ssh/id_ed25519.pub" @@ -63,10 +63,15 @@ load_balancer_type = "lb11" # hetzner_ccm_version = "" # hetzner_csi_version = "" +# If you want to use traefik ingress controller with a loadbalancer +# traefik_enabled = true # If you want to use letsencrypt with tls Challenge, the email address is used to send you certificates expiration notices # traefik_acme_tls = true # traefik_acme_email = "mail@example.com" +# If you want to enable k8s metric server or not +# metric_server_enabled = false + # If you want to allow non-control-plane workloads to run on the control-plane nodes set "true" below. The default is "false". # Also good for single node clusters. # allow_scheduling_on_control_plane = true diff --git a/variables.tf b/variables.tf index 787240e..0b0936a 100644 --- a/variables.tf +++ b/variables.tf @@ -80,6 +80,12 @@ variable "hetzner_csi_version" { description = "Version of Container Storage Interface driver for Hetzner Cloud" } +variable "traefik_enabled" { + type = bool + default = false + description = "Whether to enable or disbale k3s traefik installation" +} + variable "traefik_acme_tls" { type = bool default = false @@ -98,6 +104,12 @@ variable "allow_scheduling_on_control_plane" { description = "Whether to allow non-control-plane workloads to run on the control-plane nodes" } +variable "metric_server_enabled" { + type = bool + default = true + description = "Whether to enable or disbale k3s mertric server" +} + variable "initial_k3s_channel" { type = string default = "stable" From 4167f6124bad6991364d3fc685c7cdadec7e3897 Mon Sep 17 00:00:00 2001 From: jonny Date: Fri, 11 Mar 2022 12:19:57 +0100 Subject: [PATCH 2/3] fix deprecated waring, switched to local_Sensitive_file for kubeconfig --- kubeconfig.tf | 4 ++-- main.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kubeconfig.tf b/kubeconfig.tf index 73c1faa..67832b3 100644 --- a/kubeconfig.tf +++ b/kubeconfig.tf @@ -23,8 +23,8 @@ locals { } } -resource "local_file" "kubeconfig" { - sensitive_content = local.kubeconfig_external +resource "local_sensitive_file" "kubeconfig" { + content = local.kubeconfig_external filename = "kubeconfig.yaml" file_permission = "600" } diff --git a/main.tf b/main.tf index 3186534..12502a1 100644 --- a/main.tf +++ b/main.tf @@ -78,7 +78,7 @@ resource "null_resource" "destroy_traefik_loadbalancer" { } depends_on = [ - local_file.kubeconfig, + local_sensitive_file.kubeconfig, null_resource.control_planes[0], hcloud_network_subnet.subnet, hcloud_network.k3s, From 6361990cdf664ff80f45f292063809483fb5ed43 Mon Sep 17 00:00:00 2001 From: jonny Date: Fri, 11 Mar 2022 14:24:24 +0100 Subject: [PATCH 3/3] simplify contitions, little fix --- control_planes.tf | 4 ++-- init.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/control_planes.tf b/control_planes.tf index a19b212..c410fe8 100644 --- a/control_planes.tf +++ b/control_planes.tf @@ -14,7 +14,7 @@ module "control_planes" { ipv4_subnet_id = hcloud_network_subnet.subnet["control_plane"].id # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely - # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough. + # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough. private_ipv4 = cidrhost(var.network_ipv4_subnets["control_plane"], count.index + 101) labels = { @@ -48,7 +48,7 @@ resource "null_resource" "control_planes" { server = "https://${element(module.control_planes.*.private_ipv4_address, count.index > 0 ? 0 : 1)}:6443" token = random_password.k3s_token.result disable-cloud-controller = true - disable = ["servicelb", "local-storage"] + disable = ["servicelb", "local-storage", "traefik", "metric-server"] flannel-iface = "eth1" kubelet-arg = "cloud-provider=external" node-ip = module.control_planes[count.index].private_ipv4_address diff --git a/init.tf b/init.tf index 92ab81c..3ccc6fe 100644 --- a/init.tf +++ b/init.tf @@ -91,7 +91,7 @@ resource "null_resource" "kustomization" { # Upload traefik config provisioner "file" { - content = local.is_single_node_cluster ? "" : var.traefik_enabled == false ? "" : templatefile( + content = local.is_single_node_cluster || var.traefik_enabled == false ? "" : templatefile( "${path.module}/templates/traefik_config.yaml.tpl", { name = "${var.cluster_name}-traefik" @@ -142,7 +142,7 @@ resource "null_resource" "kustomization" { "kubectl -n system-upgrade wait --for=condition=available --timeout=120s deployment/system-upgrade-controller", "kubectl -n system-upgrade apply -f /tmp/post_install/plans.yaml" ], - local.is_single_node_cluster ? [] : var.traefik_enabled == false ? [] : [<<-EOT + local.is_single_node_cluster || var.traefik_enabled == false ? [] : [<<-EOT timeout 120 bash < /dev/null)" ]; do echo "Waiting for load-balancer to get an IP..."