Added Hetzner firewall and fixed addresses

README.md

@@ -69,9 +69,11 @@ The number of control plane nodes and worker nodes, and the Hetzner datacenter l
 See the default values in the [variables.tf](variables.tf) file, they correspond to (you can copy-paste and customize):
 
 ```tfvars
-servers_num      = 2
+servers_num = 2
-agents_num       = 2
+agents_num = 2
-server_location  = "fsn1"
+location = "fsn1"
+agent_server_type = "cx21"
+control_plane_server_type = "cx11"
 ```
 
 ### Installation

agents.tf

@@ -2,17 +2,17 @@ resource "hcloud_server" "agents" {
   count = var.agents_num
   name  = "k3s-agent-${count.index}"
 
-  image       = data.hcloud_image.linux.name
+  image        = data.hcloud_image.linux.name
-  server_type = local.agent_server_type
+  server_type  = var.agent_server_type
-  location    = local.agent_locations[count.index][1]
+  location     = var.location
+  ssh_keys     = [hcloud_ssh_key.default.id]
+  firewall_ids = [hcloud_firewall.k3s.id]
 
-  ssh_keys = [hcloud_ssh_key.default.id]
 
   labels = {
-    provisioner = "terraform",
+    "provisioner" = "terraform",
-    engine      = "k3s",
+    "engine"      = "k3s",
-    node_type   = "agent"
+    "k3s_upgrade" = "true"
-    k3s_upgrade = "true"
   }
 
   user_data = data.template_cloudinit_config.init_cfg.rendered
@@ -29,7 +29,7 @@ resource "hcloud_server" "agents" {
 
   provisioner "remote-exec" {
     inline = [
-      "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - agent --server https://${local.first_control_plane_ip}:6443 --kubelet-arg='cloud-provider=external' --no-flannel"
+      "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - agent --server https://${local.first_control_plane_network_ip}:6443 --node-ip=${cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index)} ${var.k3s_agent_flags}"
     ]
 
     connection {

k3s.tf

@@ -1,8 +0,0 @@
-locals {
-  control_plane_server_type = "cx11"
-  agent_server_type         = "cx21"
-  first_control_plane_ip    = cidrhost(hcloud_network.k3s.ip_range, 2)
-  locations                 = [var.server_location, "fsn1", "fsn1"]
-  agent_locations           = setproduct(range(var.agents_num), local.locations)
-  server_locations          = setproduct(range(var.servers_num), local.locations)
-}

main.tf

@@ -20,6 +20,65 @@ resource "hcloud_network_subnet" "k3s" {
   ip_range     = "10.0.0.0/16"
 }
 
+resource "hcloud_firewall" "k3s" {
+  name = "k3s-firewall"
+
+  # Internal cluster traffic, kube api server, kubelet metrics, cilium, etcd,
+  # and Hetzner metadata service and cloud api
+  rule {
+    direction = "in"
+    protocol  = "tcp"
+    port      = "any"
+    source_ips = [
+      "127.0.0.1/32",
+      "10.0.0.0/8",
+      "169.254.169.254/32",
+      "213.239.246.1/32"
+    ]
+  }
+  rule {
+    direction = "in"
+    protocol  = "udp"
+    port      = "any"
+    source_ips = [
+      "127.0.0.1/32",
+      "10.0.0.0/8",
+      "169.254.169.254/32",
+      "213.239.246.1/32"
+    ]
+  }
+  rule {
+    direction = "in"
+    protocol  = "icmp"
+    source_ips = [
+      "127.0.0.1/32",
+      "10.0.0.0/8",
+      "169.254.169.254/32",
+      "213.239.246.1/32"
+    ]
+  }
+
+  # Allow all traffic to the kube api server
+  rule {
+    direction = "in"
+    protocol  = "tcp"
+    port      = "6443"
+    source_ips = [
+      "0.0.0.0/0"
+    ]
+  }
+
+  # Allow all traffic to the ssh port
+  rule {
+    direction = "in"
+    protocol  = "tcp"
+    port      = "22"
+    source_ips = [
+      "0.0.0.0/0"
+    ]
+  }
+}
+
 data "hcloud_image" "linux" {
   name = "fedora-34"
 }
@@ -49,3 +108,7 @@ data "template_file" "ccm_manifest" {
 data "template_file" "upgrade_plan" {
   template = file("${path.module}/manifests/upgrade/plan.yaml")
 }
+
+locals {
+  first_control_plane_network_ip = cidrhost(hcloud_network.k3s.ip_range, 2)
+}

manifests/helm/cilium/values.yaml

@@ -103,19 +103,7 @@ loadBalancer:
 
   # -- algorithm is the name of the load balancing algorithm for backend
   # selection e.g. random or maglev
-  # algorithm: random
+  algorithm: maglev
-
-  # -- mode is the operation mode of load balancing for remote backends
-  # e.g. snat, dsr, hybrid
-  mode: snat
-
-  # -- acceleration is the option to accelerate service handling via XDP
-  # e.g. native, disabled
-  # Gives "Error: virtio_net: Too few free TX rings available."
-  # acceleration: native
-
-# Breaks csi
-# devices: eth1
 
 # -- The agent can be put into one of the three policy enforcement modes:
 # default, always and never.

manifests/helm/nginx/values.yaml

@@ -2,6 +2,7 @@ controller:
   kind: DaemonSet
   service: 
     annotations:
+      # adjust the location if you have customized it in terraform.tfvars
       load-balancer.hetzner.cloud/location: "fsn1"
       load-balancer.hetzner.cloud/use-private-ip: "true"
       load-balancer.hetzner.cloud/type: "lb11"

master.tf

@@ -1,17 +1,17 @@
 resource "hcloud_server" "first_control_plane" {
   name = "k3s-control-plane-0"
 
-  image       = data.hcloud_image.linux.name
+  image        = data.hcloud_image.linux.name
-  server_type = local.control_plane_server_type
+  server_type  = var.control_plane_server_type
-  location    = local.server_locations[0][1]
+  location     = var.location
+  ssh_keys     = [hcloud_ssh_key.default.id]
+  firewall_ids = [hcloud_firewall.k3s.id]
 
-  ssh_keys = [hcloud_ssh_key.default.id]
 
   labels = {
-    provisioner = "terraform",
+    "provisioner" = "terraform",
-    engine      = "k3s",
+    "engine"      = "k3s",
-    node_type   = "control-plane"
+    "k3s_upgrade" = "true"
-    k3s_upgrade = "true"
   }
 
   user_data = data.template_cloudinit_config.init_cfg.rendered
@@ -28,7 +28,7 @@ resource "hcloud_server" "first_control_plane" {
 
   provisioner "remote-exec" {
     inline = [
-      "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --cluster-init ${var.k3s_extra_args}",
+      "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --cluster-init --node-ip=${local.first_control_plane_network_ip} --advertise-address=${local.first_control_plane_network_ip} --tls-san=${local.first_control_plane_network_ip} ${var.k3s_server_flags}",
       "until systemctl is-active --quiet k3s.service; do sleep 1; done",
       "until kubectl get node ${self.name}; do sleep 1; done",
       "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}",
@@ -62,10 +62,11 @@ resource "hcloud_server" "first_control_plane" {
 
   network {
     network_id = hcloud_network.k3s.id
-    ip         = local.first_control_plane_ip
+    ip         = local.first_control_plane_network_ip
   }
 
   depends_on = [
-    hcloud_network_subnet.k3s
+    hcloud_network_subnet.k3s,
+    hcloud_firewall.k3s
   ]
 }

servers.tf

@@ -2,17 +2,17 @@ resource "hcloud_server" "control_planes" {
   count = var.servers_num - 1
   name  = "k3s-control-plane-${count.index + 1}"
 
-  image       = data.hcloud_image.linux.name
+  image        = data.hcloud_image.linux.name
-  server_type = local.control_plane_server_type
+  server_type  = var.control_plane_server_type
-  location    = local.server_locations[count.index + 1][1]
+  location     = var.location
+  ssh_keys     = [hcloud_ssh_key.default.id]
+  firewall_ids = [hcloud_firewall.k3s.id]
 
-  ssh_keys = [hcloud_ssh_key.default.id]
 
   labels = {
-    provisioner = "terraform",
+    "provisioner" = "terraform",
-    engine      = "k3s",
+    "engine"      = "k3s",
-    node_type   = "control-plane"
+    "k3s_upgrade" = "true"
-    k3s_upgrade = "true"
   }
 
   user_data = data.template_cloudinit_config.init_cfg.rendered
@@ -29,7 +29,7 @@ resource "hcloud_server" "control_planes" {
 
   provisioner "remote-exec" {
     inline = [
-      "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --server https://${local.first_control_plane_ip}:6443 ${var.k3s_extra_args}"
+      "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --server https://${local.first_control_plane_network_ip}:6443 --node-ip=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} --advertise-address=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} --tls-san=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} ${var.k3s_server_flags}",
     ]
 
     connection {

variables.tf

@@ -27,14 +27,31 @@ variable "agents_num" {
   default     = 2
 }
 
-variable "server_location" {
+variable "location" {
   description = "Default server location"
   default     = "fsn1"
 }
 
-variable "k3s_extra_args" {
+
+variable "control_plane_server_type" {
+  description = "Default control plane server type"
+  default     = "cx11"
+
+}
+
+variable "agent_server_type" {
+  description = "Default agent server type"
+  default     = "cx21"
+}
+
+variable "k3s_server_flags" {
   description = "Important flags to make our setup work"
-  default     = "--disable-cloud-controller --disable-network-policy --no-deploy=traefik --no-deploy=servicelb --disable local-storage --disable traefik --disable servicelb --kubelet-arg='cloud-provider=external' --no-flannel"
+  default     = "--disable-cloud-controller --disable-network-policy --disable=traefik --disable=servicelb --disable='local-storage' --kubelet-arg='cloud-provider=external' --flannel-backend=none --flannel-iface=ens10"
+}
+
+variable "k3s_agent_flags" {
+  description = "Important flags to make our setup work"
+  default     = "--kubelet-arg='cloud-provider=external' --flannel-iface=ens10"
 }
 
 variable "initial_commands" {
@@ -46,6 +63,7 @@ variable "initial_commands" {
     "systemctl enable --now dnf-automatic.timer",
     "systemctl disable firewalld",
     "grubby --args='systemd.unified_cgroup_hierarchy=0' --update-kernel=ALL",
-    "sleep 10; shutdown -r +0"
+    "sleep 11; shutdown -r +0"
   ]
 }
+