From ca2bf4dc82965858c0a816d14d0b3fdb50416d8e Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 1 Sep 2021 00:37:11 +0200 Subject: [PATCH] Added Hetzner firewall and fixed addresses --- README.md | 8 ++-- agents.tf | 18 ++++----- k3s.tf | 8 ---- main.tf | 63 +++++++++++++++++++++++++++++++ manifests/helm/cilium/values.yaml | 14 +------ manifests/helm/nginx/values.yaml | 1 + master.tf | 23 +++++------ servers.tf | 18 ++++----- variables.tf | 26 +++++++++++-- 9 files changed, 122 insertions(+), 57 deletions(-) delete mode 100644 k3s.tf diff --git a/README.md b/README.md index 727a4a3..ac777df 100644 --- a/README.md +++ b/README.md @@ -69,9 +69,11 @@ The number of control plane nodes and worker nodes, and the Hetzner datacenter l See the default values in the [variables.tf](variables.tf) file, they correspond to (you can copy-paste and customize): ```tfvars -servers_num = 2 -agents_num = 2 -server_location = "fsn1" +servers_num = 2 +agents_num = 2 +location = "fsn1" +agent_server_type = "cx21" +control_plane_server_type = "cx11" ``` ### Installation diff --git a/agents.tf b/agents.tf index 402fa05..62f50e5 100644 --- a/agents.tf +++ b/agents.tf @@ -2,17 +2,17 @@ resource "hcloud_server" "agents" { count = var.agents_num name = "k3s-agent-${count.index}" - image = data.hcloud_image.linux.name - server_type = local.agent_server_type - location = local.agent_locations[count.index][1] + image = data.hcloud_image.linux.name + server_type = var.agent_server_type + location = var.location + ssh_keys = [hcloud_ssh_key.default.id] + firewall_ids = [hcloud_firewall.k3s.id] - ssh_keys = [hcloud_ssh_key.default.id] labels = { - provisioner = "terraform", - engine = "k3s", - node_type = "agent" - k3s_upgrade = "true" + "provisioner" = "terraform", + "engine" = "k3s", + "k3s_upgrade" = "true" } user_data = data.template_cloudinit_config.init_cfg.rendered @@ -29,7 +29,7 @@ resource "hcloud_server" "agents" { provisioner "remote-exec" { inline = [ - "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - agent --server https://${local.first_control_plane_ip}:6443 --kubelet-arg='cloud-provider=external' --no-flannel" + "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - agent --server https://${local.first_control_plane_network_ip}:6443 --node-ip=${cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index)} ${var.k3s_agent_flags}" ] connection { diff --git a/k3s.tf b/k3s.tf deleted file mode 100644 index a3a6a1b..0000000 --- a/k3s.tf +++ /dev/null @@ -1,8 +0,0 @@ -locals { - control_plane_server_type = "cx11" - agent_server_type = "cx21" - first_control_plane_ip = cidrhost(hcloud_network.k3s.ip_range, 2) - locations = [var.server_location, "fsn1", "fsn1"] - agent_locations = setproduct(range(var.agents_num), local.locations) - server_locations = setproduct(range(var.servers_num), local.locations) -} diff --git a/main.tf b/main.tf index d035b5f..fe076ec 100644 --- a/main.tf +++ b/main.tf @@ -20,6 +20,65 @@ resource "hcloud_network_subnet" "k3s" { ip_range = "10.0.0.0/16" } +resource "hcloud_firewall" "k3s" { + name = "k3s-firewall" + + # Internal cluster traffic, kube api server, kubelet metrics, cilium, etcd, + # and Hetzner metadata service and cloud api + rule { + direction = "in" + protocol = "tcp" + port = "any" + source_ips = [ + "127.0.0.1/32", + "10.0.0.0/8", + "169.254.169.254/32", + "213.239.246.1/32" + ] + } + rule { + direction = "in" + protocol = "udp" + port = "any" + source_ips = [ + "127.0.0.1/32", + "10.0.0.0/8", + "169.254.169.254/32", + "213.239.246.1/32" + ] + } + rule { + direction = "in" + protocol = "icmp" + source_ips = [ + "127.0.0.1/32", + "10.0.0.0/8", + "169.254.169.254/32", + "213.239.246.1/32" + ] + } + + # Allow all traffic to the kube api server + rule { + direction = "in" + protocol = "tcp" + port = "6443" + source_ips = [ + "0.0.0.0/0" + ] + } + + # Allow all traffic to the ssh port + rule { + direction = "in" + protocol = "tcp" + port = "22" + source_ips = [ + "0.0.0.0/0" + ] + } +} + data "hcloud_image" "linux" { name = "fedora-34" } @@ -49,3 +108,7 @@ data "template_file" "ccm_manifest" { data "template_file" "upgrade_plan" { template = file("${path.module}/manifests/upgrade/plan.yaml") } + +locals { + first_control_plane_network_ip = cidrhost(hcloud_network.k3s.ip_range, 2) +} diff --git a/manifests/helm/cilium/values.yaml b/manifests/helm/cilium/values.yaml index 1586c62..9d8e786 100644 --- a/manifests/helm/cilium/values.yaml +++ b/manifests/helm/cilium/values.yaml @@ -103,19 +103,7 @@ loadBalancer: # -- algorithm is the name of the load balancing algorithm for backend # selection e.g. random or maglev - # algorithm: random - - # -- mode is the operation mode of load balancing for remote backends - # e.g. snat, dsr, hybrid - mode: snat - - # -- acceleration is the option to accelerate service handling via XDP - # e.g. native, disabled - # Gives "Error: virtio_net: Too few free TX rings available." - # acceleration: native - -# Breaks csi -# devices: eth1 + algorithm: maglev # -- The agent can be put into one of the three policy enforcement modes: # default, always and never. diff --git a/manifests/helm/nginx/values.yaml b/manifests/helm/nginx/values.yaml index 4733b9c..a78f674 100644 --- a/manifests/helm/nginx/values.yaml +++ b/manifests/helm/nginx/values.yaml @@ -2,6 +2,7 @@ controller: kind: DaemonSet service: annotations: + # adjust the location if you have customized it in terraform.tfvars load-balancer.hetzner.cloud/location: "fsn1" load-balancer.hetzner.cloud/use-private-ip: "true" load-balancer.hetzner.cloud/type: "lb11" diff --git a/master.tf b/master.tf index 274ae7c..b7ce3a4 100644 --- a/master.tf +++ b/master.tf @@ -1,17 +1,17 @@ resource "hcloud_server" "first_control_plane" { name = "k3s-control-plane-0" - image = data.hcloud_image.linux.name - server_type = local.control_plane_server_type - location = local.server_locations[0][1] + image = data.hcloud_image.linux.name + server_type = var.control_plane_server_type + location = var.location + ssh_keys = [hcloud_ssh_key.default.id] + firewall_ids = [hcloud_firewall.k3s.id] - ssh_keys = [hcloud_ssh_key.default.id] labels = { - provisioner = "terraform", - engine = "k3s", - node_type = "control-plane" - k3s_upgrade = "true" + "provisioner" = "terraform", + "engine" = "k3s", + "k3s_upgrade" = "true" } user_data = data.template_cloudinit_config.init_cfg.rendered @@ -28,7 +28,7 @@ resource "hcloud_server" "first_control_plane" { provisioner "remote-exec" { inline = [ - "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --cluster-init ${var.k3s_extra_args}", + "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --cluster-init --node-ip=${local.first_control_plane_network_ip} --advertise-address=${local.first_control_plane_network_ip} --tls-san=${local.first_control_plane_network_ip} ${var.k3s_server_flags}", "until systemctl is-active --quiet k3s.service; do sleep 1; done", "until kubectl get node ${self.name}; do sleep 1; done", "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}", @@ -62,10 +62,11 @@ resource "hcloud_server" "first_control_plane" { network { network_id = hcloud_network.k3s.id - ip = local.first_control_plane_ip + ip = local.first_control_plane_network_ip } depends_on = [ - hcloud_network_subnet.k3s + hcloud_network_subnet.k3s, + hcloud_firewall.k3s ] } diff --git a/servers.tf b/servers.tf index b242e3e..4355430 100644 --- a/servers.tf +++ b/servers.tf @@ -2,17 +2,17 @@ resource "hcloud_server" "control_planes" { count = var.servers_num - 1 name = "k3s-control-plane-${count.index + 1}" - image = data.hcloud_image.linux.name - server_type = local.control_plane_server_type - location = local.server_locations[count.index + 1][1] + image = data.hcloud_image.linux.name + server_type = var.control_plane_server_type + location = var.location + ssh_keys = [hcloud_ssh_key.default.id] + firewall_ids = [hcloud_firewall.k3s.id] - ssh_keys = [hcloud_ssh_key.default.id] labels = { - provisioner = "terraform", - engine = "k3s", - node_type = "control-plane" - k3s_upgrade = "true" + "provisioner" = "terraform", + "engine" = "k3s", + "k3s_upgrade" = "true" } user_data = data.template_cloudinit_config.init_cfg.rendered @@ -29,7 +29,7 @@ resource "hcloud_server" "control_planes" { provisioner "remote-exec" { inline = [ - "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --server https://${local.first_control_plane_ip}:6443 ${var.k3s_extra_args}" + "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --server https://${local.first_control_plane_network_ip}:6443 --node-ip=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} --advertise-address=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} --tls-san=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} ${var.k3s_server_flags}", ] connection { diff --git a/variables.tf b/variables.tf index 13d8ffd..c0c736e 100644 --- a/variables.tf +++ b/variables.tf @@ -27,14 +27,31 @@ variable "agents_num" { default = 2 } -variable "server_location" { +variable "location" { description = "Default server location" default = "fsn1" } -variable "k3s_extra_args" { + +variable "control_plane_server_type" { + description = "Default control plane server type" + default = "cx11" + +} + +variable "agent_server_type" { + description = "Default agent server type" + default = "cx21" +} + +variable "k3s_server_flags" { description = "Important flags to make our setup work" - default = "--disable-cloud-controller --disable-network-policy --no-deploy=traefik --no-deploy=servicelb --disable local-storage --disable traefik --disable servicelb --kubelet-arg='cloud-provider=external' --no-flannel" + default = "--disable-cloud-controller --disable-network-policy --disable=traefik --disable=servicelb --disable='local-storage' --kubelet-arg='cloud-provider=external' --flannel-backend=none --flannel-iface=ens10" +} + +variable "k3s_agent_flags" { + description = "Important flags to make our setup work" + default = "--kubelet-arg='cloud-provider=external' --flannel-iface=ens10" } variable "initial_commands" { @@ -46,6 +63,7 @@ variable "initial_commands" { "systemctl enable --now dnf-automatic.timer", "systemctl disable firewalld", "grubby --args='systemd.unified_cgroup_hierarchy=0' --update-kernel=ALL", - "sleep 10; shutdown -r +0" + "sleep 11; shutdown -r +0" ] } +