From 4fe79625b13cbb002c0596f82eae64a78be16656 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Sun, 27 Feb 2022 23:15:46 +0100 Subject: [PATCH 1/5] added dynamic rule and var extra_firewall_rules --- locals.tf | 121 +++++++++++++++++++++++++++++++++++++ main.tf | 127 +++------------------------------------ terraform.tfvars.example | 14 +++++ variables.tf | 6 ++ 4 files changed, 150 insertions(+), 118 deletions(-) diff --git a/locals.tf b/locals.tf index 08d3fd7..db3a115 100644 --- a/locals.tf +++ b/locals.tf @@ -17,6 +17,127 @@ locals { csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag kured_version = data.github_release.kured.release_tag + base_firewall_rules = [ + # Allowing internal cluster traffic and Hetzner metadata service and cloud API IPs + { + direction = "in" + protocol = "tcp" + port = "any" + source_ips = [ + var.network_ipv4_range, + "127.0.0.1/32", + "169.254.169.254/32", + "213.239.246.1/32" + ] + }, + { + direction = "in" + protocol = "udp" + port = "any" + source_ips = [ + var.network_ipv4_range, + "127.0.0.1/32", + "169.254.169.254/32", + "213.239.246.1/32" + ] + }, + { + direction = "in" + protocol = "icmp" + source_ips = [ + var.network_ipv4_range, + "127.0.0.1/32", + "169.254.169.254/32", + "213.239.246.1/32" + ] + }, + + # Allow all traffic to the kube api server + { + direction = "in" + protocol = "tcp" + port = "6443" + source_ips = [ + "0.0.0.0/0" + ] + }, + + # Allow all traffic to the ssh port + { + direction = "in" + protocol = "tcp" + port = "22" + source_ips = [ + "0.0.0.0/0" + ] + }, + + # Allow ping on ipv4 + { + direction = "in" + protocol = "icmp" + source_ips = [ + "0.0.0.0/0" + ] + }, + + # Allow basic out traffic + # ICMP to ping outside services + { + direction = "out" + protocol = "icmp" + destination_ips = [ + "0.0.0.0/0" + ] + }, + + # DNS + { + direction = "out" + protocol = "tcp" + port = "53" + destination_ips = [ + "0.0.0.0/0" + ] + }, + { + direction = "out" + protocol = "udp" + port = "53" + destination_ips = [ + "0.0.0.0/0" + ] + }, + + # HTTP(s) + { + direction = "out" + protocol = "tcp" + port = "80" + destination_ips = [ + "0.0.0.0/0" + ] + }, + { + direction = "out" + protocol = "tcp" + port = "443" + destination_ips = [ + "0.0.0.0/0" + ] + }, + + #NTP + { + direction = "out" + protocol = "udp" + port = "123" + destination_ips = [ + "0.0.0.0/0" + ] + } + ] + common_commands_install_k3s = [ "set -ex", # prepare the k3s config directory diff --git a/main.tf b/main.tf index 190c5ce..3ef8b0d 100644 --- a/main.tf +++ b/main.tf @@ -24,125 +24,16 @@ resource "hcloud_network_subnet" "subnet" { resource "hcloud_firewall" "k3s" { name = "k3s" - # Allowing internal cluster traffic and Hetzner metadata service and cloud API IPs - rule { - direction = "in" - protocol = "tcp" - port = "any" - source_ips = [ - var.network_ipv4_range, - "127.0.0.1/32", - "169.254.169.254/32", - "213.239.246.1/32" - ] + dynamic "rule" { + for_each = concat(local.base_firewall_rules, var.extra_firewall_rules) + content { + direction = rule.value.direction + protocol = rule.value.protocol + port = lookup(rule.value, "port", null) + destination_ips = lookup(rule.value, "destination_ips", null) + source_ips = lookup(rule.value, "source_ips", null) + } } - rule { - direction = "in" - protocol = "udp" - port = "any" - source_ips = [ - var.network_ipv4_range, - "127.0.0.1/32", - "169.254.169.254/32", - "213.239.246.1/32" - ] - } - rule { - direction = "in" - protocol = "icmp" - source_ips = [ - var.network_ipv4_range, - "127.0.0.1/32", - "169.254.169.254/32", - "213.239.246.1/32" - ] - } - - # Allow all traffic to the kube api server - rule { - direction = "in" - protocol = "tcp" - port = "6443" - source_ips = [ - "0.0.0.0/0" - ] - } - - # Allow all traffic to the ssh port - rule { - direction = "in" - protocol = "tcp" - port = "22" - source_ips = [ - "0.0.0.0/0" - ] - } - - # Allow ping on ipv4 - rule { - direction = "in" - protocol = "icmp" - source_ips = [ - "0.0.0.0/0" - ] - } - - # Allow basic out traffic - # ICMP to ping outside services - rule { - direction = "out" - protocol = "icmp" - destination_ips = [ - "0.0.0.0/0" - ] - } - - # DNS - rule { - direction = "out" - protocol = "tcp" - port = "53" - destination_ips = [ - "0.0.0.0/0" - ] - } - rule { - direction = "out" - protocol = "udp" - port = "53" - destination_ips = [ - "0.0.0.0/0" - ] - } - - # HTTP(s) - rule { - direction = "out" - protocol = "tcp" - port = "80" - destination_ips = [ - "0.0.0.0/0" - ] - } - rule { - direction = "out" - protocol = "tcp" - port = "443" - destination_ips = [ - "0.0.0.0/0" - ] - } - - #NTP - rule { - direction = "out" - protocol = "udp" - port = "123" - destination_ips = [ - "0.0.0.0/0" - ] - } - } resource "hcloud_placement_group" "k3s" { diff --git a/terraform.tfvars.example b/terraform.tfvars.example index a9568ce..504e2f1 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -50,3 +50,17 @@ agent_nodepools = { # Allows you to specify either stable, latest, or testing (defaults to stable), see https://rancher.com/docs/k3s/latest/en/upgrades/basic/ # initial_k3s_channel = "latest" + +# Adding extra firewall rules, like opening a port +# In this example with allow port TCP 5432 for a Postgres service we will open via a nodeport +# More info on the format here https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/firewall +extra_firewall_rules = [ + { + direction = "in" + protocol = "tcp" + port = "5432" + source_ips = [ + "0.0.0.0/0" + ] + }, +] diff --git a/variables.tf b/variables.tf index 58bdd0a..15ecb17 100644 --- a/variables.tf +++ b/variables.tf @@ -113,3 +113,9 @@ variable "automatically_upgrade_k3s" { default = true description = "Whether to automatically upgrade k3s based on the selected channel" } + +variable "extra_firewall_rules" { + type = list(any) + default = [] + description = "Additional firewall rules to apply to the cluster" +} From c0c27229f4b337c7bdc93a1c54b97c1b777417e8 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Sun, 27 Feb 2022 23:30:56 +0100 Subject: [PATCH 2/5] tweaked tfvars.example --- terraform.tfvars.example | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 504e2f1..a7fc11b 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -54,13 +54,13 @@ agent_nodepools = { # Adding extra firewall rules, like opening a port # In this example with allow port TCP 5432 for a Postgres service we will open via a nodeport # More info on the format here https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/firewall -extra_firewall_rules = [ - { - direction = "in" - protocol = "tcp" - port = "5432" - source_ips = [ - "0.0.0.0/0" - ] - }, -] +# extra_firewall_rules = [ +# { +# direction = "in" +# protocol = "tcp" +# port = "5432" +# source_ips = [ +# "0.0.0.0/0" +# ] +# }, +# ] From 48b15fafbf163f7ead5629393409911c7670c339 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Sun, 27 Feb 2022 23:51:05 +0100 Subject: [PATCH 3/5] clarified Hetzner IPs --- locals.tf | 48 ++++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/locals.tf b/locals.tf index db3a115..f5a5c6e 100644 --- a/locals.tf +++ b/locals.tf @@ -17,39 +17,35 @@ locals { csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag kured_version = data.github_release.kured.release_tag + # The following IPs are important to be whitelisted because they communicate with Hetzner services and enable the CCM and CSI to work properly. + # Source https://github.com/hetznercloud/csi-driver/issues/204#issuecomment-848625566 + hetzner_metadata_service_ipv4 = "169.254.169.254/32" + hetzner_cloud_api_ipv4 = "213.239.246.1/32" + whitelisted_ips = [ + var.network_ipv4_range, + local.hetzner_metadata_service_ipv4, + local.hetzner_cloud_api_ipv4, + "127.0.0.1/32", + ] + base_firewall_rules = [ # Allowing internal cluster traffic and Hetzner metadata service and cloud API IPs { - direction = "in" - protocol = "tcp" - port = "any" - source_ips = [ - var.network_ipv4_range, - "127.0.0.1/32", - "169.254.169.254/32", - "213.239.246.1/32" - ] + direction = "in" + protocol = "tcp" + port = "any" + source_ips = local.whitelisted_ips }, { - direction = "in" - protocol = "udp" - port = "any" - source_ips = [ - var.network_ipv4_range, - "127.0.0.1/32", - "169.254.169.254/32", - "213.239.246.1/32" - ] + direction = "in" + protocol = "udp" + port = "any" + source_ips = local.whitelisted_ips }, { - direction = "in" - protocol = "icmp" - source_ips = [ - var.network_ipv4_range, - "127.0.0.1/32", - "169.254.169.254/32", - "213.239.246.1/32" - ] + direction = "in" + protocol = "icmp" + source_ips = local.whitelisted_ips }, # Allow all traffic to the kube api server From 852a3f30c154e3932d681a7cea06fbeed384417c Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Sun, 27 Feb 2022 23:58:46 +0100 Subject: [PATCH 4/5] clarified Hetzner IPs --- locals.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/locals.tf b/locals.tf index f5a5c6e..f47913e 100644 --- a/locals.tf +++ b/locals.tf @@ -21,6 +21,7 @@ locals { # Source https://github.com/hetznercloud/csi-driver/issues/204#issuecomment-848625566 hetzner_metadata_service_ipv4 = "169.254.169.254/32" hetzner_cloud_api_ipv4 = "213.239.246.1/32" + whitelisted_ips = [ var.network_ipv4_range, local.hetzner_metadata_service_ipv4, From 9abc1026011f84dcab355b643b4438dea4dee8a6 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Mon, 28 Feb 2022 00:07:31 +0100 Subject: [PATCH 5/5] small tweak --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 3ef8b0d..c54204f 100644 --- a/main.tf +++ b/main.tf @@ -30,8 +30,8 @@ resource "hcloud_firewall" "k3s" { direction = rule.value.direction protocol = rule.value.protocol port = lookup(rule.value, "port", null) - destination_ips = lookup(rule.value, "destination_ips", null) - source_ips = lookup(rule.value, "source_ips", null) + destination_ips = lookup(rule.value, "destination_ips", []) + source_ips = lookup(rule.value, "source_ips", []) } } }