Added Hetzner firewall and fixed addresses
This commit is contained in:
Karim Naufal
parent
6e9ee5855b
commit
ca2bf4dc82
@ -69,9 +69,11 @@ The number of control plane nodes and worker nodes, and the Hetzner datacenter l
|
||||
See the default values in the [variables.tf](variables.tf) file, they correspond to (you can copy-paste and customize):
|
||||
|
||||
```tfvars
|
||||
servers_num = 2
|
||||
agents_num = 2
|
||||
server_location = "fsn1"
|
||||
servers_num = 2
|
||||
agents_num = 2
|
||||
location = "fsn1"
|
||||
agent_server_type = "cx21"
|
||||
control_plane_server_type = "cx11"
|
||||
```
|
||||
|
||||
### Installation
|
||||
|
||||
18
agents.tf
18
agents.tf
@ -2,17 +2,17 @@ resource "hcloud_server" "agents" {
|
||||
count = var.agents_num
|
||||
name = "k3s-agent-${count.index}"
|
||||
|
||||
image = data.hcloud_image.linux.name
|
||||
server_type = local.agent_server_type
|
||||
location = local.agent_locations[count.index][1]
|
||||
image = data.hcloud_image.linux.name
|
||||
server_type = var.agent_server_type
|
||||
location = var.location
|
||||
ssh_keys = [hcloud_ssh_key.default.id]
|
||||
firewall_ids = [hcloud_firewall.k3s.id]
|
||||
|
||||
ssh_keys = [hcloud_ssh_key.default.id]
|
||||
|
||||
labels = {
|
||||
provisioner = "terraform",
|
||||
engine = "k3s",
|
||||
node_type = "agent"
|
||||
k3s_upgrade = "true"
|
||||
"provisioner" = "terraform",
|
||||
"engine" = "k3s",
|
||||
"k3s_upgrade" = "true"
|
||||
}
|
||||
|
||||
user_data = data.template_cloudinit_config.init_cfg.rendered
|
||||
@ -29,7 +29,7 @@ resource "hcloud_server" "agents" {
|
||||
|
||||
provisioner "remote-exec" {
|
||||
inline = [
|
||||
"curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - agent --server https://${local.first_control_plane_ip}:6443 --kubelet-arg='cloud-provider=external' --no-flannel"
|
||||
"curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - agent --server https://${local.first_control_plane_network_ip}:6443 --node-ip=${cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index)} ${var.k3s_agent_flags}"
|
||||
]
|
||||
|
||||
connection {
|
||||
|
||||
8
k3s.tf
8
k3s.tf
@ -1,8 +0,0 @@
|
||||
locals {
|
||||
control_plane_server_type = "cx11"
|
||||
agent_server_type = "cx21"
|
||||
first_control_plane_ip = cidrhost(hcloud_network.k3s.ip_range, 2)
|
||||
locations = [var.server_location, "fsn1", "fsn1"]
|
||||
agent_locations = setproduct(range(var.agents_num), local.locations)
|
||||
server_locations = setproduct(range(var.servers_num), local.locations)
|
||||
}
|
||||
63
main.tf
63
main.tf
@ -20,6 +20,65 @@ resource "hcloud_network_subnet" "k3s" {
|
||||
ip_range = "10.0.0.0/16"
|
||||
}
|
||||
|
||||
resource "hcloud_firewall" "k3s" {
|
||||
name = "k3s-firewall"
|
||||
|
||||
# Internal cluster traffic, kube api server, kubelet metrics, cilium, etcd,
|
||||
# and Hetzner metadata service and cloud api
|
||||
rule {
|
||||
direction = "in"
|
||||
protocol = "tcp"
|
||||
port = "any"
|
||||
source_ips = [
|
||||
"127.0.0.1/32",
|
||||
"10.0.0.0/8",
|
||||
"169.254.169.254/32",
|
||||
"213.239.246.1/32"
|
||||
]
|
||||
}
|
||||
rule {
|
||||
direction = "in"
|
||||
protocol = "udp"
|
||||
port = "any"
|
||||
source_ips = [
|
||||
"127.0.0.1/32",
|
||||
"10.0.0.0/8",
|
||||
"169.254.169.254/32",
|
||||
"213.239.246.1/32"
|
||||
]
|
||||
}
|
||||
rule {
|
||||
direction = "in"
|
||||
protocol = "icmp"
|
||||
source_ips = [
|
||||
"127.0.0.1/32",
|
||||
"10.0.0.0/8",
|
||||
"169.254.169.254/32",
|
||||
"213.239.246.1/32"
|
||||
]
|
||||
}
|
||||
|
||||
# Allow all traffic to the kube api server
|
||||
rule {
|
||||
direction = "in"
|
||||
protocol = "tcp"
|
||||
port = "6443"
|
||||
source_ips = [
|
||||
"0.0.0.0/0"
|
||||
]
|
||||
}
|
||||
|
||||
# Allow all traffic to the ssh port
|
||||
rule {
|
||||
direction = "in"
|
||||
protocol = "tcp"
|
||||
port = "22"
|
||||
source_ips = [
|
||||
"0.0.0.0/0"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
data "hcloud_image" "linux" {
|
||||
name = "fedora-34"
|
||||
}
|
||||
@ -49,3 +108,7 @@ data "template_file" "ccm_manifest" {
|
||||
data "template_file" "upgrade_plan" {
|
||||
template = file("${path.module}/manifests/upgrade/plan.yaml")
|
||||
}
|
||||
|
||||
locals {
|
||||
first_control_plane_network_ip = cidrhost(hcloud_network.k3s.ip_range, 2)
|
||||
}
|
||||
|
||||
@ -103,19 +103,7 @@ loadBalancer:
|
||||
|
||||
# -- algorithm is the name of the load balancing algorithm for backend
|
||||
# selection e.g. random or maglev
|
||||
# algorithm: random
|
||||
|
||||
# -- mode is the operation mode of load balancing for remote backends
|
||||
# e.g. snat, dsr, hybrid
|
||||
mode: snat
|
||||
|
||||
# -- acceleration is the option to accelerate service handling via XDP
|
||||
# e.g. native, disabled
|
||||
# Gives "Error: virtio_net: Too few free TX rings available."
|
||||
# acceleration: native
|
||||
|
||||
# Breaks csi
|
||||
# devices: eth1
|
||||
algorithm: maglev
|
||||
|
||||
# -- The agent can be put into one of the three policy enforcement modes:
|
||||
# default, always and never.
|
||||
|
||||
@ -2,6 +2,7 @@ controller:
|
||||
kind: DaemonSet
|
||||
service:
|
||||
annotations:
|
||||
# adjust the location if you have customized it in terraform.tfvars
|
||||
load-balancer.hetzner.cloud/location: "fsn1"
|
||||
load-balancer.hetzner.cloud/use-private-ip: "true"
|
||||
load-balancer.hetzner.cloud/type: "lb11"
|
||||
|
||||
23
master.tf
23
master.tf
@ -1,17 +1,17 @@
|
||||
resource "hcloud_server" "first_control_plane" {
|
||||
name = "k3s-control-plane-0"
|
||||
|
||||
image = data.hcloud_image.linux.name
|
||||
server_type = local.control_plane_server_type
|
||||
location = local.server_locations[0][1]
|
||||
image = data.hcloud_image.linux.name
|
||||
server_type = var.control_plane_server_type
|
||||
location = var.location
|
||||
ssh_keys = [hcloud_ssh_key.default.id]
|
||||
firewall_ids = [hcloud_firewall.k3s.id]
|
||||
|
||||
ssh_keys = [hcloud_ssh_key.default.id]
|
||||
|
||||
labels = {
|
||||
provisioner = "terraform",
|
||||
engine = "k3s",
|
||||
node_type = "control-plane"
|
||||
k3s_upgrade = "true"
|
||||
"provisioner" = "terraform",
|
||||
"engine" = "k3s",
|
||||
"k3s_upgrade" = "true"
|
||||
}
|
||||
|
||||
user_data = data.template_cloudinit_config.init_cfg.rendered
|
||||
@ -28,7 +28,7 @@ resource "hcloud_server" "first_control_plane" {
|
||||
|
||||
provisioner "remote-exec" {
|
||||
inline = [
|
||||
"curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --cluster-init ${var.k3s_extra_args}",
|
||||
"curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --cluster-init --node-ip=${local.first_control_plane_network_ip} --advertise-address=${local.first_control_plane_network_ip} --tls-san=${local.first_control_plane_network_ip} ${var.k3s_server_flags}",
|
||||
"until systemctl is-active --quiet k3s.service; do sleep 1; done",
|
||||
"until kubectl get node ${self.name}; do sleep 1; done",
|
||||
"kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}",
|
||||
@ -62,10 +62,11 @@ resource "hcloud_server" "first_control_plane" {
|
||||
|
||||
network {
|
||||
network_id = hcloud_network.k3s.id
|
||||
ip = local.first_control_plane_ip
|
||||
ip = local.first_control_plane_network_ip
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
hcloud_network_subnet.k3s
|
||||
hcloud_network_subnet.k3s,
|
||||
hcloud_firewall.k3s
|
||||
]
|
||||
}
|
||||
|
||||
18
servers.tf
18
servers.tf
@ -2,17 +2,17 @@ resource "hcloud_server" "control_planes" {
|
||||
count = var.servers_num - 1
|
||||
name = "k3s-control-plane-${count.index + 1}"
|
||||
|
||||
image = data.hcloud_image.linux.name
|
||||
server_type = local.control_plane_server_type
|
||||
location = local.server_locations[count.index + 1][1]
|
||||
image = data.hcloud_image.linux.name
|
||||
server_type = var.control_plane_server_type
|
||||
location = var.location
|
||||
ssh_keys = [hcloud_ssh_key.default.id]
|
||||
firewall_ids = [hcloud_firewall.k3s.id]
|
||||
|
||||
ssh_keys = [hcloud_ssh_key.default.id]
|
||||
|
||||
labels = {
|
||||
provisioner = "terraform",
|
||||
engine = "k3s",
|
||||
node_type = "control-plane"
|
||||
k3s_upgrade = "true"
|
||||
"provisioner" = "terraform",
|
||||
"engine" = "k3s",
|
||||
"k3s_upgrade" = "true"
|
||||
}
|
||||
|
||||
user_data = data.template_cloudinit_config.init_cfg.rendered
|
||||
@ -29,7 +29,7 @@ resource "hcloud_server" "control_planes" {
|
||||
|
||||
provisioner "remote-exec" {
|
||||
inline = [
|
||||
"curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --server https://${local.first_control_plane_ip}:6443 ${var.k3s_extra_args}"
|
||||
"curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --server https://${local.first_control_plane_network_ip}:6443 --node-ip=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} --advertise-address=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} --tls-san=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} ${var.k3s_server_flags}",
|
||||
]
|
||||
|
||||
connection {
|
||||
|
||||
26
variables.tf
26
variables.tf
@ -27,14 +27,31 @@ variable "agents_num" {
|
||||
default = 2
|
||||
}
|
||||
|
||||
variable "server_location" {
|
||||
variable "location" {
|
||||
description = "Default server location"
|
||||
default = "fsn1"
|
||||
}
|
||||
|
||||
variable "k3s_extra_args" {
|
||||
|
||||
variable "control_plane_server_type" {
|
||||
description = "Default control plane server type"
|
||||
default = "cx11"
|
||||
|
||||
}
|
||||
|
||||
variable "agent_server_type" {
|
||||
description = "Default agent server type"
|
||||
default = "cx21"
|
||||
}
|
||||
|
||||
variable "k3s_server_flags" {
|
||||
description = "Important flags to make our setup work"
|
||||
default = "--disable-cloud-controller --disable-network-policy --no-deploy=traefik --no-deploy=servicelb --disable local-storage --disable traefik --disable servicelb --kubelet-arg='cloud-provider=external' --no-flannel"
|
||||
default = "--disable-cloud-controller --disable-network-policy --disable=traefik --disable=servicelb --disable='local-storage' --kubelet-arg='cloud-provider=external' --flannel-backend=none --flannel-iface=ens10"
|
||||
}
|
||||
|
||||
variable "k3s_agent_flags" {
|
||||
description = "Important flags to make our setup work"
|
||||
default = "--kubelet-arg='cloud-provider=external' --flannel-iface=ens10"
|
||||
}
|
||||
|
||||
variable "initial_commands" {
|
||||
@ -46,6 +63,7 @@ variable "initial_commands" {
|
||||
"systemctl enable --now dnf-automatic.timer",
|
||||
"systemctl disable firewalld",
|
||||
"grubby --args='systemd.unified_cgroup_hierarchy=0' --update-kernel=ALL",
|
||||
"sleep 10; shutdown -r +0"
|
||||
"sleep 11; shutdown -r +0"
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user