k3os ok

agents.tf

@@ -0,0 +1,60 @@
+resource "hcloud_server" "agents" {
+  count = var.agents_num
+  name  = "k3s-agent-${count.index}"
+
+  image        = data.hcloud_image.linux.name
+  rescue       = "linux64"
+  server_type  = var.agent_server_type
+  location     = var.location
+  ssh_keys     = [hcloud_ssh_key.default.id]
+  firewall_ids = [hcloud_firewall.k3s.id]
+
+
+  labels = {
+    "provisioner" = "terraform",
+    "engine"      = "k3s",
+    "k3s_upgrade" = "true"
+  }
+
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/agent.tpl", {
+      name           = self.name
+      ssh_public_key = local.ssh_public_key
+      k3s_token      = random_password.k3s_token.result
+      master_ip      = local.first_control_plane_network_ip
+      node_ip        = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index)
+    })
+    destination = "/tmp/config.yaml"
+
+    connection {
+      user        = "root"
+      private_key = file(var.private_key)
+      host        = self.ipv4_address
+    }
+  }
+
+
+  provisioner "remote-exec" {
+    inline = local.k3os_install_commands
+
+    connection {
+      user        = "root"
+      private_key = file(var.private_key)
+      host        = self.ipv4_address
+    }
+  }
+
+  provisioner "local-exec" {
+    command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 60"
+  }
+
+  network {
+    network_id = hcloud_network.k3s.id
+    ip         = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index)
+  }
+
+  depends_on = [
+    hcloud_server.first_control_plane,
+    hcloud_network_subnet.k3s
+  ]
+}

main.tf

@@ -88,23 +88,24 @@ resource "hcloud_firewall" "k3s" {
   }
 }
 
-data "hcloud_image" "linux" {
-  name = "ubuntu-20.04"
-}
 
 locals {
   first_control_plane_network_ip = cidrhost(hcloud_network.k3s.ip_range, 2)
-  name_master                    = "k3s-control-plane-0"
   ssh_public_key                 = trimspace(file(var.public_key))
+  hcloud_image_name = "ubuntu-20.04"
+
+  k3os_install_commands = [
+    "apt install -y grub-efi grub-pc-bin mtools xorriso",
+    "latest=$(curl -s https://api.github.com/repos/rancher/k3os/releases | jq '.[0].tag_name')",
+    "curl -Lo ./install.sh https://raw.githubusercontent.com/rancher/k3os/$(echo $latest | xargs)/install.sh",
+    "chmod +x ./install.sh",
+    "./install.sh --config /tmp/config.yaml /dev/sda https://github.com/rancher/k3os/releases/download/$(echo $latest | xargs)/k3os-amd64.iso",
+    "shutdown -r +1",
+    "sleep 3",
+    "exit 0"
+  ]
 }
 
-data "template_file" "master" {
+data "hcloud_image" "linux" {
-  template = file("${path.module}/templates/master.tpl")
+  name = local.hcloud_image_name
-
-  vars = {
-    name           = local.name_master
-    ssh_public_key = local.ssh_public_key
-    k3s_token      = random_password.k3s_token.result
-    ip             = local.first_control_plane_network_ip
-  }
 }

master.tf

@@ -1,5 +1,5 @@
 resource "hcloud_server" "first_control_plane" {
-  name = local.name_master
+  name = "k3s-control-plane-0"
 
   image        = data.hcloud_image.linux.name
   rescue       = "linux64"
@@ -14,7 +14,12 @@ resource "hcloud_server" "first_control_plane" {
   }
 
   provisioner "file" {
-    content     = data.template_file.master.rendered
+    content = templatefile("${path.module}/templates/master.tpl", {
+      name           = self.name
+      ssh_public_key = local.ssh_public_key
+      k3s_token      = random_password.k3s_token.result
+      master_ip      = local.first_control_plane_network_ip
+    })
     destination = "/tmp/config.yaml"
 
     connection {
@@ -26,14 +31,7 @@ resource "hcloud_server" "first_control_plane" {
 
 
   provisioner "remote-exec" {
-    inline = [
+    inline = local.k3os_install_commands
-      "apt install -y grub-efi grub-pc-bin mtools xorriso",
-      "latest=$(curl -s https://api.github.com/repos/rancher/k3os/releases | jq '.[0].tag_name')",
-      "curl -Lo ./install.sh https://raw.githubusercontent.com/rancher/k3os/$(echo $latest | xargs)/install.sh",
-      "chmod +x ./install.sh",
-      "./install.sh --config /tmp/config.yaml /dev/sda https://github.com/rancher/k3os/releases/download/$(echo $latest | xargs)/k3os-amd64.iso",
-      "shutdown -r now"
-    ]
 
     connection {
       user        = "root"
@@ -44,7 +42,7 @@ resource "hcloud_server" "first_control_plane" {
 
   provisioner "local-exec" {
     command = <<-EOT
-      ping ${self.ipv4_address} | grep --line-buffered "bytes from" | head -1 && sleep 60 && scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.private_key} rancher@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml
+      sleep 60 && ping ${self.ipv4_address} | grep --line-buffered "bytes from" | head -1 && sleep 60 && scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.private_key} rancher@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml
       sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml
     EOT
   }

output.tf

@@ -0,0 +1,9 @@
+output "controlplanes_public_ip" {
+  value       = concat([hcloud_server.first_control_plane.ipv4_address], hcloud_server.control_planes.*.ipv4_address)
+  description = "The public IP addresses of the controlplane server."
+}
+
+output "agents_public_ip" {
+  value       = hcloud_server.agents.*.ipv4_address
+  description = "The public IP addresses of the agent server."
+}

servers.tf

@@ -0,0 +1,59 @@
+resource "hcloud_server" "control_planes" {
+  count = var.servers_num - 1
+  name  = "k3s-control-plane-${count.index + 1}"
+
+  image        = data.hcloud_image.linux.name
+  rescue       = "linux64"
+  server_type  = var.control_plane_server_type
+  location     = var.location
+  ssh_keys     = [hcloud_ssh_key.default.id]
+  firewall_ids = [hcloud_firewall.k3s.id]
+
+  labels = {
+    "provisioner" = "terraform",
+    "engine"      = "k3s",
+    "k3s_upgrade" = "true"
+  }
+
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/server.tpl", {
+      name           = self.name
+      ssh_public_key = local.ssh_public_key
+      k3s_token      = random_password.k3s_token.result
+      master_ip      = local.first_control_plane_network_ip
+      node_ip        = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)
+    })
+    destination = "/tmp/config.yaml"
+
+    connection {
+      user        = "root"
+      private_key = file(var.private_key)
+      host        = self.ipv4_address
+    }
+  }
+
+
+  provisioner "remote-exec" {
+    inline = local.k3os_install_commands
+
+    connection {
+      user        = "root"
+      private_key = file(var.private_key)
+      host        = self.ipv4_address
+    }
+  }
+
+  provisioner "local-exec" {
+    command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 60"
+  }
+
+  network {
+    network_id = hcloud_network.k3s.id
+    ip         = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)
+  }
+
+  depends_on = [
+    hcloud_server.first_control_plane,
+    hcloud_network_subnet.k3s
+  ]
+}

templates/agent.tpl

@@ -1,19 +1,27 @@
 ssh_authorized_keys:
 - ${ssh_public_key}
 hostname: ${name}
+boot_cmd:
+- |
+  echo 'auto eth0
+  iface eth0 inet dhcp
+  auto eth1
+  iface eth1 inet dhcp' > /etc/network/interfaces
+- rc-update del connman boot
+- rc-update add networking boot
+- rc-update add ntpd default
+run_cmd:
+- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1"
 k3os:
   k3s_args:
-  - server
+  - agent
-  --node-ip=${ip} 
+  - "--server"
-  --advertise-address=${ip} 
+  - "https://${master_ip}:6443"
-  --bind-address=${ip} 
+  - "--node-ip"
-  --tls-san=${ip}
+  - "${node_ip}"
-  --disable-cloud-controller
+  - "--kubelet-arg"
-  --disable-network-policy
+  - "cloud-provider=external"
-  --disable=traefik
+  - "--flannel-iface=eth1"
-  --disable=servicelb
-  --disable='local-storage'
-  --kubelet-arg='cloud-provider=external'
   token: ${k3s_token}
   ntp_servers:
   - 0.de.pool.ntp.org
@@ -21,9 +29,5 @@ k3os:
   dns_nameservers:
   - 8.8.8.8
   - 1.1.1.1
-  - 8.8.4.4
-  - 1.0.0.1
   - 2001:4860:4860::8888
   - 2606:4700:4700::1111
-  - 2001:4860:4860::8844
-  - 2606:4700:4700::1001

templates/master.tpl

@@ -10,6 +10,8 @@ boot_cmd:
 - rc-update del connman boot
 - rc-update add networking boot
 - rc-update add ntpd default
+run_cmd:
+- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1"
 k3os:
   k3s_args:
   - server
@@ -20,11 +22,11 @@ k3os:
   - "--disable=local-storage"
   - "--flannel-iface=eth1"
   - "--node-ip"
-  - "${ip}"
+  - "${master_ip}"
   - "--advertise-address"
-  - "${ip}"
+  - "${master_ip}"
   - "--tls-san"
-  - "${ip}"
+  - "${master_ip}"
   - "--kubelet-arg"
   - "cloud-provider=external"
   token: ${k3s_token}

templates/server.tpl

@@ -0,0 +1,41 @@
+ssh_authorized_keys:
+- ${ssh_public_key}
+hostname: ${name}
+boot_cmd:
+- |
+  echo 'auto eth0
+  iface eth0 inet dhcp
+  auto eth1
+  iface eth1 inet dhcp' > /etc/network/interfaces
+- rc-update del connman boot
+- rc-update add networking boot
+- rc-update add ntpd default
+run_cmd:
+- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1"
+k3os:
+  k3s_args:
+  - server
+  - "--server"
+  - "https://${master_ip}:6443"
+  - "--disable-cloud-controller"
+  - "--disable=traefik"
+  - "--disable=servicelb"
+  - "--disable=local-storage"
+  - "--flannel-iface=eth1"
+  - "--node-ip"
+  - "${node_ip}"
+  - "--advertise-address"
+  - "${node_ip}"
+  - "--tls-san"
+  - "${node_ip}"
+  - "--kubelet-arg"
+  - "cloud-provider=external"
+  token: ${k3s_token}
+  ntp_servers:
+  - 0.de.pool.ntp.org
+  - 1.de.pool.ntp.org
+  dns_nameservers:
+  - 8.8.8.8
+  - 1.1.1.1
+  - 2001:4860:4860::8888
+  - 2606:4700:4700::1111