initial k3s on MicroOS on Hetzner ok

agents.tf

@@ -0,0 +1,110 @@
+resource "hcloud_server" "agents" {
+  count = var.agents_num
+  name  = "k3s-agent-${count.index}"
+
+  image              = data.hcloud_image.linux.name
+  rescue             = "linux64"
+  server_type        = var.agent_server_type
+  location           = var.location
+  ssh_keys           = [hcloud_ssh_key.k3s.id]
+  firewall_ids       = [hcloud_firewall.k3s.id]
+  placement_group_id = hcloud_placement_group.k3s_placement_group.id
+
+
+  labels = {
+    "provisioner" = "terraform",
+    "engine"      = "k3s",
+  }
+
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/config.ign.tpl", {
+      name           = self.name
+      ssh_public_key = local.ssh_public_key
+    })
+    destination = "/root/config.ign"
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
+  # Install MicroOS
+  provisioner "remote-exec" {
+    inline = local.MicroOS_install_commands
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
+  # Wait for MicroOS to reboot and be ready
+  provisioner "local-exec" {
+    command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 30"
+  }
+
+  # Generating and uploading the angent.conf file
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/agent.conf.tpl", {
+      server_url = "https://${local.first_control_plane_network_ip}:6443"
+      node_token = random_password.k3s_token.result
+    })
+    destination = "/etc/rancher/k3s/agent.conf"
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
+  # Generating k3s server config file
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/agent_config.yaml.tpl", {
+      node_ip   = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index)
+      node_name = self.name
+    })
+    destination = "/etc/rancher/k3s/config.yaml"
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
+  # Run the agent
+  provisioner "remote-exec" {
+    inline = [
+      "set -ex",
+      # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured
+      "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf",
+      # then turn on k3s and join the cluster
+      "systemctl --now enable k3s-agent",
+    ]
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
+  network {
+    network_id = hcloud_network.k3s.id
+    ip         = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index)
+  }
+
+  depends_on = [
+    hcloud_server.first_control_plane,
+    hcloud_network_subnet.k3s
+  ]
+}

locals.tf

@@ -11,10 +11,10 @@ locals {
   # if an ssh agent is used.
   ssh_identity_file = var.private_key == null ? var.public_key : var.private_key
 
-  microOS_install_commands = [
+  MicroOS_install_commands = [
     "set -ex",
     "apt-get install -y aria2",
-    "aria2c  --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-k3s-kvm-and-xen.qcow2.meta4",
+    "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-k3s-kvm-and-xen.qcow2.meta4",
     "qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*k3s.*qcow2$') /dev/sda",
     "sgdisk -e /dev/sda",
     "partprobe /dev/sda",

master.tf

@@ -14,23 +14,6 @@ resource "hcloud_server" "first_control_plane" {
     "engine"      = "k3s"
   }
 
-  provisioner "file" {
-    content = templatefile("${path.module}/templates/master.tpl", {
-      name           = self.name
-      ssh_public_key = local.ssh_public_key
-      k3s_token      = random_password.k3s_token.result
-      master_ip      = local.first_control_plane_network_ip
-    })
-    destination = "/tmp/config.yaml"
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
-  }
-
   provisioner "file" {
     content = templatefile("${path.module}/templates/config.ign.tpl", {
       name           = self.name
@@ -46,9 +29,9 @@ resource "hcloud_server" "first_control_plane" {
     }
   }
 
-  # Install k3os
+  # Install MicroOS
   provisioner "remote-exec" {
-    inline = local.microOS_install_commands
+    inline = local.MicroOS_install_commands
 
     connection {
       user           = "root"
@@ -57,18 +40,61 @@ resource "hcloud_server" "first_control_plane" {
       host           = self.ipv4_address
     }
   }
-  /*
-  # Wait for MicroOS to be ready and fetch kubeconfig.yaml
+
+  # Wait for MicroOS to reboot and be ready
+  provisioner "local-exec" {
+    command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 30"
+  }
+
+  # Generating k3s master config file
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/master_config.yaml.tpl", {
+      node_ip   = local.first_control_plane_network_ip
+      token     = random_password.k3s_token.result
+      node_name = self.name
+    })
+    destination = "/etc/rancher/k3s/config.yaml"
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+  # Run the first control plane
+  provisioner "remote-exec" {
+    inline = [
+      "set -ex",
+      # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured
+      "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf",
+      # then we initiate the cluster
+      "systemctl --now enable k3s-server",
+    ]
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
+  # Get the Kubeconfig, and wait for the node to be available
   provisioner "local-exec" {
     command = <<-EOT
-      sleep 60 && ping ${self.ipv4_address} | grep --line-buffered "bytes from" | head -1 && sleep 100 && scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${local.ssh_identity_file} rancher@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml
+      set -ex
+      sleep 30
+      scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${local.ssh_identity_file} root@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml
       sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml
+      sleep 30
     EOT
   }
 
-  # Install Hetzner CCM and CSI
+  # Install the Hetzner CCM and CSI
   provisioner "local-exec" {
     command = <<-EOT
+      set -ex
       kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --kubeconfig ${path.module}/kubeconfig.yaml
       kubectl apply -k ${dirname(local_file.hetzner_ccm_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml
       kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --kubeconfig ${path.module}/kubeconfig.yaml
@@ -80,7 +106,7 @@ resource "hcloud_server" "first_control_plane" {
   provisioner "local-exec" {
     command = "kubectl apply -f ${local_file.traefik_config.filename} --kubeconfig ${path.module}/kubeconfig.yaml"
   }
-*/
+
   network {
     network_id = hcloud_network.k3s.id
     ip         = local.first_control_plane_network_ip

servers.tf

@@ -0,0 +1,95 @@
+resource "hcloud_server" "control_planes" {
+  count = var.servers_num - 1
+  name  = "k3s-control-plane-${count.index + 1}"
+
+  image              = data.hcloud_image.linux.name
+  rescue             = "linux64"
+  server_type        = var.control_plane_server_type
+  location           = var.location
+  ssh_keys           = [hcloud_ssh_key.k3s.id]
+  firewall_ids       = [hcloud_firewall.k3s.id]
+  placement_group_id = hcloud_placement_group.k3s_placement_group.id
+
+  labels = {
+    "provisioner" = "terraform",
+    "engine"      = "k3s",
+  }
+
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/config.ign.tpl", {
+      name           = self.name
+      ssh_public_key = local.ssh_public_key
+    })
+    destination = "/root/config.ign"
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
+  # Install MicroOS
+  provisioner "remote-exec" {
+    inline = local.MicroOS_install_commands
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
+  # Wait for MicroOS to reboot and be ready
+  provisioner "local-exec" {
+    command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 30"
+  }
+
+  # Generating k3s server config file
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/server_config.yaml.tpl", {
+      first_control_plane_url = "https://${local.first_control_plane_network_ip}:6443"
+      node_ip                 = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)
+      token                   = random_password.k3s_token.result
+      node_name               = self.name
+    })
+    destination = "/etc/rancher/k3s/config.yaml"
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
+  # Run the other control plane
+  provisioner "remote-exec" {
+    inline = [
+      "set -ex",
+      # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured
+      "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf",
+      # then we initiate the cluster
+      "systemctl --now enable k3s-server",
+    ]
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
+  network {
+    network_id = hcloud_network.k3s.id
+    ip         = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)
+  }
+
+  depends_on = [
+    hcloud_server.first_control_plane,
+    hcloud_network_subnet.k3s
+  ]
+}

temp/agents.tf

@@ -1,63 +0,0 @@
-resource "hcloud_server" "agents" {
-  count = var.agents_num
-  name  = "k3s-agent-${count.index}"
-
-  image              = data.hcloud_image.linux.name
-  rescue             = "linux64"
-  server_type        = var.agent_server_type
-  location           = var.location
-  ssh_keys           = [hcloud_ssh_key.k3s.id]
-  firewall_ids       = [hcloud_firewall.k3s.id]
-  placement_group_id = hcloud_placement_group.k3s_placement_group.id
-
-
-  labels = {
-    "provisioner" = "terraform",
-    "engine"      = "k3s",
-    "k3s_upgrade" = "true"
-  }
-
-  provisioner "file" {
-    content = templatefile("${path.module}/templates/agent.tpl", {
-      name           = self.name
-      ssh_public_key = local.ssh_public_key
-      k3s_token      = random_password.k3s_token.result
-      master_ip      = local.first_control_plane_network_ip
-      node_ip        = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index)
-    })
-    destination = "/tmp/config.yaml"
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
-  }
-
-
-  provisioner "remote-exec" {
-    inline = local.microOS_install_commands
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
-  }
-
-  provisioner "local-exec" {
-    command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 100"
-  }
-
-  network {
-    network_id = hcloud_network.k3s.id
-    ip         = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index)
-  }
-
-  depends_on = [
-    hcloud_server.first_control_plane,
-    hcloud_network_subnet.k3s
-  ]
-}

temp/servers.tf

@@ -1,62 +0,0 @@
-resource "hcloud_server" "control_planes" {
-  count = var.servers_num - 1
-  name  = "k3s-control-plane-${count.index + 1}"
-
-  image              = data.hcloud_image.linux.name
-  rescue             = "linux64"
-  server_type        = var.control_plane_server_type
-  location           = var.location
-  ssh_keys           = [hcloud_ssh_key.k3s.id]
-  firewall_ids       = [hcloud_firewall.k3s.id]
-  placement_group_id = hcloud_placement_group.k3s_placement_group.id
-
-  labels = {
-    "provisioner" = "terraform",
-    "engine"      = "k3s",
-    "k3s_upgrade" = "true"
-  }
-
-  provisioner "file" {
-    content = templatefile("${path.module}/templates/server.tpl", {
-      name           = self.name
-      ssh_public_key = local.ssh_public_key
-      k3s_token      = random_password.k3s_token.result
-      master_ip      = local.first_control_plane_network_ip
-      node_ip        = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)
-    })
-    destination = "/tmp/config.yaml"
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
-  }
-
-
-  provisioner "remote-exec" {
-    inline = local.microOS_install_commands
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
-  }
-
-  provisioner "local-exec" {
-    command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 100"
-  }
-
-  network {
-    network_id = hcloud_network.k3s.id
-    ip         = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)
-  }
-
-  depends_on = [
-    hcloud_server.first_control_plane,
-    hcloud_network_subnet.k3s
-  ]
-}

templates/agent.conf.tpl

@@ -0,0 +1,3 @@
+SERVER_URL="${server_url}"
+NODE_TOKEN="${node_token}"
+AGENT_OPTS=

templates/agent.tpl

@@ -1,33 +0,0 @@
-ssh_authorized_keys:
-- ${ssh_public_key}
-hostname: ${name}
-boot_cmd:
-- |
-  echo 'auto eth0
-  iface eth0 inet dhcp
-  auto eth1
-  iface eth1 inet dhcp' > /etc/network/interfaces
-- rc-update del connman boot
-- rc-update add networking boot
-- rc-update add ntpd default
-run_cmd:
-- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1"
-k3os:
-  k3s_args:
-  - agent
-  - "--server"
-  - "https://${master_ip}:6443"
-  - "--node-ip"
-  - "${node_ip}"
-  - "--kubelet-arg"
-  - "cloud-provider=external"
-  - "--flannel-iface=eth1"
-  token: ${k3s_token}
-  ntp_servers:
-  - 0.de.pool.ntp.org
-  - 1.de.pool.ntp.org
-  dns_nameservers:
-  - 8.8.8.8
-  - 1.1.1.1
-  - 2001:4860:4860::8888
-  - 2606:4700:4700::1111

templates/agent_config.yaml.tpl

@@ -0,0 +1,4 @@
+node-ip: ${node_ip}
+kubelet-arg: "cloud-provider=external"
+flannel-iface: eth1
+node-name: ${node_name}

templates/master.tpl

@@ -1,39 +0,0 @@
-ssh_authorized_keys:
-- ${ssh_public_key}
-hostname: ${name}
-boot_cmd:
-- |
-  echo 'auto eth0
-  iface eth0 inet dhcp
-  auto eth1
-  iface eth1 inet dhcp' > /etc/network/interfaces
-- rc-update del connman boot
-- rc-update add networking boot
-- rc-update add ntpd default
-run_cmd:
-- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1"
-k3os:
-  k3s_args:
-  - server
-  - "--cluster-init"
-  - "--disable-cloud-controller"
-  - "--disable=servicelb"
-  - "--disable=local-storage"
-  - "--flannel-iface=eth1"
-  - "--node-ip"
-  - "${master_ip}"
-  - "--advertise-address"
-  - "${master_ip}"
-  - "--tls-san"
-  - "${master_ip}"
-  - "--kubelet-arg"
-  - "cloud-provider=external"
-  token: ${k3s_token}
-  ntp_servers:
-  - 0.de.pool.ntp.org
-  - 1.de.pool.ntp.org
-  dns_nameservers:
-  - 8.8.8.8
-  - 1.1.1.1
-  - 2001:4860:4860::8888
-  - 2606:4700:4700::1111

templates/master_config.yaml.tpl

@@ -0,0 +1,11 @@
+cluster-init: true
+disable-cloud-controller: true
+disable: servicelb 
+disable: local-storage
+flannel-iface: eth1
+node-ip: ${node_ip}
+advertise-address: ${node_ip}
+tls-san: ${node_ip}
+kubelet-arg: "cloud-provider=external"
+token: ${token}
+node-name: ${node_name}

templates/server.tpl

@@ -1,42 +0,0 @@
-ssh_authorized_keys:
-- ${ssh_public_key}
-hostname: ${name}
-boot_cmd:
-- |
-  echo 'auto eth0
-  iface eth0 inet dhcp
-  auto eth1
-  iface eth1 inet dhcp' > /etc/network/interfaces
-- rc-update del connman boot
-- rc-update add networking boot
-- rc-update add ntpd default
-run_cmd:
-- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1"
-k3os:
-  k3s_args:
-  - server
-  - "--server"
-  - "https://${master_ip}:6443"
-  - "--disable-cloud-controller"
-  - "--disable-network-policy"
-  - "--disable=traefik"
-  - "--disable=servicelb"
-  - "--disable=local-storage"
-  - "--flannel-iface=eth1"
-  - "--node-ip"
-  - "${node_ip}"
-  - "--advertise-address"
-  - "${node_ip}"
-  - "--tls-san"
-  - "${node_ip}"
-  - "--kubelet-arg"
-  - "cloud-provider=external"
-  token: ${k3s_token}
-  ntp_servers:
-  - 0.de.pool.ntp.org
-  - 1.de.pool.ntp.org
-  dns_nameservers:
-  - 8.8.8.8
-  - 1.1.1.1
-  - 2001:4860:4860::8888
-  - 2606:4700:4700::1111

templates/server_config.yaml.tpl

@@ -0,0 +1,11 @@
+server: ${first_control_plane_url}
+disable-cloud-controller: true
+disable: servicelb 
+disable: local-storage
+flannel-iface: eth1
+node-ip: ${node_ip}
+advertise-address: ${node_ip}
+tls-san: ${node_ip}
+kubelet-arg: "cloud-provider=external"
+token: ${token}
+node-name: ${node_name}