Fist attempt to factor microOS install

agents.tf

@@ -1,65 +1,45 @@
-resource "hcloud_server" "agents" {
+module "agents" {
+  source = "./modules/host"
+
   count = var.agents_num
   name  = "k3s-agent-${count.index}"
 
-  image              = data.hcloud_image.linux.name
-  rescue             = "linux64"
-  server_type        = var.agent_server_type
-  location           = var.location
   ssh_keys           = [hcloud_ssh_key.k3s.id]
+  public_key         = var.public_key
+  private_key        = var.private_key
   firewall_ids       = [hcloud_firewall.k3s.id]
   placement_group_id = hcloud_placement_group.k3s.id
-
+  location           = var.location
+  network_id         = hcloud_network.k3s.id
+  ip                 = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index)
+  server_type        = var.control_plane_server_type
 
   labels = {
     "provisioner" = "terraform",
-    "engine"      = "k3s",
+    "engine"      = "k3s"
+  }
+
+  hcloud_token = var.hcloud_token
+}
+
+resource "null_resource" "agents" {
+  count = var.agents_num
+
+  triggers = {
+    agent_id = module.agents[count.index].id
   }
 
   connection {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = self.ipv4_address
+    host           = module.agents[count.index].ipv4_address
-  }
-
-  provisioner "file" {
-    content = templatefile("${path.module}/templates/config.ign.tpl", {
-      name           = self.name
-      ssh_public_key = local.ssh_public_key
-    })
-    destination = "/root/config.ign"
-  }
-
-  # Combustion script file to install k3s-selinux
-  provisioner "file" {
-    content     = local.combustion_script
-    destination = "/root/script"
-  }
-
-  # Install MicroOS
-  provisioner "remote-exec" {
-    inline = local.microOS_install_commands
-  }
-
-  # Issue a reboot command and wait for the node to reboot
-  provisioner "local-exec" {
-    command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
-  }
-  provisioner "local-exec" {
-    command = <<-EOT
-      until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
-      do
-        echo "Waiting for MicroOS to reboot and become available..."
-        sleep 3
-      done
-    EOT
   }
 
   # Generating k3s agent config file
   provisioner "file" {
     content = yamlencode({
-      node-name     = self.name
+      node-name     = module.agents[count.index].name
       server        = "https://${local.first_control_plane_network_ip}:6443"
       token         = random_password.k3s_token.result
       kubelet-arg   = "cloud-provider=external"
@@ -89,13 +69,8 @@ resource "hcloud_server" "agents" {
     ]
   }
 
-  network {
-    network_id = hcloud_network.k3s.id
-    ip         = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index)
-  }
-
   depends_on = [
-    hcloud_server.first_control_plane,
+    null_resource.first_control_plane,
     hcloud_network_subnet.k3s
   ]
 }

data.tf

@@ -15,8 +15,4 @@ data "github_release" "kured" {
   repository  = "kured"
   owner       = "weaveworks"
   retrieve_by = "latest"
-}
+}
-
-data "hcloud_image" "linux" {
-  name = local.hcloud_image_name
-}

kubeconfig.tf

@@ -1,17 +1,19 @@
 
 data "remote_file" "kubeconfig" {
   conn {
-    host        = hcloud_server.first_control_plane.ipv4_address
+    host        = module.first_control_plane.ipv4_address
     port        = 22
     user        = "root"
     private_key = local.ssh_private_key
     agent       = var.private_key == null
   }
   path = "/etc/rancher/k3s/k3s.yaml"
+
+  depends_on = [null_resource.first_control_plane]
 }
 
 locals {
-  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", hcloud_server.first_control_plane.ipv4_address)
+  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", module.first_control_plane.ipv4_address)
   kubeconfig_parsed   = yamldecode(local.kubeconfig_external)
   kubeconfig_data = {
     host                   = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"]

locals.tf

@@ -1,6 +1,5 @@
 locals {
   first_control_plane_network_ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 257)
-  hcloud_image_name              = "ubuntu-20.04"
 
   ssh_public_key = trimspace(file(var.public_key))
   # ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
@@ -18,39 +17,9 @@ locals {
   csi_version   = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag
   kured_version = data.github_release.kured.release_tag
 
-  microOS_install_commands = [
-    "set -ex",
-    "apt-get update",
-    "apt-get install -y aria2",
-    "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-kvm-and-xen.qcow2.meta4",
-    "qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda",
-    "sgdisk -e /dev/sda",
-    "parted -s /dev/sda resizepart 4 99%",
-    "parted -s /dev/sda mkpart primary ext2 99% 100%",
-    "partprobe /dev/sda && udevadm settle && fdisk -l /dev/sda",
-    "mount /dev/sda4 /mnt/ && btrfs filesystem resize max /mnt && umount /mnt",
-    "mke2fs -L ignition /dev/sda5",
-    "mount /dev/sda5 /mnt",
-    "mkdir /mnt/ignition",
-    "cp /root/config.ign /mnt/ignition/config.ign",
-    "mkdir /mnt/combustion",
-    "cp /root/script /mnt/combustion/script",
-    "umount /mnt"
-  ]
-
-  combustion_script = <<EOF
-#!/bin/bash
-# combustion: network
-rpm --import https://rpm.rancher.io/public.key
-zypper refresh
-zypper --gpg-auto-import-keys install -y https://rpm.rancher.io/k3s/stable/common/microos/noarch/k3s-selinux-0.4-1.sle.noarch.rpm
-udevadm settle
-    EOF
 
   common_commands_install_k3s = [
     "set -ex",
-    # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured
-    "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf",
     # prepare the k3s config directory
     "mkdir -p /etc/rancher/k3s",
     # move the config file into place

master.tf

@@ -1,63 +1,45 @@
-resource "hcloud_server" "first_control_plane" {
+module "first_control_plane" {
+  source = "./modules/host"
+
   name = "k3s-control-plane-0"
 
-  image              = data.hcloud_image.linux.name
-  rescue             = "linux64"
-  server_type        = var.control_plane_server_type
-  location           = var.location
   ssh_keys           = [hcloud_ssh_key.k3s.id]
+  public_key         = var.public_key
+  private_key        = var.private_key
   firewall_ids       = [hcloud_firewall.k3s.id]
   placement_group_id = hcloud_placement_group.k3s.id
+  location           = var.location
+  network_id         = hcloud_network.k3s.id
+  ip                 = local.first_control_plane_network_ip
+  server_type        = var.control_plane_server_type
 
   labels = {
     "provisioner" = "terraform",
     "engine"      = "k3s"
   }
 
+  hcloud_token = var.hcloud_token
+}
+
+resource "null_resource" "first_control_plane" {
+
+  triggers = {
+    first_control_plane_id = module.first_control_plane.id
+  }
+
+
   connection {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = self.ipv4_address
+    host           = module.first_control_plane.ipv4_address
-  }
-
-  provisioner "file" {
-    content = templatefile("${path.module}/templates/config.ign.tpl", {
-      name           = self.name
-      ssh_public_key = local.ssh_public_key
-    })
-    destination = "/root/config.ign"
-  }
-
-  # Combustion script file to install k3s-selinux
-  provisioner "file" {
-    content     = local.combustion_script
-    destination = "/root/script"
-  }
-
-  # Install MicroOS
-  provisioner "remote-exec" {
-    inline = local.microOS_install_commands
-  }
-
-  # Issue a reboot command and wait for the node to reboot
-  provisioner "local-exec" {
-    command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
-  }
-  provisioner "local-exec" {
-    command = <<-EOT
-      until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
-      do
-        echo "Waiting for MicroOS to reboot and become available..."
-        sleep 3
-      done
-    EOT
   }
 
   # Generating k3s master config file
   provisioner "file" {
     content = yamlencode({
-      node-name                = self.name
+      node-name                = module.first_control_plane.name
+      token                    = random_password.k3s_token.result
       cluster-init             = true
       disable-cloud-controller = true
       disable                  = ["servicelb", "local-storage"]
@@ -65,7 +47,7 @@ resource "hcloud_server" "first_control_plane" {
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = local.first_control_plane_network_ip
       advertise-address        = local.first_control_plane_network_ip
-      token                    = random_password.k3s_token.result
+      tls-san                  = local.first_control_plane_network_ip
       node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
       node-label               = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
     })
@@ -177,11 +159,6 @@ resource "hcloud_server" "first_control_plane" {
     ]
   }
 
-  network {
-    network_id = hcloud_network.k3s.id
-    ip         = local.first_control_plane_network_ip
-  }
-
   depends_on = [
     hcloud_network_subnet.k3s,
     hcloud_firewall.k3s

modules/host/locals.tf

@@ -0,0 +1,43 @@
+locals {
+  ssh_public_key = trimspace(file(var.public_key))
+  # ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
+  ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key))
+  # ssh_identity is not set if the private key is passed directly, but if ssh agent is used, the public key tells ssh agent which private key to use.
+  # For terraforms provisioner.connection.agent_identity, we need the public key as a string.
+  ssh_identity = var.private_key == null ? local.ssh_public_key : null
+  # ssh_identity_file is used for ssh "-i" flag, its the private key if that is set, or a public key file
+  # if an ssh agent is used.
+  ssh_identity_file = var.private_key == null ? var.public_key : var.private_key
+  # shared flags for ssh to ignore host keys, to use our ssh identity file for all connections during provisioning.
+  ssh_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${local.ssh_identity_file}"
+
+  microOS_install_commands = [
+    "set -ex",
+    "apt-get update",
+    "apt-get install -y aria2",
+    "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-kvm-and-xen.qcow2.meta4",
+    "qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda",
+    "sgdisk -e /dev/sda",
+    "parted -s /dev/sda resizepart 4 99%",
+    "parted -s /dev/sda mkpart primary ext2 99% 100%",
+    "partprobe /dev/sda && udevadm settle && fdisk -l /dev/sda",
+    "mount /dev/sda4 /mnt/ && btrfs filesystem resize max /mnt && umount /mnt",
+    "mke2fs -L ignition /dev/sda5",
+    "mount /dev/sda5 /mnt",
+    "mkdir /mnt/ignition",
+    "cp /root/config.ign /mnt/ignition/config.ign",
+    "mkdir /mnt/combustion",
+    "cp /root/script /mnt/combustion/script",
+    "umount /mnt"
+  ]
+
+  combustion_script = <<EOF
+#!/bin/bash
+# combustion: network
+rpm --import https://rpm.rancher.io/public.key
+zypper refresh
+zypper --gpg-auto-import-keys install -y https://rpm.rancher.io/k3s/stable/common/microos/noarch/k3s-selinux-0.4-1.sle.noarch.rpm
+udevadm settle
+EOF
+
+}

modules/host/main.tf

@@ -0,0 +1,71 @@
+resource "hcloud_server" "server" {
+  name = var.name
+
+  image              = "ubuntu-20.04"
+  rescue             = "linux64"
+  server_type        = var.server_type
+  location           = var.location
+  ssh_keys           = var.ssh_keys
+  firewall_ids       = var.firewall_ids
+  placement_group_id = var.placement_group_id
+
+
+  labels = var.labels
+
+  network {
+    network_id = var.network_id
+    ip         = var.ip
+  }
+
+  connection {
+    user           = "root"
+    private_key    = local.ssh_private_key
+    agent_identity = local.ssh_identity
+    host           = self.ipv4_address
+  }
+
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/config.ign.tpl", {
+      name           = self.name
+      ssh_public_key = local.ssh_public_key
+    })
+    destination = "/root/config.ign"
+  }
+
+  # Combustion script file to install k3s-selinux
+  provisioner "file" {
+    content     = local.combustion_script
+    destination = "/root/script"
+  }
+
+  # Install MicroOS
+  provisioner "remote-exec" {
+    inline = local.microOS_install_commands
+  }
+
+  # Issue a reboot command
+  provisioner "local-exec" {
+    command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
+  }
+
+  # Wait for MicroOS to reboot and be ready
+  provisioner "local-exec" {
+    command = <<-EOT
+      until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
+      do
+        echo "Waiting for MicroOS to reboot and become available..."
+        sleep 2
+      done
+    EOT
+  }
+
+  # Run the agent
+  provisioner "remote-exec" {
+    inline = [
+      # set the hostname in a persistent fashion
+      "hostnamectl set-hostname ${self.name}",
+      # Disable automatic reboot (after transactional updates), and configure the reboot method as kured
+      "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf"
+    ]
+  }
+}

modules/host/out.tf

@@ -0,0 +1,12 @@
+output "ipv4_address" {
+  value = hcloud_server.server.ipv4_address
+}
+
+
+output "name" {
+  value = hcloud_server.server.name
+}
+
+output "id" {
+  value = hcloud_server.server.id
+}

modules/host/templates/config.ign.tpl

@@ -0,0 +1,31 @@
+{
+  "ignition": {
+    "version": "3.0.0"
+  },
+  "passwd": {
+    "users": [
+      {
+        "name": "root",
+        "sshAuthorizedKeys": [
+          "${ssh_public_key}"
+        ]
+      }
+    ]
+  },
+  "storage": {
+    "files": [
+      {
+        "path": "/etc/sysconfig/network/ifcfg-eth1",
+        "mode": 420,
+        "overwrite": true,
+        "contents": { "source": "data:,BOOTPROTO%3D%27dhcp%27%0ASTARTMODE%3D%27auto%27" }
+      },
+      {
+        "path": "/etc/ssh/sshd_config.d/kube-hetzner.conf",
+        "mode": 420,
+        "overwrite": true,
+        "contents": { "source": "data:,PasswordAuthentication%20no%0AX11Forwarding%20no%0AMaxAuthTries%202%0AAllowTcpForwarding%20no%0AAllowAgentForwarding%20no%0AAuthorizedKeysFile%20.ssh%2Fauthorized_keys" }
+      }
+    ]
+  }
+}

modules/host/variables.tf

@@ -0,0 +1,65 @@
+variable "hcloud_token" {
+  description = "Hetzner Cloud API Token"
+  type        = string
+  sensitive   = true
+}
+
+variable "name" {
+  description = "Host name"
+  type        = string
+}
+
+variable "public_key" {
+  description = "SSH public Key."
+  type        = string
+}
+
+variable "private_key" {
+  description = "SSH private Key."
+  type        = string
+}
+
+variable "ssh_keys" {
+  description = "List of SSH key IDs"
+  type        = list(string)
+  nullable    = true
+}
+
+variable "firewall_ids" {
+  description = "Set of firewal IDs"
+  type        = set(number)
+  nullable    = true
+}
+
+variable "placement_group_id" {
+  description = "Placement group ID"
+  type        = number
+  nullable    = true
+}
+
+variable "labels" {
+  description = "Labels"
+  type        = map(any)
+  nullable    = true
+}
+
+variable "location" {
+  description = "The server location"
+  type        = string
+}
+
+variable "network_id" {
+  description = "The network or subnet id"
+  type        = number
+}
+
+variable "ip" {
+  description = "The IP"
+  type        = string
+  nullable    = true
+}
+
+variable "server_type" {
+  description = "The server type"
+  type        = string
+}

modules/host/versions.tf

@@ -0,0 +1,16 @@
+terraform {
+  required_providers {
+    hcloud = {
+      source  = "hetznercloud/hcloud"
+      version = ">= 1.0.0, < 2.0.0"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">= 2.0.0, < 3.0.0"
+    }
+    remote = {
+      source  = "tenstad/remote"
+      version = "~> 0.0.23"
+    }
+  }
+}

output.tf

@@ -1,10 +1,10 @@
 output "controlplanes_public_ip" {
-  value       = concat([hcloud_server.first_control_plane.ipv4_address], hcloud_server.control_planes.*.ipv4_address)
+  value       = concat([module.first_control_plane.ipv4_address], module.control_planes.*.ipv4_address)
   description = "The public IP addresses of the controlplane server."
 }
 
 output "agents_public_ip" {
-  value       = hcloud_server.agents.*.ipv4_address
+  value       = module.agents.*.ipv4_address
   description = "The public IP addresses of the agent server."
 }
 

servers.tf

@@ -1,69 +1,50 @@
-resource "hcloud_server" "control_planes" {
+module "control_planes" {
+  source = "./modules/host"
+
   count = var.servers_num - 1
   name  = "k3s-control-plane-${count.index + 1}"
 
-  image              = data.hcloud_image.linux.name
-  rescue             = "linux64"
-  server_type        = var.control_plane_server_type
-  location           = var.location
   ssh_keys           = [hcloud_ssh_key.k3s.id]
+  public_key         = var.public_key
+  private_key        = var.private_key
   firewall_ids       = [hcloud_firewall.k3s.id]
   placement_group_id = hcloud_placement_group.k3s.id
+  location           = var.location
+  network_id         = hcloud_network.k3s.id
+  ip                 = cidrhost(hcloud_network_subnet.k3s.ip_range, 258 + count.index)
+  server_type        = var.control_plane_server_type
 
   labels = {
     "provisioner" = "terraform",
-    "engine"      = "k3s",
+    "engine"      = "k3s"
+  }
+
+  hcloud_token = var.hcloud_token
+}
+
+resource "null_resource" "control_planes" {
+  count = var.servers_num - 1
+
+  triggers = {
+    control_plane_id = module.control_planes[count.index].id
   }
 
   connection {
     user           = "root"
     private_key    = local.ssh_private_key
     agent_identity = local.ssh_identity
-    host           = self.ipv4_address
+    host           = module.control_planes[count.index].ipv4_address
-  }
-
-  provisioner "file" {
-    content = templatefile("${path.module}/templates/config.ign.tpl", {
-      name           = self.name
-      ssh_public_key = local.ssh_public_key
-    })
-    destination = "/root/config.ign"
-  }
-
-  # Combustion script file to install k3s-selinux
-  provisioner "file" {
-    content     = local.combustion_script
-    destination = "/root/script"
-  }
-
-  # Install MicroOS
-  provisioner "remote-exec" {
-    inline = local.microOS_install_commands
-  }
-
-  # Issue a reboot command and wait for the node to reboot
-  provisioner "local-exec" {
-    command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
-  }
-  provisioner "local-exec" {
-    command = <<-EOT
-      until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
-      do
-        echo "Waiting for MicroOS to reboot and become available..."
-        sleep 3
-      done
-    EOT
   }
 
   # Generating k3s server config file
   provisioner "file" {
     content = yamlencode({
-      node-name                = self.name
+      node-name                = module.control_planes[count.index].name
       server                   = "https://${local.first_control_plane_network_ip}:6443"
       token                    = random_password.k3s_token.result
       cluster-init             = true
       disable-cloud-controller = true
-      disable                  = "servicelb, local-storage"
+      disable                  = ["servicelb", "local-storage"]
       flannel-iface            = "eth1"
       kubelet-arg              = "cloud-provider=external"
       node-ip                  = cidrhost(hcloud_network_subnet.k3s.ip_range, 258 + count.index)
@@ -95,13 +76,8 @@ resource "hcloud_server" "control_planes" {
     ]
   }
 
-  network {
-    network_id = hcloud_network.k3s.id
-    ip         = cidrhost(hcloud_network_subnet.k3s.ip_range, 258 + count.index)
-  }
-
   depends_on = [
-    hcloud_server.first_control_plane,
+    null_resource.first_control_plane,
     hcloud_network_subnet.k3s
   ]
 }