k3s-install ready for testing

agents.tf

@@ -33,15 +33,13 @@ resource "hcloud_server" "agents" {
 
   # Install MicroOS
   provisioner "remote-exec" {
-    inline = local.MicroOS_install_commands
+    inline = local.microOS_install_commands
   }
 
-  # Issue a reboot command
+  # Issue a reboot command and wait for the node to reboot
   provisioner "local-exec" {
     command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
   }
-
-  # Wait for MicroOS to reboot and be ready
   provisioner "local-exec" {
     command = <<-EOT
       until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
@@ -52,16 +50,6 @@ resource "hcloud_server" "agents" {
     EOT
   }
 
-
-  # Generating and uploading the agent.conf file
-  provisioner "file" {
-    content = templatefile("${path.module}/templates/agent.conf.tpl", {
-      server = "https://${local.first_control_plane_network_ip}:6443"
-      token  = random_password.k3s_token.result
-    })
-    destination = "/etc/rancher/k3s/agent.conf"
-  }
-
   # Generating k3s agent config file
   provisioner "file" {
     content = yamlencode({
@@ -69,30 +57,60 @@ resource "hcloud_server" "agents" {
       kubelet-arg   = "cloud-provider=external"
       flannel-iface = "eth1"
       node-ip       = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index)
+      node-label    = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
     })
-    destination = "/etc/rancher/k3s/config.yaml"
+    destination = "/tmp/config.yaml"
   }
 
-  # Run the agent
+  # Install k3s agent
   provisioner "remote-exec" {
     inline = [
-      # set the hostname in a persistent fashion
+      "set -ex",
-      "hostnamectl set-hostname ${self.name}",
       # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured
       "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf",
-      # then we start k3s agent and join the cluster
+      # prepare the k3s config directory
-      "systemctl enable k3s-agent",
+      "mkdir -p /etc/rancher/k3s",
+      # move the config file into place
+      "mv /tmp/config.yaml /etc/rancher/k3s/config.yaml",
+      # install k3s
       <<-EOT
-        until systemctl status k3s-agent > /dev/null
+      INSTALL_K3S_SKIP_START=true \
-        do
+      K3S_URL=${"https://${local.first_control_plane_network_ip}:6443"} \
-          systemctl start k3s-agent
+      K3S_TOKEN=${random_password.k3s_token.result} \
-          echo "Starting k3s-agent and joining the cluster..."
+      curl -sfL https://get.k3s.io | sh -
-          sleep 2
-        done
       EOT
     ]
   }
 
+  # Issue a reboot command and wait for the node to reboot
+  provisioner "local-exec" {
+    command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
+  }
+  provisioner "local-exec" {
+    command = <<-EOT
+      until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
+      do
+        echo "Waiting for MicroOS to reboot and become available..."
+        sleep 2
+      done
+    EOT
+  }
+
+  # Upon reboot verify that k3s agent starts correctly
+  provisioner "remote-exec" {
+    inline = [
+      <<-EOT
+      timeout 120 bash <<EOF
+        until systemctl status k3s-agent > /dev/null; do
+          echo "Waiting for the k3s agent to start..."
+          sleep 1
+        done
+      EOF
+      EOT
+    ]
+  }
+
+
   network {
     network_id = hcloud_network.k3s.id
     ip         = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index)

locals.tf

@@ -18,7 +18,7 @@ locals {
   csi_version   = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag
   kured_version = data.github_release.kured.release_tag
 
-  MicroOS_install_commands = [
+  microOS_install_commands = [
     "set -ex",
     "apt-get install -y aria2",
     "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-kvm-and-xen.qcow2.meta4",
@@ -34,4 +34,16 @@ locals {
     "cp /root/config.ign /mnt/ignition/config.ign",
     "umount /mnt"
   ]
+
+  install_k3s_server = [
+    "set -ex",
+    # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured
+    "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf",
+    # prepare the k3s config directory
+    "mkdir -p /etc/rancher/k3s",
+    # move the config file into place
+    "mv /tmp/config.yaml /etc/rancher/k3s/config.yaml",
+    # install k3s
+    "INSTALL_K3S_SKIP_START=true curl -sfL https://get.k3s.io | sh -",
+  ]
 }

master.tf

@@ -31,15 +31,13 @@ resource "hcloud_server" "first_control_plane" {
 
   # Install MicroOS
   provisioner "remote-exec" {
-    inline = local.MicroOS_install_commands
+    inline = local.microOS_install_commands
   }
 
-  # Issue a reboot command
+  # Issue a reboot command and wait for the node to reboot
   provisioner "local-exec" {
     command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
   }
-
-  # Wait for MicroOS to reboot and be ready
   provisioner "local-exec" {
     command = <<-EOT
       until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
@@ -63,27 +61,42 @@ resource "hcloud_server" "first_control_plane" {
       advertise-address        = local.first_control_plane_network_ip
       token                    = random_password.k3s_token.result
       node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
+      node-label               = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
     })
-    destination = "/etc/rancher/k3s/config.yaml"
+    destination = "/tmp/config.yaml"
   }
 
-  /*  # Run the first control plane
+
+
+  # Install k3s server
+  provisioner "remote-exec" {
+    inline = local.install_k3s_server
+  }
+
+  # Issue a reboot command and wait for the node to reboot
+  provisioner "local-exec" {
+    command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
+  }
+  provisioner "local-exec" {
+    command = <<-EOT
+      until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
+      do
+        echo "Waiting for MicroOS to reboot and become available..."
+        sleep 2
+      done
+    EOT
+  }
+
+  # Upon reboot verify that the k3s server is starts, and wait for k3s to be ready to receive commands
   provisioner "remote-exec" {
     inline = [
-      # set the hostname in a persistent fashion
+      # prepare the post_install directory
-      "hostnamectl set-hostname ${self.name}",
-      # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured
-      "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf",
-      # prepare a directory for our post-installation kustomizations
       "mkdir -p /tmp/post_install",
-      # then we initiate the cluster
+      # wait for k3s to become ready
-      "systemctl enable k3s-server",
-      # wait for k3s to get ready
       <<-EOT
       timeout 120 bash <<EOF
-        until systemctl status k3s-server > /dev/null; do
+        until systemctl status k3s > /dev/null; do
-          systemctl start k3s-server
+          echo "Waiting for the k3s server to start..."
-          echo "Initiating the cluster..."
           sleep 1
         done
         until [ -e /etc/rancher/k3s/k3s.yaml ]; do
@@ -108,7 +121,8 @@ resource "hcloud_server" "first_control_plane" {
         "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${local.ccm_version}/ccm-networks.yaml",
         "https://raw.githubusercontent.com/hetznercloud/csi-driver/${local.csi_version}/deploy/kubernetes/hcloud-csi.yml",
         "https://github.com/weaveworks/kured/releases/download/${local.kured_version}/kured-${local.kured_version}-dockerhub.yaml",
-        "./traefik.yaml"
+        "https://raw.githubusercontent.com/rancher/system-upgrade-controller/master/manifests/system-upgrade-controller.yaml",
+        "./traefik.yaml",
       ]
       patchesStrategicMerge = [
         file("${path.module}/patches/kured.yaml"),
@@ -132,9 +146,20 @@ resource "hcloud_server" "first_control_plane" {
     destination = "/tmp/post_install/traefik.yaml"
   }
 
+  # Upload the system upgrade controller plans config
+  provisioner "file" {
+    content = templatefile(
+      "${path.module}/templates/plans.yaml.tpl",
+      {
+        channel = var.k3s_upgrade_channel
+    })
+    destination = "/tmp/post_install/plans.yaml"
+  }
+
   # Deploy secrets, logging is automatically disabled due to sensitive variables
   provisioner "remote-exec" {
     inline = [
+      "set -ex",
       "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}",
       "kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token}",
     ]
@@ -143,6 +168,7 @@ resource "hcloud_server" "first_control_plane" {
   # Deploy our post-installation kustomization
   provisioner "remote-exec" {
     inline = [
+      "set -ex",
       # This ugly hack is here, because terraform serializes the
       # embedded yaml files with "- |2", when there is more than
       # one yamldocument in the embedded file. Kustomize does not understand
@@ -153,8 +179,10 @@ resource "hcloud_server" "first_control_plane" {
       # manifests themselves
       "sed -i 's/^- |[0-9]\\+$/- |/g' /tmp/post_install/kustomization.yaml",
       "kubectl apply -k /tmp/post_install",
+      "echo 'Waiting for the system-upgrade-controller deployment to become available...' && kubectl -n system-upgrade wait --for=condition=available --timeout=300s deployment/system-upgrade-controller",
+      "kubectl apply -f /tmp/post_install/plans.yaml"
     ]
-  } */
+  }
 
   network {
     network_id = hcloud_network.k3s.id

servers.tf

@@ -32,15 +32,13 @@ resource "hcloud_server" "control_planes" {
 
   # Install MicroOS
   provisioner "remote-exec" {
-    inline = local.MicroOS_install_commands
+    inline = local.microOS_install_commands
   }
 
-  # Issue a reboot command
+  # Issue a reboot command and wait for the node to reboot
   provisioner "local-exec" {
     command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
   }
-
-  # Wait for MicroOS to reboot and be ready
   provisioner "local-exec" {
     command = <<-EOT
       until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
@@ -66,27 +64,41 @@ resource "hcloud_server" "control_planes" {
       tls-san                  = cidrhost(hcloud_network_subnet.k3s.ip_range, 3 + count.index)
       token                    = random_password.k3s_token.result
       node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
+      node-label               = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
     })
-    destination = "/etc/rancher/k3s/config.yaml"
+    destination = "/tmp/config.yaml"
   }
 
-  # Run an other control plane server
+  # Install k3s server
   provisioner "remote-exec" {
-    inline = [
+    inline = local.install_k3s_server
-      # set the hostname in a persistent fashion
+  }
-      "hostnamectl set-hostname ${self.name}",
+
-      # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured
+  # Issue a reboot command and wait for the node to reboot
-      "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf",
+  provisioner "local-exec" {
-      # then then we start k3s in server mode and join the cluster
+    command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
-      "systemctl enable k3s-server",
+  }
-      <<-EOT
+  provisioner "local-exec" {
-        until systemctl status k3s-server > /dev/null
+    command = <<-EOT
+      until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
       do
-          systemctl start k3s-server
+        echo "Waiting for MicroOS to reboot and become available..."
-          echo "Waiting on other 'learning' control planes, patience is the mother of all virtues..."
         sleep 2
       done
     EOT
+  }
+
+  # Upon reboot verify that the k3s server starts correctly
+  provisioner "remote-exec" {
+    inline = [
+      <<-EOT
+      timeout 120 bash <<EOF
+        until systemctl status k3s > /dev/null; do
+          echo "Waiting for the k3s server to start..."
+          sleep 1
+        done
+      EOF
+      EOT
     ]
   }
 

templates/agent.conf.tpl

@@ -1,3 +0,0 @@
-SERVER_URL="${server}"
-NODE_TOKEN="${token}"
-AGENT_OPTS=""

templates/plans.yaml.tpl

@@ -0,0 +1,50 @@
+# Doc: https://rancher.com/docs/k3s/latest/en/upgrades/automated/
+# agent plan
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: k3s-agent
+  namespace: system-upgrade
+  labels:
+    k3s_upgrade: agent
+spec:
+  concurrency: 1
+  channel: https://update.k3s.io/v1-release/channels/${channel}
+  nodeSelector:
+    matchExpressions:
+      - {key: k3s_upgrade, operator: Exists}
+      - {key: k3s_upgrade, operator: NotIn, values: ["disabled", "false"]}
+      - {key: node-role.kubernetes.io/master, operator: NotIn, values: ["true"]}
+  serviceAccountName: system-upgrade
+  prepare:
+    image: rancher/k3s-upgrade
+    args: ["prepare", "k3s-server"]
+  drain:
+    force: true
+    skipWaitForDeleteTimeout: 60
+  upgrade:
+    image: rancher/k3s-upgrade
+---
+# server plan
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: k3s-server
+  namespace: system-upgrade
+  labels:
+    k3s_upgrade: server
+spec:
+  concurrency: 1
+  channel: https://update.k3s.io/v1-release/channels/${channel}
+  nodeSelector:
+    matchExpressions:
+      - {key: k3s_upgrade, operator: Exists}
+      - {key: k3s_upgrade, operator: NotIn, values: ["disabled", "false"]}
+      - {key: node-role.kubernetes.io/master, operator: In, values: ["true"]}
+  tolerations:
+    - {key: node-role.kubernetes.io/master, effect: NoSchedule, operator: Exists}
+    - {key: CriticalAddonsOnly, effect: NoExecute, operator: Exists}
+  serviceAccountName: system-upgrade
+  cordon: true
+  upgrade:
+    image: rancher/k3s-upgrade

terraform.tfvars.example

@@ -29,3 +29,12 @@ agents_num                = 2
 
 # If you want to allow non-control-plane workloads to run on the control-plane nodes set "true" below. The default is "false".
 # allow_scheduling_on_control_plane = true
+
+# If you want to disable automatic upgrade of k3s (stable channel), you can set this to false, default is "true".
+# automatically_upgrade_k3s = false
+
+# If you would like to specify the k3s upgrade channel from the get go, you can do so, the default is "stable".
+# For a list of available channels, see https://rancher.com/docs/k3s/latest/en/upgrades/basic/ and https://update.k3s.io/v1-release/channels
+# k3s_upgrade_channel = "latest"
+
+

variables.tf

@@ -84,3 +84,16 @@ variable "allow_scheduling_on_control_plane" {
   default     = false
   description = "Whether to allow non-control-plane workloads to run on the control-plane nodes"
 }
+
+variable "k3s_upgrade_channel" {
+  type        = string
+  default     = "stable"
+  description = "Allows you to specify the k3s upgrade channel"
+}
+
+variable "automatically_upgrade_k3s" {
+  type        = bool
+  default     = true
+  description = "Whether to automatically upgrade k3s based on the selected channel"
+}
+