merged staging

README.md

@@ -38,6 +38,8 @@ _Please note that we are not affiliates of Hetzner; this is just an open-source
 - Possibility to have a single node cluster with a proper ingress controller.
 - Ability to add nodes and nodepools when the cluster is running.
 - Traefik ingress controller attached to a Hetzner load balancer with proxy protocol turned on.
+- Possibility to turn Longhorn on, and optionally also turn Hetzner CSI off.
+- Ability to switch to Calico as CNI, and Cilium can also be easily added.
 - Tons of flexible configuration options to suit all needs.
 
 _It uses Terraform to deploy as it's easy to use, and Hetzner provides a great [Hetzner Terraform Provider](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs)._
@@ -54,7 +56,7 @@ Follow those simple steps, and your world's cheapest Kube cluster will be up and
 
 First and foremost, you need to have a Hetzner Cloud account. You can sign up for free [here](https://hetzner.com/cloud/).
 
-Then you'll need to have [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli),  [kubectl](https://kubernetes.io/docs/tasks/tools/) cli, and [hcloud](<https://github.com/hetznercloud/cli>) the Hetzner cli. The easiest way is to use the [homebrew](https://brew.sh/) package manager to install them (available on Linux, Mac, and Windows Linux Subsystem).
+Then you'll need to have [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli),  [kubectl](https://kubernetes.io/docs/tasks/tools/) cli and [hcloud](<https://github.com/hetznercloud/cli>) the Hetzner cli. The easiest way is to use the [homebrew](https://brew.sh/) package manager to install them (available on Linux, Mac, and Windows Linux Subsystem).
 
 ```sh
 brew install terraform

agents.tf

@@ -13,6 +13,7 @@ module "agents" {
   location               = each.value.location
   server_type            = each.value.server_type
   ipv4_subnet_id         = hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0]].id
+  packages_to_install    = concat(var.enable_longhorn ? ["open-iscsi"] : [], [])
 
   private_ipv4 = cidrhost(hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0]].ip_range, each.value.index + 101)
 

control_planes.tf

@@ -13,6 +13,7 @@ module "control_planes" {
   location               = each.value.location
   server_type            = each.value.server_type
   ipv4_subnet_id         = hcloud_network_subnet.control_plane[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0]].id
+  packages_to_install    = concat(var.enable_longhorn ? ["open-iscsi"] : [], [])
 
   # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely
   # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough.

init.tf

@@ -80,18 +80,26 @@ resource "null_resource" "kustomization" {
     content = yamlencode({
       apiVersion = "kustomize.config.k8s.io/v1beta1"
       kind       = "Kustomization"
-      resources = concat([
+
-        "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${local.ccm_version}/ccm-networks.yaml",
+      resources = concat(
-        "https://raw.githubusercontent.com/hetznercloud/csi-driver/${local.csi_version}/deploy/kubernetes/hcloud-csi.yml",
+        [
-        "https://github.com/weaveworks/kured/releases/download/${local.kured_version}/kured-${local.kured_version}-dockerhub.yaml",
+          "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${local.ccm_version}/ccm-networks.yaml",
-        "https://raw.githubusercontent.com/rancher/system-upgrade-controller/master/manifests/system-upgrade-controller.yaml",
+          "https://github.com/weaveworks/kured/releases/download/${local.kured_version}/kured-${local.kured_version}-dockerhub.yaml",
-        ], local.is_single_node_cluster ? [] : var.traefik_enabled ? ["traefik_config.yaml"] : []
+          "https://raw.githubusercontent.com/rancher/system-upgrade-controller/master/manifests/system-upgrade-controller.yaml",
-      , var.cni_plugin == "calico" ? ["https://projectcalico.docs.tigera.io/manifests/calico.yaml"] : []),
+        ],
-      patchesStrategicMerge = concat([
+        var.disable_hetzner_csi ? [] : ["https://raw.githubusercontent.com/hetznercloud/csi-driver/${local.csi_version}/deploy/kubernetes/hcloud-csi.yml"],
-        file("${path.module}/kustomize/kured.yaml"),
+        var.enable_longhorn ? ["longhorn.yaml"] : [],
-        file("${path.module}/kustomize/system-upgrade-controller.yaml"),
+        local.is_single_node_cluster ? [] : var.traefik_enabled ? ["traefik_config.yaml"] : [],
-        "ccm.yaml"
+        var.cni_plugin == "calico" ? ["https://projectcalico.docs.tigera.io/manifests/calico.yaml"] : []
-      ], var.cni_plugin == "calico" ? ["calico.yaml"] : [])
+      ),
+      patchesStrategicMerge = concat(
+        [
+          file("${path.module}/kustomize/kured.yaml"),
+          file("${path.module}/kustomize/ccm.yaml"),
+          file("${path.module}/kustomize/system-upgrade-controller.yaml")
+        ],
+        var.cni_plugin == "calico" ? [file("${path.module}/kustomize/calico.yaml")] : []
+      )
     })
     destination = "/var/post_install/kustomization.yaml"
   }
@@ -142,6 +150,16 @@ resource "null_resource" "kustomization" {
     destination = "/var/post_install/plans.yaml"
   }
 
+  # Upload the Longhorn config
+  provisioner "file" {
+    content = templatefile(
+      "${path.module}/templates/longhorn.yaml.tpl",
+      {
+        disable_hetzner_csi = var.disable_hetzner_csi
+    })
+    destination = "/var/post_install/longhorn.yaml"
+  }
+
   # Deploy secrets, logging is automatically disabled due to sensitive variables
   provisioner "remote-exec" {
     inline = [
@@ -181,6 +199,7 @@ resource "null_resource" "kustomization" {
   }
 
   depends_on = [
-    null_resource.first_control_plane
+    null_resource.first_control_plane,
+    local_sensitive_file.kubeconfig
   ]
 }

locals.tf

@@ -13,7 +13,7 @@ locals {
 
   ccm_version   = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag
   csi_version   = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag
-  kured_version = data.github_release.kured.release_tag
+  kured_version = var.kured_version != null ? var.kured_version : data.github_release.kured.release_tag
 
   common_commands_install_k3s = [
     "set -ex",

modules/host/locals.tf

@@ -1,16 +1,23 @@
 locals {
+  # ssh public key
   ssh_public_key = trimspace(file(var.public_key))
   # ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
   ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key))
+
   # ssh_identity is not set if the private key is passed directly, but if ssh agent is used, the public key tells ssh agent which private key to use.
   # For terraforms provisioner.connection.agent_identity, we need the public key as a string.
   ssh_identity = var.private_key == null ? local.ssh_public_key : null
+
   # ssh_identity_file is used for ssh "-i" flag, its the private key if that is set, or a public key file
   # if an ssh agent is used.
   ssh_identity_file = var.private_key == null ? var.public_key : var.private_key
+
   # shared flags for ssh to ignore host keys, to use our ssh identity file for all connections during provisioning.
   ssh_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${local.ssh_identity_file}"
 
+  # Final list of packages to install
+  needed_packages = join(" ", concat(["k3s-selinux"], var.packages_to_install))
+
   # the hosts name with its unique suffix attached
   name = "${var.name}-${random_string.server.id}"
 }

modules/host/main.tf

@@ -65,11 +65,12 @@ resource "hcloud_server" "server" {
     EOT
   }
 
-  # Install k3s-selinux (compatible version)
+  # Install k3s-selinux (compatible version) and open-iscsi
   provisioner "remote-exec" {
-    inline = [
+    inline = [<<-EOT
-      "set -ex",
+      set -ex
-      "transactional-update shell <<< 'rpm --import https://rpm.rancher.io/public.key;zypper install -y https://github.com/k3s-io/k3s-selinux/releases/download/v0.5.stable.1/k3s-selinux-0.5-1.sle.noarch.rpm'"
+      transactional-update shell <<< "zypper --gpg-auto-import-keys install -y ${local.needed_packages}"
+      EOT
     ]
   }
 
@@ -84,6 +85,17 @@ resource "hcloud_server" "server" {
       done
     EOT
   }
+
+  # Enable open-iscsi
+  provisioner "remote-exec" {
+    inline = [<<-EOT
+      set -ex
+      if [[ $(systemctl list-units --all -t service --full --no-legend "iscsid.service" | sed 's/^\s*//g' | cut -f1 -d' ') == iscsid.service ]]; then
+        systemctl enable --now iscsid
+      fi
+      EOT
+    ]
+  }
 }
 
 resource "hcloud_server_network" "server" {

modules/host/templates/userdata.yaml.tpl

@@ -23,6 +23,17 @@ write_files:
     REBOOT_METHOD=kured
   path: /etc/transactional-update.conf
 
+# Create Rancher repo config
+- content: |
+    [rancher-k3s-common-stable]
+    name=Rancher K3s Common (stable)
+    baseurl=https://rpm.rancher.io/k3s/stable/common/microos/noarch
+    enabled=1
+    gpgcheck=1
+    repo_gpgcheck=0
+    gpgkey=https://rpm.rancher.io/public.key
+  path: /etc/zypp/repos.d/rancher-k3s-common.repo
+
 # Add ssh authorized keys
 ssh_authorized_keys:
 %{ for key in sshAuthorizedKeys ~}

modules/host/variables.tf

@@ -62,3 +62,9 @@ variable "server_type" {
   description = "The server type"
   type        = string
 }
+
+variable "packages_to_install" {
+  description = "Packages to install"
+  type        = list(string)
+  default     = []
+}

templates/longhorn.yaml.tpl

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: longhorn
+  namespace: kube-system
+spec:
+  chart: longhorn
+  repo: https://charts.longhorn.io
+  targetNamespace: longhorn
+  valuesContent: |-
+    defaultSettings:
+      defaultDataPath: /var/longhorn
+    persistence:
+      defaultFsType: ext4
+      defaultClassReplicaCount: 2
+      %{ if disable_hetzner_csi ~}defaultClass: true%{ else ~}defaultClass: false%{ endif ~}

terraform.tfvars.example

@@ -103,10 +103,18 @@ load_balancer_location = "fsn1"
 
 ### The following values are entirely optional
 
-# If you want to use a specific Hetzner CCM and CSI version, set them below; otherwise, leave them as-is for the latest versions
+# To use local storage on the nodes, you can enable Longhorn, default is "false".
+# enable_longhorn = true
+
+# To disable Hetzner CSI storage, you can set the following to true, default is "false".
+# disable_hetzner_csi = true
+
+# If you want to use a specific Hetzner CCM and CSI version, set them below; otherwise, leave them as-is for the latest versions.
 # hetzner_ccm_version = ""
 # hetzner_csi_version = ""
 
+# If you want to specify the Kured version, set it below - otherwise it'll use the latest version available
+# kured_version = ""
 
 # We give you the possibility to use letsencrypt directly with Traefik because it's an easy setup, however it's not optimal,
 # as the free version of Traefik causes a little bit of downtime when when the certificates get renewed. For proper SSL management,
@@ -127,7 +135,8 @@ load_balancer_location = "fsn1"
 # If you want to disable the automatic upgrade of k3s, you can set this to false. The default is "true".
 # automatically_upgrade_k3s = false
 
-# Allows you to specify either stable, latest, or testing (defaults to stable), see https://rancher.com/docs/k3s/latest/en/upgrades/basic/
+# Allows you to specify either stable, latest, testing or supported minor versions (defaults to stable)
+# see https://rancher.com/docs/k3s/latest/en/upgrades/basic/ and https://update.k3s.io/v1-release/channels
 # initial_k3s_channel = "latest"
 
 # The cluster name, by default "k3s"
@@ -167,8 +176,8 @@ load_balancer_location = "fsn1"
 # CAVEATS: Calico is not supported for single node setups, because of the following issue https://github.com/k3s-io/klipper-lb/issues/6.
 # cni_plugin = "calico"
 
-# If you want to disable the k3s default network policy controller, use this flag
+# If you want to disable the k3s default network policy controller, use this flag!
-# Calico overrides this value to true automatically
+# Calico overrides this value to true automatically, the default is "false".
 # disable_network_policy = true
 
 # If you want to disable the automatic use of placement group "spread". See https://docs.hetzner.com/cloud/placement-groups/overview/

variables.tf

@@ -65,6 +65,12 @@ variable "hetzner_csi_version" {
   description = "Version of Container Storage Interface driver for Hetzner Cloud"
 }
 
+variable "kured_version" {
+  type        = string
+  default     = null
+  description = "Version of Kured"
+}
+
 variable "traefik_enabled" {
   type        = bool
   default     = true
@@ -101,7 +107,7 @@ variable "initial_k3s_channel" {
   description = "Allows you to specify an initial k3s channel"
 
   validation {
-    condition     = contains(["stable", "latest", "testing"], var.initial_k3s_channel)
+    condition     = contains(["stable", "latest", "testing", "v1.16", "v1.17", "v1.18", "v1.19", "v1.20", "v1.21", "v1.22", "v1.23"], var.initial_k3s_channel)
     error_message = "The initial k3s channel must be one of stable, latest or testing."
   }
 }
@@ -157,3 +163,15 @@ variable "cni_plugin" {
   default     = "flannel"
   description = "CNI plugin for k3s"
 }
+
+variable "enable_longhorn" {
+  type        = bool
+  default     = false
+  description = "Enable Longhorn"
+}
+
+variable "disable_hetzner_csi" {
+  type        = bool
+  default     = false
+  description = "Disable hetzner csi driver"
+}