Expose kubeconfig in outputs...

* To do so, we need to ensure that the generated kubeconfig is part of
  terraforms dependency graph. This has the additional benefit of not
  depending on local files anymore which should enable multi-user
  setups.

* This also means that we can't deploy CCM, CSI & Traefik from our local
  host, because we don't have kubeconfig.yaml locally while provisioning
  the control plane, only afterwards.

* So we just run kubectl apply on the control plane itself, after k3s is
  ready.

* To do so, we need to deploy all manifests. I've merged the patches
  into a single kustomization.yaml file, because that makes the
  deployment of those files to the control-plane server easier.

* we could also put the traefik config into the same kustomization file,
  which would save us one of the file provisioner blocks. I didn't want
  this PR to get any bigger, and will consider merging this config later
  on. kustomization.yaml is small enough that we could yamlencode() for
  it and store the patches in separate files again, not as
  inline-strings which is kind of ugly.

.gitignore

@@ -1,8 +1,6 @@
 .terraform*
 *.tfstate*
 crash.log
-hetzner/ccm/kustomization.yaml
-hetzner/csi/kustomization.yaml
 kured/kustomization.yaml
 kubeconfig.yaml
 kubeconfig.yaml-e

hetzner/ccm/patch.yaml

@@ -1,17 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: hcloud-cloud-controller-manager
-  namespace: kube-system
-spec:
-  template:
-    spec:
-      containers:
-        - name: hcloud-cloud-controller-manager
-          command:
-            - "/bin/hcloud-cloud-controller-manager"
-            - "--cloud-provider=hcloud"
-            - "--leader-elect=false"
-            - "--allow-untagged-cloud"
-            - "--allocate-node-cidrs=true"
-            - "--cluster-cidr=10.42.0.0/16"

hetzner/ccm/patch_latest.yaml

@@ -1,19 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: hcloud-cloud-controller-manager
-  namespace: kube-system
-spec:
-  template:
-    spec:
-      containers:
-        - image: hetznercloud/hcloud-cloud-controller-manager:latest
-          imagePullPolicy: Always
-          name: hcloud-cloud-controller-manager
-          command:
-            - "/bin/hcloud-cloud-controller-manager"
-            - "--cloud-provider=hcloud"
-            - "--leader-elect=false"
-            - "--allow-untagged-cloud"
-            - "--allocate-node-cidrs=true"
-            - "--cluster-cidr=10.42.0.0/16"

hetzner/csi/patch_latest.yaml

@@ -1,54 +0,0 @@
-kind: StatefulSet
-apiVersion: apps/v1
-metadata:
-  name: hcloud-csi-controller
-  namespace: kube-system
-spec:
-  template:
-    metadata:
-      labels:
-        app: hcloud-csi-controller
-    spec:
-      containers:
-        - name: csi-attacher
-          image: quay.io/k8scsi/csi-attacher:canary
-          imagePullPolicy: Always
-        - name: csi-resizer
-          image: quay.io/k8scsi/csi-resizer:canary
-          imagePullPolicy: Always
-        - name: csi-provisioner
-          image: quay.io/k8scsi/csi-provisioner:canary
-          imagePullPolicy: Always
-        - name: hcloud-csi-driver
-          image: hetznercloud/hcloud-csi-driver:latest
-          imagePullPolicy: Always
-        - name: liveness-probe
-          image: quay.io/k8scsi/livenessprobe:canary
-          imagePullPolicy: Always
-      volumes:
-        - name: socket-dir
-          emptyDir: {}
----
-kind: DaemonSet
-apiVersion: apps/v1
-metadata:
-  name: hcloud-csi-node
-  namespace: kube-system
-  labels:
-    app: hcloud-csi
-spec:
-  selector:
-    matchLabels:
-      app: hcloud-csi
-  template:
-    spec:
-      containers:
-        - name: csi-node-driver-registrar
-          image: quay.io/k8scsi/csi-node-driver-registrar:canary
-          imagePullPolicy: Always
-        - name: hcloud-csi-driver
-          image: hetznercloud/hcloud-csi-driver:latest
-          imagePullPolicy: Always
-        - name: liveness-probe
-          image: quay.io/k8scsi/livenessprobe:canary
-          imagePullPolicy: Always

kubeconfig.tf

@@ -0,0 +1,28 @@
+
+data "remote_file" "kubeconfig" {
+  conn {
+    host        = hcloud_server.first_control_plane.ipv4_address
+    port        = 22
+    user        = "root"
+    private_key = local.ssh_private_key
+    agent       = var.private_key == null
+  }
+  path = "/etc/rancher/k3s/k3s.yaml"
+}
+
+locals {
+  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", hcloud_server.first_control_plane.ipv4_address)
+  kubeconfig_parsed   = yamldecode(local.kubeconfig_external)
+  kubeconfig_data = {
+    host                   = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"]
+    client_certificate     = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-certificate-data"])
+    client_key             = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-key-data"])
+    cluster_ca_certificate = base64decode(local.kubeconfig_parsed["clusters"][0]["cluster"]["certificate-authority-data"])
+  }
+}
+
+resource "local_file" "kubeconfig" {
+  sensitive_content = local.kubeconfig_external
+  filename          = "kubeconfig.yaml"
+  file_permission   = "600"
+}

kured/patch.yaml

@@ -1,20 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kured
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kured
-  template:
-    metadata:
-      labels:
-        name: kured
-    spec:
-      serviceAccountName: kured
-      containers:
-        - name: kured
-          command:
-            - /usr/bin/kured
-            - --reboot-command=/usr/bin/systemctl reboot

locals.tf

@@ -30,4 +30,24 @@ locals {
     "cp /root/config.ign /mnt/ignition/config.ign",
     "umount /mnt"
   ]
+
+  post_install_kustomization = templatefile(
+    "${path.module}/templates/kustomization.yaml.tpl",
+    {
+      ccm_version   = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag
+      ccm_latest    = var.hetzner_ccm_containers_latest
+      csi_version   = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag
+      csi_latest    = var.hetzner_csi_containers_latest
+      kured_version = data.github_release.kured.release_tag
+  })
+
+  traefik_config = templatefile(
+    "${path.module}/templates/traefik_config.yaml.tpl",
+    {
+      lb_disable_ipv6    = var.lb_disable_ipv6
+      lb_server_type     = var.lb_server_type
+      location           = var.location
+      traefik_acme_tls   = var.traefik_acme_tls
+      traefik_acme_email = var.traefik_acme_email
+  })
 }

main.tf

@@ -144,49 +144,6 @@ resource "hcloud_firewall" "k3s" {
 
 }
 
-resource "local_file" "hetzner_ccm_config" {
-  content = templatefile("${path.module}/templates/hetzner_ccm.yaml.tpl", {
-    ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag
-    patch_name  = var.hetzner_ccm_containers_latest ? "patch_latest" : "patch"
-  })
-  filename             = "${path.module}/hetzner/ccm/kustomization.yaml"
-  file_permission      = "0644"
-  directory_permission = "0755"
-}
-
-resource "local_file" "hetzner_csi_config" {
-  content = templatefile("${path.module}/templates/hetzner_csi.yaml.tpl", {
-    csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag
-    patch_name  = var.hetzner_csi_containers_latest ? "patch_latest" : ""
-  })
-  filename             = "${path.module}/hetzner/csi/kustomization.yaml"
-  file_permission      = "0644"
-  directory_permission = "0755"
-}
-
-resource "local_file" "kured_config" {
-  content = templatefile("${path.module}/templates/kured.yaml.tpl", {
-    version = data.github_release.kured.release_tag
-  })
-  filename             = "${path.module}/kured/kustomization.yaml"
-  file_permission      = "0644"
-  directory_permission = "0755"
-}
-
-resource "local_file" "traefik_config" {
-  content = templatefile("${path.module}/templates/traefik_config.yaml.tpl", {
-    lb_disable_ipv6    = var.lb_disable_ipv6
-    lb_server_type     = var.lb_server_type
-    location           = var.location
-    traefik_acme_tls   = var.traefik_acme_tls
-    traefik_acme_email = var.traefik_acme_email
-  })
-  filename             = "${path.module}/templates/rendered/traefik_config.yaml"
-  file_permission      = "0644"
-  directory_permission = "0755"
-}
-
-
 resource "hcloud_placement_group" "k3s" {
   name = "k3s"
   type = "spread"

master.tf

@@ -81,6 +81,32 @@ resource "hcloud_server" "first_control_plane" {
     }
   }
 
+  # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured.
+  provisioner "file" {
+    content     = local.post_install_kustomization
+    destination = "/tmp/kustomization.yaml"
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
+  # Upload traefik config
+  provisioner "file" {
+    content     = local.traefik_config
+    destination = "/tmp/traefik.yaml"
+
+    connection {
+      user           = "root"
+      private_key    = local.ssh_private_key
+      agent_identity = local.ssh_identity
+      host           = self.ipv4_address
+    }
+  }
+
   # Run the first control plane
   provisioner "remote-exec" {
     inline = [
@@ -98,6 +124,14 @@ resource "hcloud_server" "first_control_plane" {
           sleep 2
         done
       EOT
+      , <<-EOT
+         timeout 120 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%%{http_code}'' curl -k https://localhost:6443/readyz)" != "200" ]]; do sleep 1; done'
+      EOT
+      , "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}",
+      "kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token}",
+      "kubectl apply -k /tmp/",
+      "kubectl apply -f /tmp/traefik.yaml",
+      "rm /tmp/traefik.yaml /tmp/kustomization.yaml"
     ]
 
     connection {
@@ -108,51 +142,6 @@ resource "hcloud_server" "first_control_plane" {
     }
   }
 
-  # Get the Kubeconfig, and wait for the node to be available
-  provisioner "local-exec" {
-    command = <<-EOT
-      until ssh -q ${local.ssh_args} root@${self.ipv4_address} [[ -f /etc/rancher/k3s/k3s.yaml ]]
-      do
-        echo "Waiting for the k3s config file to be ready..."
-        sleep 2
-      done
-      scp ${local.ssh_args} root@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml
-      sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml
-      until kubectl get node ${self.name} --kubeconfig ${path.module}/kubeconfig.yaml 2> /dev/null || false
-      do 
-        echo "Waiting for the node to become available...";
-        sleep 2
-      done
-    EOT
-  }
-
-  # Install the Hetzner CCM and CSI
-  provisioner "local-exec" {
-    command = <<-EOT
-      set -ex
-      kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --kubeconfig ${path.module}/kubeconfig.yaml
-      kubectl apply -k ${dirname(local_file.hetzner_ccm_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml
-      kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --kubeconfig ${path.module}/kubeconfig.yaml
-      kubectl apply -k ${dirname(local_file.hetzner_csi_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml
-    EOT
-  }
-
-  # Install Kured
-  provisioner "local-exec" {
-    command = <<-EOT
-      set -ex
-      kubectl -n kube-system apply -k ${dirname(local_file.kured_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml
-    EOT
-  }
-
-  # Configure the Traefik ingress controller
-  provisioner "local-exec" {
-    command = <<-EOT
-      set -ex
-      kubectl apply -f ${local_file.traefik_config.filename} --kubeconfig ${path.module}/kubeconfig.yaml
-    EOT
-  }
-
   network {
     network_id = hcloud_network.k3s.id
     ip         = local.first_control_plane_network_ip

output.tf

@@ -7,3 +7,15 @@ output "agents_public_ip" {
   value       = hcloud_server.agents.*.ipv4_address
   description = "The public IP addresses of the agent server."
 }
+
+output "kubeconfig_file" {
+  value       = local.kubeconfig_external
+  description = "Kubeconfig file content with external IP address"
+  sensitive   = true
+}
+
+output "kubeconfig" {
+  description = "Structured kubeconfig data to supply to other providers"
+  value       = local.kubeconfig_data
+  sensitive   = true
+}

templates/hetzner_ccm.yaml.tpl

@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml"
-
-patchesStrategicMerge:
-- ${patch_name}.yaml

templates/hetzner_csi.yaml.tpl

@@ -1,10 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml"
-
-%{ if patch_name != "" }
-patchesStrategicMerge:
-- ${patch_name}.yaml
-%{ endif }

templates/kured.yaml.tpl

@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-- "https://github.com/weaveworks/kured/releases/download/${version}/kured-${version}-dockerhub.yaml"
-
-patchesStrategicMerge:
-- patch.yaml

templates/kustomization.yaml.tpl

@@ -0,0 +1,128 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml"
+- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml"
+- "https://github.com/weaveworks/kured/releases/download/${kured_version}/kured-${kured_version}-dockerhub.yaml"
+
+patchesStrategicMerge:
+- |-
+  apiVersion: apps/v1
+  kind: DaemonSet
+  metadata:
+    name: kured
+    namespace: kube-system
+  spec:
+    selector:
+      matchLabels:
+        name: kured
+    template:
+      metadata:
+        labels:
+          name: kured
+      spec:
+        serviceAccountName: kured
+        containers:
+          - name: kured
+            command:
+              - /usr/bin/kured
+              - --reboot-command=/usr/bin/systemctl reboot
+- |-
+  apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: hcloud-cloud-controller-manager
+    namespace: kube-system
+  spec:
+    template:
+      spec:
+        containers:
+          - name: hcloud-cloud-controller-manager
+            command:
+              - "/bin/hcloud-cloud-controller-manager"
+              - "--cloud-provider=hcloud"
+              - "--leader-elect=false"
+              - "--allow-untagged-cloud"
+              - "--allocate-node-cidrs=true"
+              - "--cluster-cidr=10.42.0.0/16"
+%{ if ccm_latest ~}
+- |-
+  apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: hcloud-cloud-controller-manager
+    namespace: kube-system
+  spec:
+    template:
+      spec:
+        containers:
+          - name: hcloud-cloud-controller-manager
+            command:
+              - "/bin/hcloud-cloud-controller-manager"
+              - "--cloud-provider=hcloud"
+              - "--leader-elect=false"
+              - "--allow-untagged-cloud"
+              - "--allocate-node-cidrs=true"
+              - "--cluster-cidr=10.42.0.0/16"
+            image: hetznercloud/hcloud-cloud-controller-manager:latest
+            imagePullPolicy: Always
+%{ endif ~}
+%{ if csi_latest ~}
+- |-
+  kind: StatefulSet
+  apiVersion: apps/v1
+  metadata:
+    name: hcloud-csi-controller
+    namespace: kube-system
+  spec:
+    template:
+      metadata:
+        labels:
+          app: hcloud-csi-controller
+      spec:
+        containers:
+          - name: csi-attacher
+            image: quay.io/k8scsi/csi-attacher:canary
+            imagePullPolicy: Always
+          - name: csi-resizer
+            image: quay.io/k8scsi/csi-resizer:canary
+            imagePullPolicy: Always
+          - name: csi-provisioner
+            image: quay.io/k8scsi/csi-provisioner:canary
+            imagePullPolicy: Always
+          - name: hcloud-csi-driver
+            image: hetznercloud/hcloud-csi-driver:latest
+            imagePullPolicy: Always
+          - name: liveness-probe
+            image: quay.io/k8scsi/livenessprobe:canary
+            imagePullPolicy: Always
+        volumes:
+          - name: socket-dir
+            emptyDir: {}
+  ---
+  kind: DaemonSet
+  apiVersion: apps/v1
+  metadata:
+    name: hcloud-csi-node
+    namespace: kube-system
+    labels:
+      app: hcloud-csi
+  spec:
+    selector:
+      matchLabels:
+        app: hcloud-csi
+    template:
+      spec:
+        containers:
+          - name: csi-node-driver-registrar
+            image: quay.io/k8scsi/csi-node-driver-registrar:canary
+            imagePullPolicy: Always
+          - name: hcloud-csi-driver
+            image: hetznercloud/hcloud-csi-driver:latest
+            imagePullPolicy: Always
+          - name: liveness-probe
+            image: quay.io/k8scsi/livenessprobe:canary
+            imagePullPolicy: Always
+%{ endif ~}

versions.tf

@@ -12,5 +12,9 @@ terraform {
       source  = "hashicorp/local"
       version = ">= 2.0.0, < 3.0.0"
     }
+    remote = {
+      source  = "tenstad/remote"
+      version = "~> 0.0.23"
+    }
   }
 }