Merge pull request #66 from phaer/expose-kubeconfig-microos

Expose kubeconfig in outputs...

.gitignore

@@ -1,8 +1,6 @@
 .terraform*
 *.tfstate*
 crash.log
-hetzner/ccm/kustomization.yaml
-hetzner/csi/kustomization.yaml
 kured/kustomization.yaml
 kubeconfig.yaml
 kubeconfig.yaml-e

agents.tf

@@ -16,31 +16,24 @@ resource "hcloud_server" "agents" {
     "engine"      = "k3s",
   }
 
+  connection {
+    user           = "root"
+    private_key    = local.ssh_private_key
+    agent_identity = local.ssh_identity
+    host           = self.ipv4_address
+  }
+
   provisioner "file" {
     content = templatefile("${path.module}/templates/config.ign.tpl", {
       name           = self.name
       ssh_public_key = local.ssh_public_key
     })
     destination = "/root/config.ign"
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   # Install MicroOS
   provisioner "remote-exec" {
     inline = local.MicroOS_install_commands
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   # Issue a reboot command
@@ -67,13 +60,6 @@ resource "hcloud_server" "agents" {
       token  = random_password.k3s_token.result
     })
     destination = "/etc/rancher/k3s/agent.conf"
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   # Generating k3s agent config file
@@ -85,13 +71,6 @@ resource "hcloud_server" "agents" {
       node-ip       = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index)
     })
     destination = "/etc/rancher/k3s/config.yaml"
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   # Run the agent
@@ -112,13 +91,6 @@ resource "hcloud_server" "agents" {
         done
       EOT
     ]
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   network {

hetzner/csi/patch_latest.yaml

@@ -1,54 +0,0 @@
-kind: StatefulSet
-apiVersion: apps/v1
-metadata:
-  name: hcloud-csi-controller
-  namespace: kube-system
-spec:
-  template:
-    metadata:
-      labels:
-        app: hcloud-csi-controller
-    spec:
-      containers:
-        - name: csi-attacher
-          image: quay.io/k8scsi/csi-attacher:canary
-          imagePullPolicy: Always
-        - name: csi-resizer
-          image: quay.io/k8scsi/csi-resizer:canary
-          imagePullPolicy: Always
-        - name: csi-provisioner
-          image: quay.io/k8scsi/csi-provisioner:canary
-          imagePullPolicy: Always
-        - name: hcloud-csi-driver
-          image: hetznercloud/hcloud-csi-driver:latest
-          imagePullPolicy: Always
-        - name: liveness-probe
-          image: quay.io/k8scsi/livenessprobe:canary
-          imagePullPolicy: Always
-      volumes:
-        - name: socket-dir
-          emptyDir: {}
----
-kind: DaemonSet
-apiVersion: apps/v1
-metadata:
-  name: hcloud-csi-node
-  namespace: kube-system
-  labels:
-    app: hcloud-csi
-spec:
-  selector:
-    matchLabels:
-      app: hcloud-csi
-  template:
-    spec:
-      containers:
-        - name: csi-node-driver-registrar
-          image: quay.io/k8scsi/csi-node-driver-registrar:canary
-          imagePullPolicy: Always
-        - name: hcloud-csi-driver
-          image: hetznercloud/hcloud-csi-driver:latest
-          imagePullPolicy: Always
-        - name: liveness-probe
-          image: quay.io/k8scsi/livenessprobe:canary
-          imagePullPolicy: Always

kubeconfig.tf

@@ -0,0 +1,28 @@
+
+data "remote_file" "kubeconfig" {
+  conn {
+    host        = hcloud_server.first_control_plane.ipv4_address
+    port        = 22
+    user        = "root"
+    private_key = local.ssh_private_key
+    agent       = var.private_key == null
+  }
+  path = "/etc/rancher/k3s/k3s.yaml"
+}
+
+locals {
+  kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", hcloud_server.first_control_plane.ipv4_address)
+  kubeconfig_parsed   = yamldecode(local.kubeconfig_external)
+  kubeconfig_data = {
+    host                   = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"]
+    client_certificate     = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-certificate-data"])
+    client_key             = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-key-data"])
+    cluster_ca_certificate = base64decode(local.kubeconfig_parsed["clusters"][0]["cluster"]["certificate-authority-data"])
+  }
+}
+
+resource "local_file" "kubeconfig" {
+  sensitive_content = local.kubeconfig_external
+  filename          = "kubeconfig.yaml"
+  file_permission   = "600"
+}

locals.tf

@@ -1,7 +1,8 @@
 locals {
   first_control_plane_network_ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 2)
   hcloud_image_name              = "ubuntu-20.04"
-  ssh_public_key                 = trimspace(file(var.public_key))
+
+  ssh_public_key = trimspace(file(var.public_key))
   # ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
   ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key))
   # ssh_identity is not set if the private key is passed directly, but if ssh agent is used, the public key tells ssh agent which private key to use.
@@ -10,10 +11,15 @@ locals {
   # ssh_identity_file is used for ssh "-i" flag, its the private key if that is set, or a public key file
   # if an ssh agent is used.
   ssh_identity_file = var.private_key == null ? var.public_key : var.private_key
-
   # shared flags for ssh to ignore host keys, to use root and our ssh identity file for all connections during provisioning.
   ssh_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${local.ssh_identity_file}"
 
+  ccm_version   = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag
+  ccm_latest    = var.hetzner_ccm_containers_latest
+  csi_version   = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag
+  csi_latest    = var.hetzner_csi_containers_latest
+  kured_version = data.github_release.kured.release_tag
+
   MicroOS_install_commands = [
     "set -ex",
     "apt-get install -y aria2",

main.tf

@@ -144,49 +144,6 @@ resource "hcloud_firewall" "k3s" {
 
 }
 
-resource "local_file" "hetzner_ccm_config" {
-  content = templatefile("${path.module}/templates/hetzner_ccm.yaml.tpl", {
-    ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag
-    patch_name  = var.hetzner_ccm_containers_latest ? "patch_latest" : "patch"
-  })
-  filename             = "${path.module}/hetzner/ccm/kustomization.yaml"
-  file_permission      = "0644"
-  directory_permission = "0755"
-}
-
-resource "local_file" "hetzner_csi_config" {
-  content = templatefile("${path.module}/templates/hetzner_csi.yaml.tpl", {
-    csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag
-    patch_name  = var.hetzner_csi_containers_latest ? "patch_latest" : ""
-  })
-  filename             = "${path.module}/hetzner/csi/kustomization.yaml"
-  file_permission      = "0644"
-  directory_permission = "0755"
-}
-
-resource "local_file" "kured_config" {
-  content = templatefile("${path.module}/templates/kured.yaml.tpl", {
-    version = data.github_release.kured.release_tag
-  })
-  filename             = "${path.module}/kured/kustomization.yaml"
-  file_permission      = "0644"
-  directory_permission = "0755"
-}
-
-resource "local_file" "traefik_config" {
-  content = templatefile("${path.module}/templates/traefik_config.yaml.tpl", {
-    lb_disable_ipv6    = var.lb_disable_ipv6
-    lb_server_type     = var.lb_server_type
-    location           = var.location
-    traefik_acme_tls   = var.traefik_acme_tls
-    traefik_acme_email = var.traefik_acme_email
-  })
-  filename             = "${path.module}/templates/rendered/traefik_config.yaml"
-  file_permission      = "0644"
-  directory_permission = "0755"
-}
-
-
 resource "hcloud_placement_group" "k3s" {
   name = "k3s"
   type = "spread"

master.tf

@@ -14,31 +14,24 @@ resource "hcloud_server" "first_control_plane" {
     "engine"      = "k3s"
   }
 
+  connection {
+    user           = "root"
+    private_key    = local.ssh_private_key
+    agent_identity = local.ssh_identity
+    host           = self.ipv4_address
+  }
+
   provisioner "file" {
     content = templatefile("${path.module}/templates/config.ign.tpl", {
       name           = self.name
       ssh_public_key = local.ssh_public_key
     })
     destination = "/root/config.ign"
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   # Install MicroOS
   provisioner "remote-exec" {
     inline = local.MicroOS_install_commands
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   # Issue a reboot command
@@ -72,13 +65,6 @@ resource "hcloud_server" "first_control_plane" {
       node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
     })
     destination = "/etc/rancher/k3s/config.yaml"
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   # Run the first control plane
@@ -88,69 +74,87 @@ resource "hcloud_server" "first_control_plane" {
       "hostnamectl set-hostname ${self.name}",
       # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured
       "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf",
+      # prepare a directory for our post-installation kustomizations
+      "mkdir -p /tmp/post_install",
       # then we initiate the cluster
       "systemctl enable k3s-server",
+      # wait for k3s to get ready
       <<-EOT
-        until systemctl status k3s-server > /dev/null
+      timeout 120 bash <<EOF
-        do
+        until systemctl status k3s-server > /dev/null; do
           systemctl start k3s-server
           echo "Initiating the cluster..."
-          sleep 2
+          sleep 1
         done
+        until [ -e /etc/rancher/k3s/k3s.yaml ]; do
+          echo "Waiting for kubectl config..."
+          sleep 1
+        done
+        until [[ "\$(kubectl get --raw='/readyz')" == "ok" ]]; do
+          echo "Waiting for cluster to become ready..."
+          sleep 1
+        done
+      EOF
       EOT
     ]
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
-  # Get the Kubeconfig, and wait for the node to be available
+  # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured.
-  provisioner "local-exec" {
+  provisioner "file" {
-    command = <<-EOT
+    content = yamlencode({
-      until ssh -q ${local.ssh_args} root@${self.ipv4_address} [[ -f /etc/rancher/k3s/k3s.yaml ]]
+      apiVersion = "kustomize.config.k8s.io/v1beta1"
-      do
+      kind       = "Kustomization"
-        echo "Waiting for the k3s config file to be ready..."
+      resources = [
-        sleep 2
+        "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${local.ccm_version}/ccm-networks.yaml",
-      done
+        "https://raw.githubusercontent.com/hetznercloud/csi-driver/${local.csi_version}/deploy/kubernetes/hcloud-csi.yml",
-      scp ${local.ssh_args} root@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml
+        "https://github.com/weaveworks/kured/releases/download/${local.kured_version}/kured-${local.kured_version}-dockerhub.yaml",
-      sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml
+        "./traefik.yaml"
-      until kubectl get node ${self.name} --kubeconfig ${path.module}/kubeconfig.yaml 2> /dev/null || false
+      ]
-      do 
+      patchesStrategicMerge = [
-        echo "Waiting for the node to become available...";
+        file("${path.module}/patches/kured.yaml"),
-        sleep 2
+        local.ccm_latest ? file("${path.module}/patches/ccm_latest.yaml") : file("${path.module}/patches/ccm.yaml"),
-      done
+        local.csi_latest ? file("${path.module}/patches/csi_latest.yaml") : null,
-    EOT
+      ]
+    })
+    destination = "/tmp/post_install/kustomization.yaml"
   }
 
-  # Install the Hetzner CCM and CSI
+  # Upload traefik config
-  provisioner "local-exec" {
+  provisioner "file" {
-    command = <<-EOT
+    content = templatefile(
-      set -ex
+      "${path.module}/templates/traefik_config.yaml.tpl",
-      kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --kubeconfig ${path.module}/kubeconfig.yaml
+      {
-      kubectl apply -k ${dirname(local_file.hetzner_ccm_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml
+        lb_disable_ipv6    = var.lb_disable_ipv6
-      kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --kubeconfig ${path.module}/kubeconfig.yaml
+        lb_server_type     = var.lb_server_type
-      kubectl apply -k ${dirname(local_file.hetzner_csi_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml
+        location           = var.location
-    EOT
+        traefik_acme_tls   = var.traefik_acme_tls
+        traefik_acme_email = var.traefik_acme_email
+    })
+    destination = "/tmp/post_install/traefik.yaml"
   }
 
-  # Install Kured
+  # Deploy secrets, logging is automatically disabled due to sensitive variables
-  provisioner "local-exec" {
+  provisioner "remote-exec" {
-    command = <<-EOT
+    inline = [
-      set -ex
+      "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}",
-      kubectl -n kube-system apply -k ${dirname(local_file.kured_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml
+      "kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token}",
-    EOT
+    ]
   }
 
-  # Configure the Traefik ingress controller
+  # Deploy our post-installation kustomization
-  provisioner "local-exec" {
+  provisioner "remote-exec" {
-    command = <<-EOT
+    inline = [
-      set -ex
+      # This ugly hack is here, because terraform serializes the
-      kubectl apply -f ${local_file.traefik_config.filename} --kubeconfig ${path.module}/kubeconfig.yaml
+      # embedded yaml files with "- |2", when there is more than
-    EOT
+      # one yamldocument in the embedded file. Kustomize does not understand
+      # that syntax and tries to parse the blocks content as a file, resulting
+      # in weird errors. so gnu sed with funny escaping is used to
+      # replace lines like "- |3" by "- |" (yaml block syntax).
+      # due to indendation this should not changes the embedded
+      # manifests themselves
+      "sed -i 's/^- |[0-9]\\+$/- |/g' /tmp/post_install/kustomization.yaml",
+      "kubectl apply -k /tmp/post_install",
+    ]
   }
 
   network {

output.tf

@@ -7,3 +7,15 @@ output "agents_public_ip" {
   value       = hcloud_server.agents.*.ipv4_address
   description = "The public IP addresses of the agent server."
 }
+
+output "kubeconfig_file" {
+  value       = local.kubeconfig_external
+  description = "Kubeconfig file content with external IP address"
+  sensitive   = true
+}
+
+output "kubeconfig" {
+  description = "Structured kubeconfig data to supply to other providers"
+  value       = local.kubeconfig_data
+  sensitive   = true
+}

patches/ccm_latest.yaml

@@ -7,9 +7,7 @@ spec:
   template:
     spec:
       containers:
-        - image: hetznercloud/hcloud-cloud-controller-manager:latest
+        - name: hcloud-cloud-controller-manager
-          imagePullPolicy: Always
-          name: hcloud-cloud-controller-manager
           command:
             - "/bin/hcloud-cloud-controller-manager"
             - "--cloud-provider=hcloud"
@@ -17,3 +15,5 @@ spec:
             - "--allow-untagged-cloud"
             - "--allocate-node-cidrs=true"
             - "--cluster-cidr=10.42.0.0/16"
+          image: hetznercloud/hcloud-cloud-controller-manager:latest
+          imagePullPolicy: Always

patches/csi_latest.yaml

@@ -0,0 +1,54 @@
+  kind: StatefulSet
+  apiVersion: apps/v1
+  metadata:
+    name: hcloud-csi-controller
+    namespace: kube-system
+  spec:
+    template:
+      metadata:
+        labels:
+          app: hcloud-csi-controller
+      spec:
+        containers:
+          - name: csi-attacher
+            image: quay.io/k8scsi/csi-attacher:canary
+            imagePullPolicy: Always
+          - name: csi-resizer
+            image: quay.io/k8scsi/csi-resizer:canary
+            imagePullPolicy: Always
+          - name: csi-provisioner
+            image: quay.io/k8scsi/csi-provisioner:canary
+            imagePullPolicy: Always
+          - name: hcloud-csi-driver
+            image: hetznercloud/hcloud-csi-driver:latest
+            imagePullPolicy: Always
+          - name: liveness-probe
+            image: quay.io/k8scsi/livenessprobe:canary
+            imagePullPolicy: Always
+        volumes:
+          - name: socket-dir
+            emptyDir: {}
+  ---
+  kind: DaemonSet
+  apiVersion: apps/v1
+  metadata:
+    name: hcloud-csi-node
+    namespace: kube-system
+    labels:
+      app: hcloud-csi
+  spec:
+    selector:
+      matchLabels:
+        app: hcloud-csi
+    template:
+      spec:
+        containers:
+          - name: csi-node-driver-registrar
+            image: quay.io/k8scsi/csi-node-driver-registrar:canary
+            imagePullPolicy: Always
+          - name: hcloud-csi-driver
+            image: hetznercloud/hcloud-csi-driver:latest
+            imagePullPolicy: Always
+          - name: liveness-probe
+            image: quay.io/k8scsi/livenessprobe:canary
+            imagePullPolicy: Always

servers.tf

@@ -15,31 +15,24 @@ resource "hcloud_server" "control_planes" {
     "engine"      = "k3s",
   }
 
+  connection {
+    user           = "root"
+    private_key    = local.ssh_private_key
+    agent_identity = local.ssh_identity
+    host           = self.ipv4_address
+  }
+
   provisioner "file" {
     content = templatefile("${path.module}/templates/config.ign.tpl", {
       name           = self.name
       ssh_public_key = local.ssh_public_key
     })
     destination = "/root/config.ign"
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   # Install MicroOS
   provisioner "remote-exec" {
     inline = local.MicroOS_install_commands
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   # Issue a reboot command
@@ -75,13 +68,6 @@ resource "hcloud_server" "control_planes" {
       node-taint               = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
     })
     destination = "/etc/rancher/k3s/config.yaml"
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   # Run an other control plane server
@@ -102,13 +88,6 @@ resource "hcloud_server" "control_planes" {
         done
       EOT
     ]
-
-    connection {
-      user           = "root"
-      private_key    = local.ssh_private_key
-      agent_identity = local.ssh_identity
-      host           = self.ipv4_address
-    }
   }
 
   network {

templates/hetzner_ccm.yaml.tpl

@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml"
-
-patchesStrategicMerge:
-- ${patch_name}.yaml

templates/hetzner_csi.yaml.tpl

@@ -1,10 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml"
-
-%{ if patch_name != "" }
-patchesStrategicMerge:
-- ${patch_name}.yaml
-%{ endif }

templates/kured.yaml.tpl

@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-- "https://github.com/weaveworks/kured/releases/download/${version}/kured-${version}-dockerhub.yaml"
-
-patchesStrategicMerge:
-- patch.yaml

versions.tf

@@ -12,5 +12,9 @@ terraform {
       source  = "hashicorp/local"
       version = ">= 2.0.0, < 3.0.0"
     }
+    remote = {
+      source  = "tenstad/remote"
+      version = "~> 0.0.23"
+    }
   }
 }