From 9dc4952665392e2d7a1f9dccbb7a9acd5d2ec8f2 Mon Sep 17 00:00:00 2001 From: phaer Date: Fri, 11 Feb 2022 12:45:03 +0100 Subject: [PATCH] Expose kubeconfig in outputs... * To do so, we need to ensure that the generated kubeconfig is part of terraforms dependency graph. This has the additional benefit of not depending on local files anymore which should enable multi-user setups. * This also means that we can't deploy CCM, CSI & Traefik from our local host, because we don't have kubeconfig.yaml locally while provisioning the control plane, only afterwards. * So we just run kubectl apply on the control plane itself, after k3s is ready. * To do so, we need to deploy all manifests. I've merged the patches into a single kustomization.yaml file, because that makes the deployment of those files to the control-plane server easier. * we could also put the traefik config into the same kustomization file, which would save us one of the file provisioner blocks. I didn't want this PR to get any bigger, and will consider merging this config later on. kustomization.yaml is small enough that we could yamlencode() for it and store the patches in separate files again, not as inline-strings which is kind of ugly. --- .gitignore | 2 - hetzner/ccm/patch.yaml | 17 ---- hetzner/ccm/patch_latest.yaml | 19 ----- hetzner/csi/patch_latest.yaml | 54 ------------- kubeconfig.tf | 28 +++++++ kured/patch.yaml | 20 ----- locals.tf | 20 +++++ main.tf | 43 ----------- master.tf | 79 ++++++++----------- output.tf | 12 +++ templates/hetzner_ccm.yaml.tpl | 8 -- templates/hetzner_csi.yaml.tpl | 10 --- templates/kured.yaml.tpl | 8 -- templates/kustomization.yaml.tpl | 128 +++++++++++++++++++++++++++++++ versions.tf | 4 + 15 files changed, 226 insertions(+), 226 deletions(-) delete mode 100644 hetzner/ccm/patch.yaml delete mode 100644 hetzner/ccm/patch_latest.yaml delete mode 100644 hetzner/csi/patch_latest.yaml create mode 100644 kubeconfig.tf delete mode 100644 kured/patch.yaml delete mode 100644 templates/hetzner_ccm.yaml.tpl delete mode 100644 templates/hetzner_csi.yaml.tpl delete mode 100644 templates/kured.yaml.tpl create mode 100644 templates/kustomization.yaml.tpl diff --git a/.gitignore b/.gitignore index 64e5d1b..23f449f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ .terraform* *.tfstate* crash.log -hetzner/ccm/kustomization.yaml -hetzner/csi/kustomization.yaml kured/kustomization.yaml kubeconfig.yaml kubeconfig.yaml-e diff --git a/hetzner/ccm/patch.yaml b/hetzner/ccm/patch.yaml deleted file mode 100644 index 179b775..0000000 --- a/hetzner/ccm/patch.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hcloud-cloud-controller-manager - namespace: kube-system -spec: - template: - spec: - containers: - - name: hcloud-cloud-controller-manager - command: - - "/bin/hcloud-cloud-controller-manager" - - "--cloud-provider=hcloud" - - "--leader-elect=false" - - "--allow-untagged-cloud" - - "--allocate-node-cidrs=true" - - "--cluster-cidr=10.42.0.0/16" \ No newline at end of file diff --git a/hetzner/ccm/patch_latest.yaml b/hetzner/ccm/patch_latest.yaml deleted file mode 100644 index a631620..0000000 --- a/hetzner/ccm/patch_latest.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hcloud-cloud-controller-manager - namespace: kube-system -spec: - template: - spec: - containers: - - image: hetznercloud/hcloud-cloud-controller-manager:latest - imagePullPolicy: Always - name: hcloud-cloud-controller-manager - command: - - "/bin/hcloud-cloud-controller-manager" - - "--cloud-provider=hcloud" - - "--leader-elect=false" - - "--allow-untagged-cloud" - - "--allocate-node-cidrs=true" - - "--cluster-cidr=10.42.0.0/16" \ No newline at end of file diff --git a/hetzner/csi/patch_latest.yaml b/hetzner/csi/patch_latest.yaml deleted file mode 100644 index 743d655..0000000 --- a/hetzner/csi/patch_latest.yaml +++ /dev/null @@ -1,54 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: hcloud-csi-controller - namespace: kube-system -spec: - template: - metadata: - labels: - app: hcloud-csi-controller - spec: - containers: - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:canary - imagePullPolicy: Always - - name: csi-resizer - image: quay.io/k8scsi/csi-resizer:canary - imagePullPolicy: Always - - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:canary - imagePullPolicy: Always - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:latest - imagePullPolicy: Always - - name: liveness-probe - image: quay.io/k8scsi/livenessprobe:canary - imagePullPolicy: Always - volumes: - - name: socket-dir - emptyDir: {} ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: hcloud-csi-node - namespace: kube-system - labels: - app: hcloud-csi -spec: - selector: - matchLabels: - app: hcloud-csi - template: - spec: - containers: - - name: csi-node-driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:canary - imagePullPolicy: Always - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:latest - imagePullPolicy: Always - - name: liveness-probe - image: quay.io/k8scsi/livenessprobe:canary - imagePullPolicy: Always \ No newline at end of file diff --git a/kubeconfig.tf b/kubeconfig.tf new file mode 100644 index 0000000..07db0d2 --- /dev/null +++ b/kubeconfig.tf @@ -0,0 +1,28 @@ + +data "remote_file" "kubeconfig" { + conn { + host = hcloud_server.first_control_plane.ipv4_address + port = 22 + user = "root" + private_key = local.ssh_private_key + agent = var.private_key == null + } + path = "/etc/rancher/k3s/k3s.yaml" +} + +locals { + kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", hcloud_server.first_control_plane.ipv4_address) + kubeconfig_parsed = yamldecode(local.kubeconfig_external) + kubeconfig_data = { + host = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"] + client_certificate = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-certificate-data"]) + client_key = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-key-data"]) + cluster_ca_certificate = base64decode(local.kubeconfig_parsed["clusters"][0]["cluster"]["certificate-authority-data"]) + } +} + +resource "local_file" "kubeconfig" { + sensitive_content = local.kubeconfig_external + filename = "kubeconfig.yaml" + file_permission = "600" +} diff --git a/kured/patch.yaml b/kured/patch.yaml deleted file mode 100644 index bf72a0c..0000000 --- a/kured/patch.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kured - namespace: kube-system -spec: - selector: - matchLabels: - name: kured - template: - metadata: - labels: - name: kured - spec: - serviceAccountName: kured - containers: - - name: kured - command: - - /usr/bin/kured - - --reboot-command=/usr/bin/systemctl reboot diff --git a/locals.tf b/locals.tf index aaf8aac..98eb8d6 100644 --- a/locals.tf +++ b/locals.tf @@ -30,4 +30,24 @@ locals { "cp /root/config.ign /mnt/ignition/config.ign", "umount /mnt" ] + + post_install_kustomization = templatefile( + "${path.module}/templates/kustomization.yaml.tpl", + { + ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag + ccm_latest = var.hetzner_ccm_containers_latest + csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag + csi_latest = var.hetzner_csi_containers_latest + kured_version = data.github_release.kured.release_tag + }) + + traefik_config = templatefile( + "${path.module}/templates/traefik_config.yaml.tpl", + { + lb_disable_ipv6 = var.lb_disable_ipv6 + lb_server_type = var.lb_server_type + location = var.location + traefik_acme_tls = var.traefik_acme_tls + traefik_acme_email = var.traefik_acme_email + }) } diff --git a/main.tf b/main.tf index 3d0f8d7..e2c5d26 100644 --- a/main.tf +++ b/main.tf @@ -144,49 +144,6 @@ resource "hcloud_firewall" "k3s" { } -resource "local_file" "hetzner_ccm_config" { - content = templatefile("${path.module}/templates/hetzner_ccm.yaml.tpl", { - ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag - patch_name = var.hetzner_ccm_containers_latest ? "patch_latest" : "patch" - }) - filename = "${path.module}/hetzner/ccm/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "hetzner_csi_config" { - content = templatefile("${path.module}/templates/hetzner_csi.yaml.tpl", { - csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag - patch_name = var.hetzner_csi_containers_latest ? "patch_latest" : "" - }) - filename = "${path.module}/hetzner/csi/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "kured_config" { - content = templatefile("${path.module}/templates/kured.yaml.tpl", { - version = data.github_release.kured.release_tag - }) - filename = "${path.module}/kured/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "traefik_config" { - content = templatefile("${path.module}/templates/traefik_config.yaml.tpl", { - lb_disable_ipv6 = var.lb_disable_ipv6 - lb_server_type = var.lb_server_type - location = var.location - traefik_acme_tls = var.traefik_acme_tls - traefik_acme_email = var.traefik_acme_email - }) - filename = "${path.module}/templates/rendered/traefik_config.yaml" - file_permission = "0644" - directory_permission = "0755" -} - - resource "hcloud_placement_group" "k3s" { name = "k3s" type = "spread" diff --git a/master.tf b/master.tf index b6f5b9b..36b801b 100644 --- a/master.tf +++ b/master.tf @@ -81,6 +81,32 @@ resource "hcloud_server" "first_control_plane" { } } + # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured. + provisioner "file" { + content = local.post_install_kustomization + destination = "/tmp/kustomization.yaml" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Upload traefik config + provisioner "file" { + content = local.traefik_config + destination = "/tmp/traefik.yaml" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + # Run the first control plane provisioner "remote-exec" { inline = [ @@ -98,6 +124,14 @@ resource "hcloud_server" "first_control_plane" { sleep 2 done EOT + , <<-EOT + timeout 120 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%%{http_code}'' curl -k https://localhost:6443/readyz)" != "200" ]]; do sleep 1; done' + EOT + , "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}", + "kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token}", + "kubectl apply -k /tmp/", + "kubectl apply -f /tmp/traefik.yaml", + "rm /tmp/traefik.yaml /tmp/kustomization.yaml" ] connection { @@ -108,51 +142,6 @@ resource "hcloud_server" "first_control_plane" { } } - # Get the Kubeconfig, and wait for the node to be available - provisioner "local-exec" { - command = <<-EOT - until ssh -q ${local.ssh_args} root@${self.ipv4_address} [[ -f /etc/rancher/k3s/k3s.yaml ]] - do - echo "Waiting for the k3s config file to be ready..." - sleep 2 - done - scp ${local.ssh_args} root@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml - sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml - until kubectl get node ${self.name} --kubeconfig ${path.module}/kubeconfig.yaml 2> /dev/null || false - do - echo "Waiting for the node to become available..."; - sleep 2 - done - EOT - } - - # Install the Hetzner CCM and CSI - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl apply -k ${dirname(local_file.hetzner_ccm_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl apply -k ${dirname(local_file.hetzner_csi_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - EOT - } - - # Install Kured - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl -n kube-system apply -k ${dirname(local_file.kured_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - EOT - } - - # Configure the Traefik ingress controller - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl apply -f ${local_file.traefik_config.filename} --kubeconfig ${path.module}/kubeconfig.yaml - EOT - } - network { network_id = hcloud_network.k3s.id ip = local.first_control_plane_network_ip diff --git a/output.tf b/output.tf index 928445d..59471ea 100644 --- a/output.tf +++ b/output.tf @@ -7,3 +7,15 @@ output "agents_public_ip" { value = hcloud_server.agents.*.ipv4_address description = "The public IP addresses of the agent server." } + +output "kubeconfig_file" { + value = local.kubeconfig_external + description = "Kubeconfig file content with external IP address" + sensitive = true +} + +output "kubeconfig" { + description = "Structured kubeconfig data to supply to other providers" + value = local.kubeconfig_data + sensitive = true +} diff --git a/templates/hetzner_ccm.yaml.tpl b/templates/hetzner_ccm.yaml.tpl deleted file mode 100644 index 0d3167c..0000000 --- a/templates/hetzner_ccm.yaml.tpl +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml" - -patchesStrategicMerge: -- ${patch_name}.yaml \ No newline at end of file diff --git a/templates/hetzner_csi.yaml.tpl b/templates/hetzner_csi.yaml.tpl deleted file mode 100644 index 5f19a2a..0000000 --- a/templates/hetzner_csi.yaml.tpl +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml" - -%{ if patch_name != "" } -patchesStrategicMerge: -- ${patch_name}.yaml -%{ endif } \ No newline at end of file diff --git a/templates/kured.yaml.tpl b/templates/kured.yaml.tpl deleted file mode 100644 index 6d18068..0000000 --- a/templates/kured.yaml.tpl +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://github.com/weaveworks/kured/releases/download/${version}/kured-${version}-dockerhub.yaml" - -patchesStrategicMerge: -- patch.yaml \ No newline at end of file diff --git a/templates/kustomization.yaml.tpl b/templates/kustomization.yaml.tpl new file mode 100644 index 0000000..61d4f1a --- /dev/null +++ b/templates/kustomization.yaml.tpl @@ -0,0 +1,128 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml" +- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml" +- "https://github.com/weaveworks/kured/releases/download/${kured_version}/kured-${kured_version}-dockerhub.yaml" + +patchesStrategicMerge: +- |- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: kured + namespace: kube-system + spec: + selector: + matchLabels: + name: kured + template: + metadata: + labels: + name: kured + spec: + serviceAccountName: kured + containers: + - name: kured + command: + - /usr/bin/kured + - --reboot-command=/usr/bin/systemctl reboot +- |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system + spec: + template: + spec: + containers: + - name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=true" + - "--cluster-cidr=10.42.0.0/16" +%{ if ccm_latest ~} +- |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system + spec: + template: + spec: + containers: + - name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=true" + - "--cluster-cidr=10.42.0.0/16" + image: hetznercloud/hcloud-cloud-controller-manager:latest + imagePullPolicy: Always +%{ endif ~} +%{ if csi_latest ~} +- |- + kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: hcloud-csi-controller + namespace: kube-system + spec: + template: + metadata: + labels: + app: hcloud-csi-controller + spec: + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:canary + imagePullPolicy: Always + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:canary + imagePullPolicy: Always + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always + volumes: + - name: socket-dir + emptyDir: {} + --- + kind: DaemonSet + apiVersion: apps/v1 + metadata: + name: hcloud-csi-node + namespace: kube-system + labels: + app: hcloud-csi + spec: + selector: + matchLabels: + app: hcloud-csi + template: + spec: + containers: + - name: csi-node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always +%{ endif ~} diff --git a/versions.tf b/versions.tf index ed5848d..c1fdb5b 100644 --- a/versions.tf +++ b/versions.tf @@ -12,5 +12,9 @@ terraform { source = "hashicorp/local" version = ">= 2.0.0, < 3.0.0" } + remote = { + source = "tenstad/remote" + version = "~> 0.0.23" + } } }