From 9dc4952665392e2d7a1f9dccbb7a9acd5d2ec8f2 Mon Sep 17 00:00:00 2001 From: phaer Date: Fri, 11 Feb 2022 12:45:03 +0100 Subject: [PATCH 01/11] Expose kubeconfig in outputs... * To do so, we need to ensure that the generated kubeconfig is part of terraforms dependency graph. This has the additional benefit of not depending on local files anymore which should enable multi-user setups. * This also means that we can't deploy CCM, CSI & Traefik from our local host, because we don't have kubeconfig.yaml locally while provisioning the control plane, only afterwards. * So we just run kubectl apply on the control plane itself, after k3s is ready. * To do so, we need to deploy all manifests. I've merged the patches into a single kustomization.yaml file, because that makes the deployment of those files to the control-plane server easier. * we could also put the traefik config into the same kustomization file, which would save us one of the file provisioner blocks. I didn't want this PR to get any bigger, and will consider merging this config later on. kustomization.yaml is small enough that we could yamlencode() for it and store the patches in separate files again, not as inline-strings which is kind of ugly. --- .gitignore | 2 - hetzner/ccm/patch.yaml | 17 ---- hetzner/ccm/patch_latest.yaml | 19 ----- hetzner/csi/patch_latest.yaml | 54 ------------- kubeconfig.tf | 28 +++++++ kured/patch.yaml | 20 ----- locals.tf | 20 +++++ main.tf | 43 ----------- master.tf | 79 ++++++++----------- output.tf | 12 +++ templates/hetzner_ccm.yaml.tpl | 8 -- templates/hetzner_csi.yaml.tpl | 10 --- templates/kured.yaml.tpl | 8 -- templates/kustomization.yaml.tpl | 128 +++++++++++++++++++++++++++++++ versions.tf | 4 + 15 files changed, 226 insertions(+), 226 deletions(-) delete mode 100644 hetzner/ccm/patch.yaml delete mode 100644 hetzner/ccm/patch_latest.yaml delete mode 100644 hetzner/csi/patch_latest.yaml create mode 100644 kubeconfig.tf delete mode 100644 kured/patch.yaml delete mode 100644 templates/hetzner_ccm.yaml.tpl delete mode 100644 templates/hetzner_csi.yaml.tpl delete mode 100644 templates/kured.yaml.tpl create mode 100644 templates/kustomization.yaml.tpl diff --git a/.gitignore b/.gitignore index 64e5d1b..23f449f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ .terraform* *.tfstate* crash.log -hetzner/ccm/kustomization.yaml -hetzner/csi/kustomization.yaml kured/kustomization.yaml kubeconfig.yaml kubeconfig.yaml-e diff --git a/hetzner/ccm/patch.yaml b/hetzner/ccm/patch.yaml deleted file mode 100644 index 179b775..0000000 --- a/hetzner/ccm/patch.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hcloud-cloud-controller-manager - namespace: kube-system -spec: - template: - spec: - containers: - - name: hcloud-cloud-controller-manager - command: - - "/bin/hcloud-cloud-controller-manager" - - "--cloud-provider=hcloud" - - "--leader-elect=false" - - "--allow-untagged-cloud" - - "--allocate-node-cidrs=true" - - "--cluster-cidr=10.42.0.0/16" \ No newline at end of file diff --git a/hetzner/ccm/patch_latest.yaml b/hetzner/ccm/patch_latest.yaml deleted file mode 100644 index a631620..0000000 --- a/hetzner/ccm/patch_latest.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hcloud-cloud-controller-manager - namespace: kube-system -spec: - template: - spec: - containers: - - image: hetznercloud/hcloud-cloud-controller-manager:latest - imagePullPolicy: Always - name: hcloud-cloud-controller-manager - command: - - "/bin/hcloud-cloud-controller-manager" - - "--cloud-provider=hcloud" - - "--leader-elect=false" - - "--allow-untagged-cloud" - - "--allocate-node-cidrs=true" - - "--cluster-cidr=10.42.0.0/16" \ No newline at end of file diff --git a/hetzner/csi/patch_latest.yaml b/hetzner/csi/patch_latest.yaml deleted file mode 100644 index 743d655..0000000 --- a/hetzner/csi/patch_latest.yaml +++ /dev/null @@ -1,54 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: hcloud-csi-controller - namespace: kube-system -spec: - template: - metadata: - labels: - app: hcloud-csi-controller - spec: - containers: - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:canary - imagePullPolicy: Always - - name: csi-resizer - image: quay.io/k8scsi/csi-resizer:canary - imagePullPolicy: Always - - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:canary - imagePullPolicy: Always - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:latest - imagePullPolicy: Always - - name: liveness-probe - image: quay.io/k8scsi/livenessprobe:canary - imagePullPolicy: Always - volumes: - - name: socket-dir - emptyDir: {} ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: hcloud-csi-node - namespace: kube-system - labels: - app: hcloud-csi -spec: - selector: - matchLabels: - app: hcloud-csi - template: - spec: - containers: - - name: csi-node-driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:canary - imagePullPolicy: Always - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:latest - imagePullPolicy: Always - - name: liveness-probe - image: quay.io/k8scsi/livenessprobe:canary - imagePullPolicy: Always \ No newline at end of file diff --git a/kubeconfig.tf b/kubeconfig.tf new file mode 100644 index 0000000..07db0d2 --- /dev/null +++ b/kubeconfig.tf @@ -0,0 +1,28 @@ + +data "remote_file" "kubeconfig" { + conn { + host = hcloud_server.first_control_plane.ipv4_address + port = 22 + user = "root" + private_key = local.ssh_private_key + agent = var.private_key == null + } + path = "/etc/rancher/k3s/k3s.yaml" +} + +locals { + kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", hcloud_server.first_control_plane.ipv4_address) + kubeconfig_parsed = yamldecode(local.kubeconfig_external) + kubeconfig_data = { + host = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"] + client_certificate = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-certificate-data"]) + client_key = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-key-data"]) + cluster_ca_certificate = base64decode(local.kubeconfig_parsed["clusters"][0]["cluster"]["certificate-authority-data"]) + } +} + +resource "local_file" "kubeconfig" { + sensitive_content = local.kubeconfig_external + filename = "kubeconfig.yaml" + file_permission = "600" +} diff --git a/kured/patch.yaml b/kured/patch.yaml deleted file mode 100644 index bf72a0c..0000000 --- a/kured/patch.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kured - namespace: kube-system -spec: - selector: - matchLabels: - name: kured - template: - metadata: - labels: - name: kured - spec: - serviceAccountName: kured - containers: - - name: kured - command: - - /usr/bin/kured - - --reboot-command=/usr/bin/systemctl reboot diff --git a/locals.tf b/locals.tf index aaf8aac..98eb8d6 100644 --- a/locals.tf +++ b/locals.tf @@ -30,4 +30,24 @@ locals { "cp /root/config.ign /mnt/ignition/config.ign", "umount /mnt" ] + + post_install_kustomization = templatefile( + "${path.module}/templates/kustomization.yaml.tpl", + { + ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag + ccm_latest = var.hetzner_ccm_containers_latest + csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag + csi_latest = var.hetzner_csi_containers_latest + kured_version = data.github_release.kured.release_tag + }) + + traefik_config = templatefile( + "${path.module}/templates/traefik_config.yaml.tpl", + { + lb_disable_ipv6 = var.lb_disable_ipv6 + lb_server_type = var.lb_server_type + location = var.location + traefik_acme_tls = var.traefik_acme_tls + traefik_acme_email = var.traefik_acme_email + }) } diff --git a/main.tf b/main.tf index 3d0f8d7..e2c5d26 100644 --- a/main.tf +++ b/main.tf @@ -144,49 +144,6 @@ resource "hcloud_firewall" "k3s" { } -resource "local_file" "hetzner_ccm_config" { - content = templatefile("${path.module}/templates/hetzner_ccm.yaml.tpl", { - ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag - patch_name = var.hetzner_ccm_containers_latest ? "patch_latest" : "patch" - }) - filename = "${path.module}/hetzner/ccm/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "hetzner_csi_config" { - content = templatefile("${path.module}/templates/hetzner_csi.yaml.tpl", { - csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag - patch_name = var.hetzner_csi_containers_latest ? "patch_latest" : "" - }) - filename = "${path.module}/hetzner/csi/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "kured_config" { - content = templatefile("${path.module}/templates/kured.yaml.tpl", { - version = data.github_release.kured.release_tag - }) - filename = "${path.module}/kured/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "traefik_config" { - content = templatefile("${path.module}/templates/traefik_config.yaml.tpl", { - lb_disable_ipv6 = var.lb_disable_ipv6 - lb_server_type = var.lb_server_type - location = var.location - traefik_acme_tls = var.traefik_acme_tls - traefik_acme_email = var.traefik_acme_email - }) - filename = "${path.module}/templates/rendered/traefik_config.yaml" - file_permission = "0644" - directory_permission = "0755" -} - - resource "hcloud_placement_group" "k3s" { name = "k3s" type = "spread" diff --git a/master.tf b/master.tf index b6f5b9b..36b801b 100644 --- a/master.tf +++ b/master.tf @@ -81,6 +81,32 @@ resource "hcloud_server" "first_control_plane" { } } + # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured. + provisioner "file" { + content = local.post_install_kustomization + destination = "/tmp/kustomization.yaml" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Upload traefik config + provisioner "file" { + content = local.traefik_config + destination = "/tmp/traefik.yaml" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + # Run the first control plane provisioner "remote-exec" { inline = [ @@ -98,6 +124,14 @@ resource "hcloud_server" "first_control_plane" { sleep 2 done EOT + , <<-EOT + timeout 120 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%%{http_code}'' curl -k https://localhost:6443/readyz)" != "200" ]]; do sleep 1; done' + EOT + , "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}", + "kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token}", + "kubectl apply -k /tmp/", + "kubectl apply -f /tmp/traefik.yaml", + "rm /tmp/traefik.yaml /tmp/kustomization.yaml" ] connection { @@ -108,51 +142,6 @@ resource "hcloud_server" "first_control_plane" { } } - # Get the Kubeconfig, and wait for the node to be available - provisioner "local-exec" { - command = <<-EOT - until ssh -q ${local.ssh_args} root@${self.ipv4_address} [[ -f /etc/rancher/k3s/k3s.yaml ]] - do - echo "Waiting for the k3s config file to be ready..." - sleep 2 - done - scp ${local.ssh_args} root@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml - sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml - until kubectl get node ${self.name} --kubeconfig ${path.module}/kubeconfig.yaml 2> /dev/null || false - do - echo "Waiting for the node to become available..."; - sleep 2 - done - EOT - } - - # Install the Hetzner CCM and CSI - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl apply -k ${dirname(local_file.hetzner_ccm_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl apply -k ${dirname(local_file.hetzner_csi_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - EOT - } - - # Install Kured - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl -n kube-system apply -k ${dirname(local_file.kured_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - EOT - } - - # Configure the Traefik ingress controller - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl apply -f ${local_file.traefik_config.filename} --kubeconfig ${path.module}/kubeconfig.yaml - EOT - } - network { network_id = hcloud_network.k3s.id ip = local.first_control_plane_network_ip diff --git a/output.tf b/output.tf index 928445d..59471ea 100644 --- a/output.tf +++ b/output.tf @@ -7,3 +7,15 @@ output "agents_public_ip" { value = hcloud_server.agents.*.ipv4_address description = "The public IP addresses of the agent server." } + +output "kubeconfig_file" { + value = local.kubeconfig_external + description = "Kubeconfig file content with external IP address" + sensitive = true +} + +output "kubeconfig" { + description = "Structured kubeconfig data to supply to other providers" + value = local.kubeconfig_data + sensitive = true +} diff --git a/templates/hetzner_ccm.yaml.tpl b/templates/hetzner_ccm.yaml.tpl deleted file mode 100644 index 0d3167c..0000000 --- a/templates/hetzner_ccm.yaml.tpl +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml" - -patchesStrategicMerge: -- ${patch_name}.yaml \ No newline at end of file diff --git a/templates/hetzner_csi.yaml.tpl b/templates/hetzner_csi.yaml.tpl deleted file mode 100644 index 5f19a2a..0000000 --- a/templates/hetzner_csi.yaml.tpl +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml" - -%{ if patch_name != "" } -patchesStrategicMerge: -- ${patch_name}.yaml -%{ endif } \ No newline at end of file diff --git a/templates/kured.yaml.tpl b/templates/kured.yaml.tpl deleted file mode 100644 index 6d18068..0000000 --- a/templates/kured.yaml.tpl +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://github.com/weaveworks/kured/releases/download/${version}/kured-${version}-dockerhub.yaml" - -patchesStrategicMerge: -- patch.yaml \ No newline at end of file diff --git a/templates/kustomization.yaml.tpl b/templates/kustomization.yaml.tpl new file mode 100644 index 0000000..61d4f1a --- /dev/null +++ b/templates/kustomization.yaml.tpl @@ -0,0 +1,128 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml" +- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml" +- "https://github.com/weaveworks/kured/releases/download/${kured_version}/kured-${kured_version}-dockerhub.yaml" + +patchesStrategicMerge: +- |- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: kured + namespace: kube-system + spec: + selector: + matchLabels: + name: kured + template: + metadata: + labels: + name: kured + spec: + serviceAccountName: kured + containers: + - name: kured + command: + - /usr/bin/kured + - --reboot-command=/usr/bin/systemctl reboot +- |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system + spec: + template: + spec: + containers: + - name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=true" + - "--cluster-cidr=10.42.0.0/16" +%{ if ccm_latest ~} +- |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system + spec: + template: + spec: + containers: + - name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=true" + - "--cluster-cidr=10.42.0.0/16" + image: hetznercloud/hcloud-cloud-controller-manager:latest + imagePullPolicy: Always +%{ endif ~} +%{ if csi_latest ~} +- |- + kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: hcloud-csi-controller + namespace: kube-system + spec: + template: + metadata: + labels: + app: hcloud-csi-controller + spec: + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:canary + imagePullPolicy: Always + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:canary + imagePullPolicy: Always + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always + volumes: + - name: socket-dir + emptyDir: {} + --- + kind: DaemonSet + apiVersion: apps/v1 + metadata: + name: hcloud-csi-node + namespace: kube-system + labels: + app: hcloud-csi + spec: + selector: + matchLabels: + app: hcloud-csi + template: + spec: + containers: + - name: csi-node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always +%{ endif ~} diff --git a/versions.tf b/versions.tf index ed5848d..c1fdb5b 100644 --- a/versions.tf +++ b/versions.tf @@ -12,5 +12,9 @@ terraform { source = "hashicorp/local" version = ">= 2.0.0, < 3.0.0" } + remote = { + source = "tenstad/remote" + version = "~> 0.0.23" + } } } From c46e912d3e3b6b2ce1a7076e75391b910d9057a6 Mon Sep 17 00:00:00 2001 From: phaer Date: Fri, 11 Feb 2022 16:00:19 +0100 Subject: [PATCH 02/11] use resource-level connection blocks... this is now possible, since all our provisioners are using the same settings. And it saves a bunch of lines --- agents.tf | 42 +++++++----------------------------------- master.tf | 49 +++++++------------------------------------------ servers.tf | 35 +++++++---------------------------- 3 files changed, 21 insertions(+), 105 deletions(-) diff --git a/agents.tf b/agents.tf index e8303e6..1c6381d 100644 --- a/agents.tf +++ b/agents.tf @@ -16,31 +16,24 @@ resource "hcloud_server" "agents" { "engine" = "k3s", } + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + provisioner "file" { content = templatefile("${path.module}/templates/config.ign.tpl", { name = self.name ssh_public_key = local.ssh_public_key }) destination = "/root/config.ign" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Install MicroOS provisioner "remote-exec" { inline = local.MicroOS_install_commands - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Issue a reboot command @@ -67,13 +60,6 @@ resource "hcloud_server" "agents" { token = random_password.k3s_token.result }) destination = "/etc/rancher/k3s/agent.conf" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Generating k3s agent config file @@ -85,13 +71,6 @@ resource "hcloud_server" "agents" { node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index) }) destination = "/etc/rancher/k3s/config.yaml" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Run the agent @@ -112,13 +91,6 @@ resource "hcloud_server" "agents" { done EOT ] - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } network { diff --git a/master.tf b/master.tf index 36b801b..b75b6c9 100644 --- a/master.tf +++ b/master.tf @@ -14,31 +14,24 @@ resource "hcloud_server" "first_control_plane" { "engine" = "k3s" } + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + provisioner "file" { content = templatefile("${path.module}/templates/config.ign.tpl", { name = self.name ssh_public_key = local.ssh_public_key }) destination = "/root/config.ign" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Install MicroOS provisioner "remote-exec" { inline = local.MicroOS_install_commands - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Issue a reboot command @@ -72,39 +65,18 @@ resource "hcloud_server" "first_control_plane" { node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"] }) destination = "/etc/rancher/k3s/config.yaml" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured. provisioner "file" { content = local.post_install_kustomization destination = "/tmp/kustomization.yaml" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Upload traefik config provisioner "file" { content = local.traefik_config destination = "/tmp/traefik.yaml" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Run the first control plane @@ -133,13 +105,6 @@ resource "hcloud_server" "first_control_plane" { "kubectl apply -f /tmp/traefik.yaml", "rm /tmp/traefik.yaml /tmp/kustomization.yaml" ] - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } network { diff --git a/servers.tf b/servers.tf index 47bc94c..5f1d82c 100644 --- a/servers.tf +++ b/servers.tf @@ -15,31 +15,24 @@ resource "hcloud_server" "control_planes" { "engine" = "k3s", } + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + provisioner "file" { content = templatefile("${path.module}/templates/config.ign.tpl", { name = self.name ssh_public_key = local.ssh_public_key }) destination = "/root/config.ign" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Install MicroOS provisioner "remote-exec" { inline = local.MicroOS_install_commands - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Issue a reboot command @@ -75,13 +68,6 @@ resource "hcloud_server" "control_planes" { node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"] }) destination = "/etc/rancher/k3s/config.yaml" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Run an other control plane server @@ -102,13 +88,6 @@ resource "hcloud_server" "control_planes" { done EOT ] - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } network { From 92937f0081fb296eef0e6644bd3fd8241e51ea79 Mon Sep 17 00:00:00 2001 From: phaer Date: Fri, 11 Feb 2022 22:47:57 +0100 Subject: [PATCH 03/11] cleanup first control plane provisioning... * move yaml to subdirectory of /tmp * reformat loop waiting for /readyz endpoint * add logging message * split provisioner because sensitive var.hcloud_token prohibits log output --- master.tf | 27 ++++++++++++++++++--------- templates/kustomization.yaml.tpl | 1 + 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/master.tf b/master.tf index b75b6c9..59c2d71 100644 --- a/master.tf +++ b/master.tf @@ -70,13 +70,13 @@ resource "hcloud_server" "first_control_plane" { # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured. provisioner "file" { content = local.post_install_kustomization - destination = "/tmp/kustomization.yaml" + destination = "/tmp/post_install/kustomization.yaml" } # Upload traefik config provisioner "file" { content = local.traefik_config - destination = "/tmp/traefik.yaml" + destination = "/tmp/post_install/traefik.yaml" } # Run the first control plane @@ -95,15 +95,24 @@ resource "hcloud_server" "first_control_plane" { echo "Initiating the cluster..." sleep 2 done + timeout 120 bash < Date: Fri, 11 Feb 2022 23:10:43 +0100 Subject: [PATCH 04/11] undo readyz formatting... ...running into weird issues and keep getting 403, while it worked with the old formatting. maybe bash escaping? --- master.tf | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/master.tf b/master.tf index 59c2d71..23ed746 100644 --- a/master.tf +++ b/master.tf @@ -95,12 +95,8 @@ resource "hcloud_server" "first_control_plane" { echo "Initiating the cluster..." sleep 2 done - timeout 120 bash < Date: Fri, 11 Feb 2022 23:11:20 +0100 Subject: [PATCH 05/11] fix /tmp/post_install... ...without an explicit mkdir before, it just writes the latest file to /tmp/post_install (which is a file, not a directory) --- master.tf | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/master.tf b/master.tf index 23ed746..417da02 100644 --- a/master.tf +++ b/master.tf @@ -67,6 +67,30 @@ resource "hcloud_server" "first_control_plane" { destination = "/etc/rancher/k3s/config.yaml" } + # Run the first control plane + provisioner "remote-exec" { + inline = [ + # set the hostname in a persistent fashion + "hostnamectl set-hostname ${self.name}", + # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured + "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf", + # then we initiate the cluster + "systemctl enable k3s-server", + # prepare a directory for our post-installation kustomizations + "mkdir -p /tmp/post_install", + # start k3s and wait for the cluster to be ready + <<-EOT + until systemctl status k3s-server > /dev/null + do + systemctl start k3s-server + echo "Initiating the cluster..." + sleep 2 + done + timeout 120 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%%{http_code}'' curl -k https://localhost:6443/readyz)" != "200" ]]; do echo "Waiting for cluster to become ready" ; sleep 1; done' + EOT + ] + } + # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured. provisioner "file" { content = local.post_install_kustomization @@ -79,30 +103,6 @@ resource "hcloud_server" "first_control_plane" { destination = "/tmp/post_install/traefik.yaml" } - # Run the first control plane - provisioner "remote-exec" { - inline = [ - # set the hostname in a persistent fashion - "hostnamectl set-hostname ${self.name}", - # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured - "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf", - # then we initiate the cluster - "systemctl enable k3s-server", - <<-EOT - until systemctl status k3s-server > /dev/null - do - systemctl start k3s-server - echo "Initiating the cluster..." - sleep 2 - done - timeout 120 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%%{http_code}'' curl -k https://localhost:6443/readyz)" != "200" ]]; do echo "Waiting for cluster to become ready" -; sleep 1; done' - EOF - EOT - ] - } - - # Provision kubernetes resources for CSI, CCM, kured, traefik, etc provisioner "remote-exec" { inline = [ "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}", From ddcc473ea8619f7a6df0f56180f9de32c4d0288d Mon Sep 17 00:00:00 2001 From: phaer Date: Fri, 11 Feb 2022 23:49:16 +0100 Subject: [PATCH 06/11] (hopefully) simplify wait for cluster ready --- master.tf | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/master.tf b/master.tf index 417da02..d13b9b2 100644 --- a/master.tf +++ b/master.tf @@ -74,19 +74,24 @@ resource "hcloud_server" "first_control_plane" { "hostnamectl set-hostname ${self.name}", # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf", - # then we initiate the cluster - "systemctl enable k3s-server", # prepare a directory for our post-installation kustomizations "mkdir -p /tmp/post_install", - # start k3s and wait for the cluster to be ready + # then we initiate the cluster + "systemctl enable k3s-server", + # start k3s + "systemctl start k3s-server", + # wait for k3s to get ready <<-EOT - until systemctl status k3s-server > /dev/null - do - systemctl start k3s-server - echo "Initiating the cluster..." - sleep 2 + timeout 120 bash < Date: Fri, 11 Feb 2022 23:49:54 +0100 Subject: [PATCH 07/11] replace kustomization.yaml.tpl with yamlencode benefit is replacing inline strings in yaml with proper files locally while still just deploying a single file to the remote host. --- locals.tf | 30 ++----- master.tf | 27 ++++++- patches/ccm.yaml | 17 ++++ patches/ccm_latest.yaml | 19 +++++ patches/csi_latest.yaml | 54 +++++++++++++ patches/kured.yaml | 20 +++++ templates/kustomization.yaml.tpl | 129 ------------------------------- 7 files changed, 143 insertions(+), 153 deletions(-) create mode 100644 patches/ccm.yaml create mode 100644 patches/ccm_latest.yaml create mode 100644 patches/csi_latest.yaml create mode 100644 patches/kured.yaml delete mode 100644 templates/kustomization.yaml.tpl diff --git a/locals.tf b/locals.tf index 98eb8d6..962fbe4 100644 --- a/locals.tf +++ b/locals.tf @@ -1,7 +1,8 @@ locals { first_control_plane_network_ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 2) hcloud_image_name = "ubuntu-20.04" - ssh_public_key = trimspace(file(var.public_key)) + + ssh_public_key = trimspace(file(var.public_key)) # ssh_private_key is either the contents of var.private_key or null to use a ssh agent. ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key)) # ssh_identity is not set if the private key is passed directly, but if ssh agent is used, the public key tells ssh agent which private key to use. @@ -10,10 +11,15 @@ locals { # ssh_identity_file is used for ssh "-i" flag, its the private key if that is set, or a public key file # if an ssh agent is used. ssh_identity_file = var.private_key == null ? var.public_key : var.private_key - # shared flags for ssh to ignore host keys, to use root and our ssh identity file for all connections during provisioning. ssh_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${local.ssh_identity_file}" + ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag + ccm_latest = var.hetzner_ccm_containers_latest + csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag + csi_latest = var.hetzner_csi_containers_latest + kured_version = data.github_release.kured.release_tag + MicroOS_install_commands = [ "set -ex", "apt-get install -y aria2", @@ -30,24 +36,4 @@ locals { "cp /root/config.ign /mnt/ignition/config.ign", "umount /mnt" ] - - post_install_kustomization = templatefile( - "${path.module}/templates/kustomization.yaml.tpl", - { - ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag - ccm_latest = var.hetzner_ccm_containers_latest - csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag - csi_latest = var.hetzner_csi_containers_latest - kured_version = data.github_release.kured.release_tag - }) - - traefik_config = templatefile( - "${path.module}/templates/traefik_config.yaml.tpl", - { - lb_disable_ipv6 = var.lb_disable_ipv6 - lb_server_type = var.lb_server_type - location = var.location - traefik_acme_tls = var.traefik_acme_tls - traefik_acme_email = var.traefik_acme_email - }) } diff --git a/master.tf b/master.tf index d13b9b2..ee5ce9a 100644 --- a/master.tf +++ b/master.tf @@ -98,16 +98,39 @@ resource "hcloud_server" "first_control_plane" { # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured. provisioner "file" { - content = local.post_install_kustomization + content = yamlencode({ + apiVersion = "kustomize.config.k8s.io/v1beta1" + kind = "Kustomization" + resources = [ + "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${local.ccm_version}/ccm-networks.yaml", + "https://raw.githubusercontent.com/hetznercloud/csi-driver/${local.csi_version}/deploy/kubernetes/hcloud-csi.yml", + "https://github.com/weaveworks/kured/releases/download/${local.kured_version}/kured-${local.kured_version}-dockerhub.yaml", + "./traefik.yaml" + ] + patchesStrategicMerge = [ + file("${path.module}/patches/kured.yaml"), + local.ccm_latest ? file("${path.module}/patches/ccm_latest.yaml") : file("${path.module}/patches/ccm.yaml"), + local.csi_latest ? file("${path.module}/patches/csi_latest.yaml") : null, + ] + }) destination = "/tmp/post_install/kustomization.yaml" } # Upload traefik config provisioner "file" { - content = local.traefik_config + content = templatefile( + "${path.module}/templates/traefik_config.yaml.tpl", + { + lb_disable_ipv6 = var.lb_disable_ipv6 + lb_server_type = var.lb_server_type + location = var.location + traefik_acme_tls = var.traefik_acme_tls + traefik_acme_email = var.traefik_acme_email + }) destination = "/tmp/post_install/traefik.yaml" } + # Deploy our post-installation kustomization provisioner "remote-exec" { inline = [ "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}", diff --git a/patches/ccm.yaml b/patches/ccm.yaml new file mode 100644 index 0000000..229fa41 --- /dev/null +++ b/patches/ccm.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system +spec: + template: + spec: + containers: + - name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=true" + - "--cluster-cidr=10.42.0.0/16" diff --git a/patches/ccm_latest.yaml b/patches/ccm_latest.yaml new file mode 100644 index 0000000..b461aa7 --- /dev/null +++ b/patches/ccm_latest.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system +spec: + template: + spec: + containers: + - name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=true" + - "--cluster-cidr=10.42.0.0/16" + image: hetznercloud/hcloud-cloud-controller-manager:latest + imagePullPolicy: Always diff --git a/patches/csi_latest.yaml b/patches/csi_latest.yaml new file mode 100644 index 0000000..a5e6f74 --- /dev/null +++ b/patches/csi_latest.yaml @@ -0,0 +1,54 @@ + kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: hcloud-csi-controller + namespace: kube-system + spec: + template: + metadata: + labels: + app: hcloud-csi-controller + spec: + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:canary + imagePullPolicy: Always + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:canary + imagePullPolicy: Always + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always + volumes: + - name: socket-dir + emptyDir: {} + --- + kind: DaemonSet + apiVersion: apps/v1 + metadata: + name: hcloud-csi-node + namespace: kube-system + labels: + app: hcloud-csi + spec: + selector: + matchLabels: + app: hcloud-csi + template: + spec: + containers: + - name: csi-node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always diff --git a/patches/kured.yaml b/patches/kured.yaml new file mode 100644 index 0000000..bf72a0c --- /dev/null +++ b/patches/kured.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kured + namespace: kube-system +spec: + selector: + matchLabels: + name: kured + template: + metadata: + labels: + name: kured + spec: + serviceAccountName: kured + containers: + - name: kured + command: + - /usr/bin/kured + - --reboot-command=/usr/bin/systemctl reboot diff --git a/templates/kustomization.yaml.tpl b/templates/kustomization.yaml.tpl deleted file mode 100644 index 8042097..0000000 --- a/templates/kustomization.yaml.tpl +++ /dev/null @@ -1,129 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml" -- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml" -- "https://github.com/weaveworks/kured/releases/download/${kured_version}/kured-${kured_version}-dockerhub.yaml" -- ./traefik.yaml - -patchesStrategicMerge: -- |- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: kured - namespace: kube-system - spec: - selector: - matchLabels: - name: kured - template: - metadata: - labels: - name: kured - spec: - serviceAccountName: kured - containers: - - name: kured - command: - - /usr/bin/kured - - --reboot-command=/usr/bin/systemctl reboot -- |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: hcloud-cloud-controller-manager - namespace: kube-system - spec: - template: - spec: - containers: - - name: hcloud-cloud-controller-manager - command: - - "/bin/hcloud-cloud-controller-manager" - - "--cloud-provider=hcloud" - - "--leader-elect=false" - - "--allow-untagged-cloud" - - "--allocate-node-cidrs=true" - - "--cluster-cidr=10.42.0.0/16" -%{ if ccm_latest ~} -- |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: hcloud-cloud-controller-manager - namespace: kube-system - spec: - template: - spec: - containers: - - name: hcloud-cloud-controller-manager - command: - - "/bin/hcloud-cloud-controller-manager" - - "--cloud-provider=hcloud" - - "--leader-elect=false" - - "--allow-untagged-cloud" - - "--allocate-node-cidrs=true" - - "--cluster-cidr=10.42.0.0/16" - image: hetznercloud/hcloud-cloud-controller-manager:latest - imagePullPolicy: Always -%{ endif ~} -%{ if csi_latest ~} -- |- - kind: StatefulSet - apiVersion: apps/v1 - metadata: - name: hcloud-csi-controller - namespace: kube-system - spec: - template: - metadata: - labels: - app: hcloud-csi-controller - spec: - containers: - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:canary - imagePullPolicy: Always - - name: csi-resizer - image: quay.io/k8scsi/csi-resizer:canary - imagePullPolicy: Always - - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:canary - imagePullPolicy: Always - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:latest - imagePullPolicy: Always - - name: liveness-probe - image: quay.io/k8scsi/livenessprobe:canary - imagePullPolicy: Always - volumes: - - name: socket-dir - emptyDir: {} - --- - kind: DaemonSet - apiVersion: apps/v1 - metadata: - name: hcloud-csi-node - namespace: kube-system - labels: - app: hcloud-csi - spec: - selector: - matchLabels: - app: hcloud-csi - template: - spec: - containers: - - name: csi-node-driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:canary - imagePullPolicy: Always - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:latest - imagePullPolicy: Always - - name: liveness-probe - image: quay.io/k8scsi/livenessprobe:canary - imagePullPolicy: Always -%{ endif ~} From 1daa14defcac280dca64e629fa988e3ed703c5aa Mon Sep 17 00:00:00 2001 From: phaer Date: Fri, 11 Feb 2022 23:57:18 +0100 Subject: [PATCH 08/11] fix waiting for the cluster once again --- master.tf | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/master.tf b/master.tf index ee5ce9a..f92cb32 100644 --- a/master.tf +++ b/master.tf @@ -83,12 +83,17 @@ resource "hcloud_server" "first_control_plane" { # wait for k3s to get ready <<-EOT timeout 120 bash < /dev/null; do + systemctl start k3s-server + echo "Initiating the cluster..." sleep 1 done - until [[ "$(kubectl get --raw='/readyz')" == "ok" ]]; do - echo "Waiting for cluster to become ready" + until [ -e /etc/rancher/k3s/k3s.yaml ]; do + echo "Waiting for kubectl config..." + sleep 1 + done + until [[ "\$(kubectl get --raw='/readyz')" == "ok" ]]; do + echo "Waiting for cluster to become ready..." sleep 1 done EOF From ca64e9756339633a3c40370efb7f931ba340dfdb Mon Sep 17 00:00:00 2001 From: phaer Date: Sat, 12 Feb 2022 00:32:11 +0100 Subject: [PATCH 09/11] split provisioning for better logging --- master.tf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/master.tf b/master.tf index f92cb32..1c77478 100644 --- a/master.tf +++ b/master.tf @@ -135,11 +135,17 @@ resource "hcloud_server" "first_control_plane" { destination = "/tmp/post_install/traefik.yaml" } - # Deploy our post-installation kustomization + # Deploy secrets, logging is automatically disabled due to sensitive variables provisioner "remote-exec" { inline = [ "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}", "kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token}", + ] + } + + # Deploy our post-installation kustomization + provisioner "remote-exec" { + inline = [ "kubectl apply -k /tmp/post_install", "rm -rf /tmp/post_install" ] From 8ba33a12c807193bc1d50076ec7bd3b5d681255a Mon Sep 17 00:00:00 2001 From: phaer Date: Sat, 12 Feb 2022 00:52:13 +0100 Subject: [PATCH 10/11] fix post-install kustomization, keep file... this risks exposing secrets from the deployed manifests, but those are currently deployed beforehand so we should be good as long as kustomization.yaml does not contain any. --- master.tf | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/master.tf b/master.tf index 1c77478..8e7f32f 100644 --- a/master.tf +++ b/master.tf @@ -146,8 +146,16 @@ resource "hcloud_server" "first_control_plane" { # Deploy our post-installation kustomization provisioner "remote-exec" { inline = [ + # This ugly hack is here, because terraform serializes the + # embedded yaml files with "- |2", when there is more than + # one yamldocument in the embedded file. Kustomize does not understand + # that syntax and tries to parse the blocks content as a file, resulting + # in weird errors. so gnu sed with funny escaping is used to + # replace lines like "- |3" by "- |" (yaml block syntax). + # due to indendation this should not changes the embedded + # manifests themselves + "sed -i 's/^- |[0-9]\\+$/- |/g' /tmp/post_install/kustomization.yaml", "kubectl apply -k /tmp/post_install", - "rm -rf /tmp/post_install" ] } From 740b20966f93b82a5c5e1e3cf791e7a572c7f104 Mon Sep 17 00:00:00 2001 From: phaer Date: Sat, 12 Feb 2022 01:09:34 +0100 Subject: [PATCH 11/11] remove duplicate systemctl start k3s-server --- master.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/master.tf b/master.tf index 8e7f32f..0981154 100644 --- a/master.tf +++ b/master.tf @@ -78,8 +78,6 @@ resource "hcloud_server" "first_control_plane" { "mkdir -p /tmp/post_install", # then we initiate the cluster "systemctl enable k3s-server", - # start k3s - "systemctl start k3s-server", # wait for k3s to get ready <<-EOT timeout 120 bash <