diff --git a/.gitignore b/.gitignore index 64e5d1b..23f449f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ .terraform* *.tfstate* crash.log -hetzner/ccm/kustomization.yaml -hetzner/csi/kustomization.yaml kured/kustomization.yaml kubeconfig.yaml kubeconfig.yaml-e diff --git a/agents.tf b/agents.tf index e8303e6..1c6381d 100644 --- a/agents.tf +++ b/agents.tf @@ -16,31 +16,24 @@ resource "hcloud_server" "agents" { "engine" = "k3s", } + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + provisioner "file" { content = templatefile("${path.module}/templates/config.ign.tpl", { name = self.name ssh_public_key = local.ssh_public_key }) destination = "/root/config.ign" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Install MicroOS provisioner "remote-exec" { inline = local.MicroOS_install_commands - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Issue a reboot command @@ -67,13 +60,6 @@ resource "hcloud_server" "agents" { token = random_password.k3s_token.result }) destination = "/etc/rancher/k3s/agent.conf" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Generating k3s agent config file @@ -85,13 +71,6 @@ resource "hcloud_server" "agents" { node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index) }) destination = "/etc/rancher/k3s/config.yaml" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Run the agent @@ -112,13 +91,6 @@ resource "hcloud_server" "agents" { done EOT ] - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } network { diff --git a/hetzner/csi/patch_latest.yaml b/hetzner/csi/patch_latest.yaml deleted file mode 100644 index 743d655..0000000 --- a/hetzner/csi/patch_latest.yaml +++ /dev/null @@ -1,54 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: hcloud-csi-controller - namespace: kube-system -spec: - template: - metadata: - labels: - app: hcloud-csi-controller - spec: - containers: - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:canary - imagePullPolicy: Always - - name: csi-resizer - image: quay.io/k8scsi/csi-resizer:canary - imagePullPolicy: Always - - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:canary - imagePullPolicy: Always - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:latest - imagePullPolicy: Always - - name: liveness-probe - image: quay.io/k8scsi/livenessprobe:canary - imagePullPolicy: Always - volumes: - - name: socket-dir - emptyDir: {} ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: hcloud-csi-node - namespace: kube-system - labels: - app: hcloud-csi -spec: - selector: - matchLabels: - app: hcloud-csi - template: - spec: - containers: - - name: csi-node-driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:canary - imagePullPolicy: Always - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:latest - imagePullPolicy: Always - - name: liveness-probe - image: quay.io/k8scsi/livenessprobe:canary - imagePullPolicy: Always \ No newline at end of file diff --git a/kubeconfig.tf b/kubeconfig.tf new file mode 100644 index 0000000..07db0d2 --- /dev/null +++ b/kubeconfig.tf @@ -0,0 +1,28 @@ + +data "remote_file" "kubeconfig" { + conn { + host = hcloud_server.first_control_plane.ipv4_address + port = 22 + user = "root" + private_key = local.ssh_private_key + agent = var.private_key == null + } + path = "/etc/rancher/k3s/k3s.yaml" +} + +locals { + kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", hcloud_server.first_control_plane.ipv4_address) + kubeconfig_parsed = yamldecode(local.kubeconfig_external) + kubeconfig_data = { + host = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"] + client_certificate = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-certificate-data"]) + client_key = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-key-data"]) + cluster_ca_certificate = base64decode(local.kubeconfig_parsed["clusters"][0]["cluster"]["certificate-authority-data"]) + } +} + +resource "local_file" "kubeconfig" { + sensitive_content = local.kubeconfig_external + filename = "kubeconfig.yaml" + file_permission = "600" +} diff --git a/locals.tf b/locals.tf index aaf8aac..962fbe4 100644 --- a/locals.tf +++ b/locals.tf @@ -1,7 +1,8 @@ locals { first_control_plane_network_ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 2) hcloud_image_name = "ubuntu-20.04" - ssh_public_key = trimspace(file(var.public_key)) + + ssh_public_key = trimspace(file(var.public_key)) # ssh_private_key is either the contents of var.private_key or null to use a ssh agent. ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key)) # ssh_identity is not set if the private key is passed directly, but if ssh agent is used, the public key tells ssh agent which private key to use. @@ -10,10 +11,15 @@ locals { # ssh_identity_file is used for ssh "-i" flag, its the private key if that is set, or a public key file # if an ssh agent is used. ssh_identity_file = var.private_key == null ? var.public_key : var.private_key - # shared flags for ssh to ignore host keys, to use root and our ssh identity file for all connections during provisioning. ssh_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${local.ssh_identity_file}" + ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag + ccm_latest = var.hetzner_ccm_containers_latest + csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag + csi_latest = var.hetzner_csi_containers_latest + kured_version = data.github_release.kured.release_tag + MicroOS_install_commands = [ "set -ex", "apt-get install -y aria2", diff --git a/main.tf b/main.tf index 3d0f8d7..e2c5d26 100644 --- a/main.tf +++ b/main.tf @@ -144,49 +144,6 @@ resource "hcloud_firewall" "k3s" { } -resource "local_file" "hetzner_ccm_config" { - content = templatefile("${path.module}/templates/hetzner_ccm.yaml.tpl", { - ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag - patch_name = var.hetzner_ccm_containers_latest ? "patch_latest" : "patch" - }) - filename = "${path.module}/hetzner/ccm/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "hetzner_csi_config" { - content = templatefile("${path.module}/templates/hetzner_csi.yaml.tpl", { - csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag - patch_name = var.hetzner_csi_containers_latest ? "patch_latest" : "" - }) - filename = "${path.module}/hetzner/csi/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "kured_config" { - content = templatefile("${path.module}/templates/kured.yaml.tpl", { - version = data.github_release.kured.release_tag - }) - filename = "${path.module}/kured/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "traefik_config" { - content = templatefile("${path.module}/templates/traefik_config.yaml.tpl", { - lb_disable_ipv6 = var.lb_disable_ipv6 - lb_server_type = var.lb_server_type - location = var.location - traefik_acme_tls = var.traefik_acme_tls - traefik_acme_email = var.traefik_acme_email - }) - filename = "${path.module}/templates/rendered/traefik_config.yaml" - file_permission = "0644" - directory_permission = "0755" -} - - resource "hcloud_placement_group" "k3s" { name = "k3s" type = "spread" diff --git a/master.tf b/master.tf index b6f5b9b..0981154 100644 --- a/master.tf +++ b/master.tf @@ -14,31 +14,24 @@ resource "hcloud_server" "first_control_plane" { "engine" = "k3s" } + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + provisioner "file" { content = templatefile("${path.module}/templates/config.ign.tpl", { name = self.name ssh_public_key = local.ssh_public_key }) destination = "/root/config.ign" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Install MicroOS provisioner "remote-exec" { inline = local.MicroOS_install_commands - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Issue a reboot command @@ -72,13 +65,6 @@ resource "hcloud_server" "first_control_plane" { node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"] }) destination = "/etc/rancher/k3s/config.yaml" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Run the first control plane @@ -88,69 +74,87 @@ resource "hcloud_server" "first_control_plane" { "hostnamectl set-hostname ${self.name}", # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf", + # prepare a directory for our post-installation kustomizations + "mkdir -p /tmp/post_install", # then we initiate the cluster "systemctl enable k3s-server", + # wait for k3s to get ready <<-EOT - until systemctl status k3s-server > /dev/null - do + timeout 120 bash < /dev/null; do systemctl start k3s-server echo "Initiating the cluster..." - sleep 2 + sleep 1 done + until [ -e /etc/rancher/k3s/k3s.yaml ]; do + echo "Waiting for kubectl config..." + sleep 1 + done + until [[ "\$(kubectl get --raw='/readyz')" == "ok" ]]; do + echo "Waiting for cluster to become ready..." + sleep 1 + done + EOF EOT ] - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } - # Get the Kubeconfig, and wait for the node to be available - provisioner "local-exec" { - command = <<-EOT - until ssh -q ${local.ssh_args} root@${self.ipv4_address} [[ -f /etc/rancher/k3s/k3s.yaml ]] - do - echo "Waiting for the k3s config file to be ready..." - sleep 2 - done - scp ${local.ssh_args} root@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml - sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml - until kubectl get node ${self.name} --kubeconfig ${path.module}/kubeconfig.yaml 2> /dev/null || false - do - echo "Waiting for the node to become available..."; - sleep 2 - done - EOT + # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured. + provisioner "file" { + content = yamlencode({ + apiVersion = "kustomize.config.k8s.io/v1beta1" + kind = "Kustomization" + resources = [ + "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${local.ccm_version}/ccm-networks.yaml", + "https://raw.githubusercontent.com/hetznercloud/csi-driver/${local.csi_version}/deploy/kubernetes/hcloud-csi.yml", + "https://github.com/weaveworks/kured/releases/download/${local.kured_version}/kured-${local.kured_version}-dockerhub.yaml", + "./traefik.yaml" + ] + patchesStrategicMerge = [ + file("${path.module}/patches/kured.yaml"), + local.ccm_latest ? file("${path.module}/patches/ccm_latest.yaml") : file("${path.module}/patches/ccm.yaml"), + local.csi_latest ? file("${path.module}/patches/csi_latest.yaml") : null, + ] + }) + destination = "/tmp/post_install/kustomization.yaml" } - # Install the Hetzner CCM and CSI - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl apply -k ${dirname(local_file.hetzner_ccm_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl apply -k ${dirname(local_file.hetzner_csi_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - EOT + # Upload traefik config + provisioner "file" { + content = templatefile( + "${path.module}/templates/traefik_config.yaml.tpl", + { + lb_disable_ipv6 = var.lb_disable_ipv6 + lb_server_type = var.lb_server_type + location = var.location + traefik_acme_tls = var.traefik_acme_tls + traefik_acme_email = var.traefik_acme_email + }) + destination = "/tmp/post_install/traefik.yaml" } - # Install Kured - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl -n kube-system apply -k ${dirname(local_file.kured_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - EOT + # Deploy secrets, logging is automatically disabled due to sensitive variables + provisioner "remote-exec" { + inline = [ + "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}", + "kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token}", + ] } - # Configure the Traefik ingress controller - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl apply -f ${local_file.traefik_config.filename} --kubeconfig ${path.module}/kubeconfig.yaml - EOT + # Deploy our post-installation kustomization + provisioner "remote-exec" { + inline = [ + # This ugly hack is here, because terraform serializes the + # embedded yaml files with "- |2", when there is more than + # one yamldocument in the embedded file. Kustomize does not understand + # that syntax and tries to parse the blocks content as a file, resulting + # in weird errors. so gnu sed with funny escaping is used to + # replace lines like "- |3" by "- |" (yaml block syntax). + # due to indendation this should not changes the embedded + # manifests themselves + "sed -i 's/^- |[0-9]\\+$/- |/g' /tmp/post_install/kustomization.yaml", + "kubectl apply -k /tmp/post_install", + ] } network { diff --git a/output.tf b/output.tf index 928445d..59471ea 100644 --- a/output.tf +++ b/output.tf @@ -7,3 +7,15 @@ output "agents_public_ip" { value = hcloud_server.agents.*.ipv4_address description = "The public IP addresses of the agent server." } + +output "kubeconfig_file" { + value = local.kubeconfig_external + description = "Kubeconfig file content with external IP address" + sensitive = true +} + +output "kubeconfig" { + description = "Structured kubeconfig data to supply to other providers" + value = local.kubeconfig_data + sensitive = true +} diff --git a/hetzner/ccm/patch.yaml b/patches/ccm.yaml similarity index 90% rename from hetzner/ccm/patch.yaml rename to patches/ccm.yaml index 179b775..229fa41 100644 --- a/hetzner/ccm/patch.yaml +++ b/patches/ccm.yaml @@ -14,4 +14,4 @@ spec: - "--leader-elect=false" - "--allow-untagged-cloud" - "--allocate-node-cidrs=true" - - "--cluster-cidr=10.42.0.0/16" \ No newline at end of file + - "--cluster-cidr=10.42.0.0/16" diff --git a/hetzner/ccm/patch_latest.yaml b/patches/ccm_latest.yaml similarity index 73% rename from hetzner/ccm/patch_latest.yaml rename to patches/ccm_latest.yaml index a631620..b461aa7 100644 --- a/hetzner/ccm/patch_latest.yaml +++ b/patches/ccm_latest.yaml @@ -7,13 +7,13 @@ spec: template: spec: containers: - - image: hetznercloud/hcloud-cloud-controller-manager:latest - imagePullPolicy: Always - name: hcloud-cloud-controller-manager + - name: hcloud-cloud-controller-manager command: - "/bin/hcloud-cloud-controller-manager" - "--cloud-provider=hcloud" - "--leader-elect=false" - "--allow-untagged-cloud" - "--allocate-node-cidrs=true" - - "--cluster-cidr=10.42.0.0/16" \ No newline at end of file + - "--cluster-cidr=10.42.0.0/16" + image: hetznercloud/hcloud-cloud-controller-manager:latest + imagePullPolicy: Always diff --git a/patches/csi_latest.yaml b/patches/csi_latest.yaml new file mode 100644 index 0000000..a5e6f74 --- /dev/null +++ b/patches/csi_latest.yaml @@ -0,0 +1,54 @@ + kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: hcloud-csi-controller + namespace: kube-system + spec: + template: + metadata: + labels: + app: hcloud-csi-controller + spec: + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:canary + imagePullPolicy: Always + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:canary + imagePullPolicy: Always + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always + volumes: + - name: socket-dir + emptyDir: {} + --- + kind: DaemonSet + apiVersion: apps/v1 + metadata: + name: hcloud-csi-node + namespace: kube-system + labels: + app: hcloud-csi + spec: + selector: + matchLabels: + app: hcloud-csi + template: + spec: + containers: + - name: csi-node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always diff --git a/kured/patch.yaml b/patches/kured.yaml similarity index 100% rename from kured/patch.yaml rename to patches/kured.yaml diff --git a/servers.tf b/servers.tf index 47bc94c..5f1d82c 100644 --- a/servers.tf +++ b/servers.tf @@ -15,31 +15,24 @@ resource "hcloud_server" "control_planes" { "engine" = "k3s", } + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + provisioner "file" { content = templatefile("${path.module}/templates/config.ign.tpl", { name = self.name ssh_public_key = local.ssh_public_key }) destination = "/root/config.ign" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Install MicroOS provisioner "remote-exec" { inline = local.MicroOS_install_commands - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Issue a reboot command @@ -75,13 +68,6 @@ resource "hcloud_server" "control_planes" { node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"] }) destination = "/etc/rancher/k3s/config.yaml" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } # Run an other control plane server @@ -102,13 +88,6 @@ resource "hcloud_server" "control_planes" { done EOT ] - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } } network { diff --git a/templates/hetzner_ccm.yaml.tpl b/templates/hetzner_ccm.yaml.tpl deleted file mode 100644 index 0d3167c..0000000 --- a/templates/hetzner_ccm.yaml.tpl +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml" - -patchesStrategicMerge: -- ${patch_name}.yaml \ No newline at end of file diff --git a/templates/hetzner_csi.yaml.tpl b/templates/hetzner_csi.yaml.tpl deleted file mode 100644 index 5f19a2a..0000000 --- a/templates/hetzner_csi.yaml.tpl +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml" - -%{ if patch_name != "" } -patchesStrategicMerge: -- ${patch_name}.yaml -%{ endif } \ No newline at end of file diff --git a/templates/kured.yaml.tpl b/templates/kured.yaml.tpl deleted file mode 100644 index 6d18068..0000000 --- a/templates/kured.yaml.tpl +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://github.com/weaveworks/kured/releases/download/${version}/kured-${version}-dockerhub.yaml" - -patchesStrategicMerge: -- patch.yaml \ No newline at end of file diff --git a/versions.tf b/versions.tf index ed5848d..c1fdb5b 100644 --- a/versions.tf +++ b/versions.tf @@ -12,5 +12,9 @@ terraform { source = "hashicorp/local" version = ">= 2.0.0, < 3.0.0" } + remote = { + source = "tenstad/remote" + version = "~> 0.0.23" + } } }