diff --git a/.gitignore b/.gitignore index 64e5d1b..23f449f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ .terraform* *.tfstate* crash.log -hetzner/ccm/kustomization.yaml -hetzner/csi/kustomization.yaml kured/kustomization.yaml kubeconfig.yaml kubeconfig.yaml-e diff --git a/hetzner/ccm/patch.yaml b/hetzner/ccm/patch.yaml deleted file mode 100644 index 179b775..0000000 --- a/hetzner/ccm/patch.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hcloud-cloud-controller-manager - namespace: kube-system -spec: - template: - spec: - containers: - - name: hcloud-cloud-controller-manager - command: - - "/bin/hcloud-cloud-controller-manager" - - "--cloud-provider=hcloud" - - "--leader-elect=false" - - "--allow-untagged-cloud" - - "--allocate-node-cidrs=true" - - "--cluster-cidr=10.42.0.0/16" \ No newline at end of file diff --git a/hetzner/ccm/patch_latest.yaml b/hetzner/ccm/patch_latest.yaml deleted file mode 100644 index a631620..0000000 --- a/hetzner/ccm/patch_latest.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hcloud-cloud-controller-manager - namespace: kube-system -spec: - template: - spec: - containers: - - image: hetznercloud/hcloud-cloud-controller-manager:latest - imagePullPolicy: Always - name: hcloud-cloud-controller-manager - command: - - "/bin/hcloud-cloud-controller-manager" - - "--cloud-provider=hcloud" - - "--leader-elect=false" - - "--allow-untagged-cloud" - - "--allocate-node-cidrs=true" - - "--cluster-cidr=10.42.0.0/16" \ No newline at end of file diff --git a/hetzner/csi/patch_latest.yaml b/hetzner/csi/patch_latest.yaml deleted file mode 100644 index 743d655..0000000 --- a/hetzner/csi/patch_latest.yaml +++ /dev/null @@ -1,54 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: hcloud-csi-controller - namespace: kube-system -spec: - template: - metadata: - labels: - app: hcloud-csi-controller - spec: - containers: - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:canary - imagePullPolicy: Always - - name: csi-resizer - image: quay.io/k8scsi/csi-resizer:canary - imagePullPolicy: Always - - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:canary - imagePullPolicy: Always - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:latest - imagePullPolicy: Always - - name: liveness-probe - image: quay.io/k8scsi/livenessprobe:canary - imagePullPolicy: Always - volumes: - - name: socket-dir - emptyDir: {} ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: hcloud-csi-node - namespace: kube-system - labels: - app: hcloud-csi -spec: - selector: - matchLabels: - app: hcloud-csi - template: - spec: - containers: - - name: csi-node-driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:canary - imagePullPolicy: Always - - name: hcloud-csi-driver - image: hetznercloud/hcloud-csi-driver:latest - imagePullPolicy: Always - - name: liveness-probe - image: quay.io/k8scsi/livenessprobe:canary - imagePullPolicy: Always \ No newline at end of file diff --git a/kubeconfig.tf b/kubeconfig.tf new file mode 100644 index 0000000..07db0d2 --- /dev/null +++ b/kubeconfig.tf @@ -0,0 +1,28 @@ + +data "remote_file" "kubeconfig" { + conn { + host = hcloud_server.first_control_plane.ipv4_address + port = 22 + user = "root" + private_key = local.ssh_private_key + agent = var.private_key == null + } + path = "/etc/rancher/k3s/k3s.yaml" +} + +locals { + kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", hcloud_server.first_control_plane.ipv4_address) + kubeconfig_parsed = yamldecode(local.kubeconfig_external) + kubeconfig_data = { + host = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"] + client_certificate = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-certificate-data"]) + client_key = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-key-data"]) + cluster_ca_certificate = base64decode(local.kubeconfig_parsed["clusters"][0]["cluster"]["certificate-authority-data"]) + } +} + +resource "local_file" "kubeconfig" { + sensitive_content = local.kubeconfig_external + filename = "kubeconfig.yaml" + file_permission = "600" +} diff --git a/kured/patch.yaml b/kured/patch.yaml deleted file mode 100644 index bf72a0c..0000000 --- a/kured/patch.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kured - namespace: kube-system -spec: - selector: - matchLabels: - name: kured - template: - metadata: - labels: - name: kured - spec: - serviceAccountName: kured - containers: - - name: kured - command: - - /usr/bin/kured - - --reboot-command=/usr/bin/systemctl reboot diff --git a/locals.tf b/locals.tf index aaf8aac..98eb8d6 100644 --- a/locals.tf +++ b/locals.tf @@ -30,4 +30,24 @@ locals { "cp /root/config.ign /mnt/ignition/config.ign", "umount /mnt" ] + + post_install_kustomization = templatefile( + "${path.module}/templates/kustomization.yaml.tpl", + { + ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag + ccm_latest = var.hetzner_ccm_containers_latest + csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag + csi_latest = var.hetzner_csi_containers_latest + kured_version = data.github_release.kured.release_tag + }) + + traefik_config = templatefile( + "${path.module}/templates/traefik_config.yaml.tpl", + { + lb_disable_ipv6 = var.lb_disable_ipv6 + lb_server_type = var.lb_server_type + location = var.location + traefik_acme_tls = var.traefik_acme_tls + traefik_acme_email = var.traefik_acme_email + }) } diff --git a/main.tf b/main.tf index 3d0f8d7..e2c5d26 100644 --- a/main.tf +++ b/main.tf @@ -144,49 +144,6 @@ resource "hcloud_firewall" "k3s" { } -resource "local_file" "hetzner_ccm_config" { - content = templatefile("${path.module}/templates/hetzner_ccm.yaml.tpl", { - ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag - patch_name = var.hetzner_ccm_containers_latest ? "patch_latest" : "patch" - }) - filename = "${path.module}/hetzner/ccm/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "hetzner_csi_config" { - content = templatefile("${path.module}/templates/hetzner_csi.yaml.tpl", { - csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag - patch_name = var.hetzner_csi_containers_latest ? "patch_latest" : "" - }) - filename = "${path.module}/hetzner/csi/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "kured_config" { - content = templatefile("${path.module}/templates/kured.yaml.tpl", { - version = data.github_release.kured.release_tag - }) - filename = "${path.module}/kured/kustomization.yaml" - file_permission = "0644" - directory_permission = "0755" -} - -resource "local_file" "traefik_config" { - content = templatefile("${path.module}/templates/traefik_config.yaml.tpl", { - lb_disable_ipv6 = var.lb_disable_ipv6 - lb_server_type = var.lb_server_type - location = var.location - traefik_acme_tls = var.traefik_acme_tls - traefik_acme_email = var.traefik_acme_email - }) - filename = "${path.module}/templates/rendered/traefik_config.yaml" - file_permission = "0644" - directory_permission = "0755" -} - - resource "hcloud_placement_group" "k3s" { name = "k3s" type = "spread" diff --git a/master.tf b/master.tf index b6f5b9b..36b801b 100644 --- a/master.tf +++ b/master.tf @@ -81,6 +81,32 @@ resource "hcloud_server" "first_control_plane" { } } + # Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured. + provisioner "file" { + content = local.post_install_kustomization + destination = "/tmp/kustomization.yaml" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Upload traefik config + provisioner "file" { + content = local.traefik_config + destination = "/tmp/traefik.yaml" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + # Run the first control plane provisioner "remote-exec" { inline = [ @@ -98,6 +124,14 @@ resource "hcloud_server" "first_control_plane" { sleep 2 done EOT + , <<-EOT + timeout 120 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%%{http_code}'' curl -k https://localhost:6443/readyz)" != "200" ]]; do sleep 1; done' + EOT + , "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}", + "kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token}", + "kubectl apply -k /tmp/", + "kubectl apply -f /tmp/traefik.yaml", + "rm /tmp/traefik.yaml /tmp/kustomization.yaml" ] connection { @@ -108,51 +142,6 @@ resource "hcloud_server" "first_control_plane" { } } - # Get the Kubeconfig, and wait for the node to be available - provisioner "local-exec" { - command = <<-EOT - until ssh -q ${local.ssh_args} root@${self.ipv4_address} [[ -f /etc/rancher/k3s/k3s.yaml ]] - do - echo "Waiting for the k3s config file to be ready..." - sleep 2 - done - scp ${local.ssh_args} root@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml - sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml - until kubectl get node ${self.name} --kubeconfig ${path.module}/kubeconfig.yaml 2> /dev/null || false - do - echo "Waiting for the node to become available..."; - sleep 2 - done - EOT - } - - # Install the Hetzner CCM and CSI - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl apply -k ${dirname(local_file.hetzner_ccm_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl apply -k ${dirname(local_file.hetzner_csi_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - EOT - } - - # Install Kured - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl -n kube-system apply -k ${dirname(local_file.kured_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml - EOT - } - - # Configure the Traefik ingress controller - provisioner "local-exec" { - command = <<-EOT - set -ex - kubectl apply -f ${local_file.traefik_config.filename} --kubeconfig ${path.module}/kubeconfig.yaml - EOT - } - network { network_id = hcloud_network.k3s.id ip = local.first_control_plane_network_ip diff --git a/output.tf b/output.tf index 928445d..59471ea 100644 --- a/output.tf +++ b/output.tf @@ -7,3 +7,15 @@ output "agents_public_ip" { value = hcloud_server.agents.*.ipv4_address description = "The public IP addresses of the agent server." } + +output "kubeconfig_file" { + value = local.kubeconfig_external + description = "Kubeconfig file content with external IP address" + sensitive = true +} + +output "kubeconfig" { + description = "Structured kubeconfig data to supply to other providers" + value = local.kubeconfig_data + sensitive = true +} diff --git a/templates/hetzner_ccm.yaml.tpl b/templates/hetzner_ccm.yaml.tpl deleted file mode 100644 index 0d3167c..0000000 --- a/templates/hetzner_ccm.yaml.tpl +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml" - -patchesStrategicMerge: -- ${patch_name}.yaml \ No newline at end of file diff --git a/templates/hetzner_csi.yaml.tpl b/templates/hetzner_csi.yaml.tpl deleted file mode 100644 index 5f19a2a..0000000 --- a/templates/hetzner_csi.yaml.tpl +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml" - -%{ if patch_name != "" } -patchesStrategicMerge: -- ${patch_name}.yaml -%{ endif } \ No newline at end of file diff --git a/templates/kured.yaml.tpl b/templates/kured.yaml.tpl deleted file mode 100644 index 6d18068..0000000 --- a/templates/kured.yaml.tpl +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- "https://github.com/weaveworks/kured/releases/download/${version}/kured-${version}-dockerhub.yaml" - -patchesStrategicMerge: -- patch.yaml \ No newline at end of file diff --git a/templates/kustomization.yaml.tpl b/templates/kustomization.yaml.tpl new file mode 100644 index 0000000..61d4f1a --- /dev/null +++ b/templates/kustomization.yaml.tpl @@ -0,0 +1,128 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml" +- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml" +- "https://github.com/weaveworks/kured/releases/download/${kured_version}/kured-${kured_version}-dockerhub.yaml" + +patchesStrategicMerge: +- |- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: kured + namespace: kube-system + spec: + selector: + matchLabels: + name: kured + template: + metadata: + labels: + name: kured + spec: + serviceAccountName: kured + containers: + - name: kured + command: + - /usr/bin/kured + - --reboot-command=/usr/bin/systemctl reboot +- |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system + spec: + template: + spec: + containers: + - name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=true" + - "--cluster-cidr=10.42.0.0/16" +%{ if ccm_latest ~} +- |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system + spec: + template: + spec: + containers: + - name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=true" + - "--cluster-cidr=10.42.0.0/16" + image: hetznercloud/hcloud-cloud-controller-manager:latest + imagePullPolicy: Always +%{ endif ~} +%{ if csi_latest ~} +- |- + kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: hcloud-csi-controller + namespace: kube-system + spec: + template: + metadata: + labels: + app: hcloud-csi-controller + spec: + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:canary + imagePullPolicy: Always + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:canary + imagePullPolicy: Always + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always + volumes: + - name: socket-dir + emptyDir: {} + --- + kind: DaemonSet + apiVersion: apps/v1 + metadata: + name: hcloud-csi-node + namespace: kube-system + labels: + app: hcloud-csi + spec: + selector: + matchLabels: + app: hcloud-csi + template: + spec: + containers: + - name: csi-node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always +%{ endif ~} diff --git a/versions.tf b/versions.tf index ed5848d..c1fdb5b 100644 --- a/versions.tf +++ b/versions.tf @@ -12,5 +12,9 @@ terraform { source = "hashicorp/local" version = ">= 2.0.0, < 3.0.0" } + remote = { + source = "tenstad/remote" + version = "~> 0.0.23" + } } }