kjuulh/terraform-hcloud-kube-hetzner
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Merge pull request #66 from phaer/expose-kubeconfig-microos

Browse Source 
Expose kubeconfig in outputs...
This commit is contained in:
Karim Naufal 2022-02-12 01:23:19 +01:00 committed by GitHub
commit 75fd63da1e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 197 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,8 +1,6 @@
.terraform*
*.tfstate*
crash.log
hetzner/ccm/kustomization.yaml
hetzner/csi/kustomization.yaml
kured/kustomization.yaml
kubeconfig.yaml
kubeconfig.yaml-e

42
agents.tf
View File

@ -16,31 +16,24 @@ resource "hcloud_server" "agents" {
"engine" = "k3s",
}
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
provisioner "file" {
content = templatefile("${path.module}/templates/config.ign.tpl", {
name = self.name
ssh_public_key = local.ssh_public_key
})
destination = "/root/config.ign"
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
# Install MicroOS
provisioner "remote-exec" {
inline = local.MicroOS_install_commands
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
# Issue a reboot command
@ -67,13 +60,6 @@ resource "hcloud_server" "agents" {
token = random_password.k3s_token.result
})
destination = "/etc/rancher/k3s/agent.conf"
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
# Generating k3s agent config file
@ -85,13 +71,6 @@ resource "hcloud_server" "agents" {
node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 257 + count.index)
})
destination = "/etc/rancher/k3s/config.yaml"
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
# Run the agent
@ -112,13 +91,6 @@ resource "hcloud_server" "agents" {
done
EOT
]
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
network {

54
hetzner/csi/patch_latest.yaml
View File

@ -1,54 +0,0 @@
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: hcloud-csi-controller
namespace: kube-system
spec:
template:
metadata:
labels:
app: hcloud-csi-controller
spec:
containers:
- name: csi-attacher
image: quay.io/k8scsi/csi-attacher:canary
imagePullPolicy: Always
- name: csi-resizer
image: quay.io/k8scsi/csi-resizer:canary
imagePullPolicy: Always
- name: csi-provisioner
image: quay.io/k8scsi/csi-provisioner:canary
imagePullPolicy: Always
- name: hcloud-csi-driver
image: hetznercloud/hcloud-csi-driver:latest
imagePullPolicy: Always
- name: liveness-probe
image: quay.io/k8scsi/livenessprobe:canary
imagePullPolicy: Always
volumes:
- name: socket-dir
emptyDir: {}
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: hcloud-csi-node
namespace: kube-system
labels:
app: hcloud-csi
spec:
selector:
matchLabels:
app: hcloud-csi
template:
spec:
containers:
- name: csi-node-driver-registrar
image: quay.io/k8scsi/csi-node-driver-registrar:canary
imagePullPolicy: Always
- name: hcloud-csi-driver
image: hetznercloud/hcloud-csi-driver:latest
imagePullPolicy: Always
- name: liveness-probe
image: quay.io/k8scsi/livenessprobe:canary
imagePullPolicy: Always

28
kubeconfig.tf Normal file
View File

@ -0,0 +1,28 @@
data "remote_file" "kubeconfig" {
conn {
host = hcloud_server.first_control_plane.ipv4_address
port = 22
user = "root"
private_key = local.ssh_private_key
agent = var.private_key == null
}
path = "/etc/rancher/k3s/k3s.yaml"
}
locals {
kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", hcloud_server.first_control_plane.ipv4_address)
kubeconfig_parsed = yamldecode(local.kubeconfig_external)
kubeconfig_data = {
host = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"]
client_certificate = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-certificate-data"])
client_key = base64decode(local.kubeconfig_parsed["users"][0]["user"]["client-key-data"])
cluster_ca_certificate = base64decode(local.kubeconfig_parsed["clusters"][0]["cluster"]["certificate-authority-data"])
}
}
resource "local_file" "kubeconfig" {
sensitive_content = local.kubeconfig_external
filename = "kubeconfig.yaml"
file_permission = "600"
}

10
locals.tf
View File

@ -1,7 +1,8 @@
locals {
first_control_plane_network_ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 2)
hcloud_image_name = "ubuntu-20.04"
ssh_public_key = trimspace(file(var.public_key))
ssh_public_key = trimspace(file(var.public_key))
# ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key))
# ssh_identity is not set if the private key is passed directly, but if ssh agent is used, the public key tells ssh agent which private key to use.
@ -10,10 +11,15 @@ locals {
# ssh_identity_file is used for ssh "-i" flag, its the private key if that is set, or a public key file
# if an ssh agent is used.
ssh_identity_file = var.private_key == null ? var.public_key : var.private_key
# shared flags for ssh to ignore host keys, to use root and our ssh identity file for all connections during provisioning.
ssh_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${local.ssh_identity_file}"
ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag
ccm_latest = var.hetzner_ccm_containers_latest
csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag
csi_latest = var.hetzner_csi_containers_latest
kured_version = data.github_release.kured.release_tag
MicroOS_install_commands = [
"set -ex",
"apt-get install -y aria2",

43
main.tf
View File

@ -144,49 +144,6 @@ resource "hcloud_firewall" "k3s" {
}
resource "local_file" "hetzner_ccm_config" {
content = templatefile("${path.module}/templates/hetzner_ccm.yaml.tpl", {
ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag
patch_name = var.hetzner_ccm_containers_latest ? "patch_latest" : "patch"
})
filename = "${path.module}/hetzner/ccm/kustomization.yaml"
file_permission = "0644"
directory_permission = "0755"
}
resource "local_file" "hetzner_csi_config" {
content = templatefile("${path.module}/templates/hetzner_csi.yaml.tpl", {
csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag
patch_name = var.hetzner_csi_containers_latest ? "patch_latest" : ""
})
filename = "${path.module}/hetzner/csi/kustomization.yaml"
file_permission = "0644"
directory_permission = "0755"
}
resource "local_file" "kured_config" {
content = templatefile("${path.module}/templates/kured.yaml.tpl", {
version = data.github_release.kured.release_tag
})
filename = "${path.module}/kured/kustomization.yaml"
file_permission = "0644"
directory_permission = "0755"
}
resource "local_file" "traefik_config" {
content = templatefile("${path.module}/templates/traefik_config.yaml.tpl", {
lb_disable_ipv6 = var.lb_disable_ipv6
lb_server_type = var.lb_server_type
location = var.location
traefik_acme_tls = var.traefik_acme_tls
traefik_acme_email = var.traefik_acme_email
})
filename = "${path.module}/templates/rendered/traefik_config.yaml"
file_permission = "0644"
directory_permission = "0755"
}
resource "hcloud_placement_group" "k3s" {
name = "k3s"
type = "spread"

140
master.tf
View File

@ -14,31 +14,24 @@ resource "hcloud_server" "first_control_plane" {
"engine" = "k3s"
}
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
provisioner "file" {
content = templatefile("${path.module}/templates/config.ign.tpl", {
name = self.name
ssh_public_key = local.ssh_public_key
})
destination = "/root/config.ign"
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
# Install MicroOS
provisioner "remote-exec" {
inline = local.MicroOS_install_commands
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
# Issue a reboot command
@ -72,13 +65,6 @@ resource "hcloud_server" "first_control_plane" {
node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
})
destination = "/etc/rancher/k3s/config.yaml"
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
# Run the first control plane
@ -88,69 +74,87 @@ resource "hcloud_server" "first_control_plane" {
"hostnamectl set-hostname ${self.name}",
# first we disable automatic reboot (after transactional updates), and configure the reboot method as kured
"rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf",
# prepare a directory for our post-installation kustomizations
"mkdir -p /tmp/post_install",
# then we initiate the cluster
"systemctl enable k3s-server",
# wait for k3s to get ready
<<-EOT
until systemctl status k3s-server > /dev/null
do
timeout 120 bash <<EOF
until systemctl status k3s-server > /dev/null; do
systemctl start k3s-server
echo "Initiating the cluster..."
sleep 2
sleep 1
done
until [ -e /etc/rancher/k3s/k3s.yaml ]; do
echo "Waiting for kubectl config..."
sleep 1
done
until [[ "\$(kubectl get --raw='/readyz')" == "ok" ]]; do
echo "Waiting for cluster to become ready..."
sleep 1
done
EOF
EOT
]
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
# Get the Kubeconfig, and wait for the node to be available
provisioner "local-exec" {
command = <<-EOT
until ssh -q ${local.ssh_args} root@${self.ipv4_address} [[ -f /etc/rancher/k3s/k3s.yaml ]]
do
echo "Waiting for the k3s config file to be ready..."
sleep 2
done
scp ${local.ssh_args} root@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml
sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml
until kubectl get node ${self.name} --kubeconfig ${path.module}/kubeconfig.yaml 2> /dev/null || false
do
echo "Waiting for the node to become available...";
sleep 2
done
EOT
# Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured.
provisioner "file" {
content = yamlencode({
apiVersion = "kustomize.config.k8s.io/v1beta1"
kind = "Kustomization"
resources = [
"https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${local.ccm_version}/ccm-networks.yaml",
"https://raw.githubusercontent.com/hetznercloud/csi-driver/${local.csi_version}/deploy/kubernetes/hcloud-csi.yml",
"https://github.com/weaveworks/kured/releases/download/${local.kured_version}/kured-${local.kured_version}-dockerhub.yaml",
"./traefik.yaml"
]
patchesStrategicMerge = [
file("${path.module}/patches/kured.yaml"),
local.ccm_latest ? file("${path.module}/patches/ccm_latest.yaml") : file("${path.module}/patches/ccm.yaml"),
local.csi_latest ? file("${path.module}/patches/csi_latest.yaml") : null,
]
})
destination = "/tmp/post_install/kustomization.yaml"
}
# Install the Hetzner CCM and CSI
provisioner "local-exec" {
command = <<-EOT
set -ex
kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --kubeconfig ${path.module}/kubeconfig.yaml
kubectl apply -k ${dirname(local_file.hetzner_ccm_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml
kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --kubeconfig ${path.module}/kubeconfig.yaml
kubectl apply -k ${dirname(local_file.hetzner_csi_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml
EOT
# Upload traefik config
provisioner "file" {
content = templatefile(
"${path.module}/templates/traefik_config.yaml.tpl",
{
lb_disable_ipv6 = var.lb_disable_ipv6
lb_server_type = var.lb_server_type
location = var.location
traefik_acme_tls = var.traefik_acme_tls
traefik_acme_email = var.traefik_acme_email
})
destination = "/tmp/post_install/traefik.yaml"
}
# Install Kured
provisioner "local-exec" {
command = <<-EOT
set -ex
kubectl -n kube-system apply -k ${dirname(local_file.kured_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml
EOT
# Deploy secrets, logging is automatically disabled due to sensitive variables
provisioner "remote-exec" {
inline = [
"kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}",
"kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token}",
]
}
# Configure the Traefik ingress controller
provisioner "local-exec" {
command = <<-EOT
set -ex
kubectl apply -f ${local_file.traefik_config.filename} --kubeconfig ${path.module}/kubeconfig.yaml
EOT
# Deploy our post-installation kustomization
provisioner "remote-exec" {
inline = [
# This ugly hack is here, because terraform serializes the
# embedded yaml files with "- |2", when there is more than
# one yamldocument in the embedded file. Kustomize does not understand
# that syntax and tries to parse the blocks content as a file, resulting
# in weird errors. so gnu sed with funny escaping is used to
# replace lines like "- |3" by "- |" (yaml block syntax).
# due to indendation this should not changes the embedded
# manifests themselves
"sed -i 's/^- |[0-9]\\+$/- |/g' /tmp/post_install/kustomization.yaml",
"kubectl apply -k /tmp/post_install",
]
}
network {

12
output.tf
View File

@ -7,3 +7,15 @@ output "agents_public_ip" {
value = hcloud_server.agents.*.ipv4_address
description = "The public IP addresses of the agent server."
}
output "kubeconfig_file" {
value = local.kubeconfig_external
description = "Kubeconfig file content with external IP address"
sensitive = true
}
output "kubeconfig" {
description = "Structured kubeconfig data to supply to other providers"
value = local.kubeconfig_data
sensitive = true
}

2
hetzner/ccm/patch.yaml → patches/ccm.yaml
View File

@ -14,4 +14,4 @@ spec:
- "--leader-elect=false"
- "--allow-untagged-cloud"
- "--allocate-node-cidrs=true"
- "--cluster-cidr=10.42.0.0/16"
- "--cluster-cidr=10.42.0.0/16"

8
hetzner/ccm/patch_latest.yaml → patches/ccm_latest.yaml
View File

@ -7,13 +7,13 @@ spec:
template:
spec:
containers:
- image: hetznercloud/hcloud-cloud-controller-manager:latest
imagePullPolicy: Always
name: hcloud-cloud-controller-manager
- name: hcloud-cloud-controller-manager
command:
- "/bin/hcloud-cloud-controller-manager"
- "--cloud-provider=hcloud"
- "--leader-elect=false"
- "--allow-untagged-cloud"
- "--allocate-node-cidrs=true"
- "--cluster-cidr=10.42.0.0/16"
- "--cluster-cidr=10.42.0.0/16"
image: hetznercloud/hcloud-cloud-controller-manager:latest
imagePullPolicy: Always

54
patches/csi_latest.yaml Normal file
View File

@ -0,0 +1,54 @@
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: hcloud-csi-controller
namespace: kube-system
spec:
template:
metadata:
labels:
app: hcloud-csi-controller
spec:
containers:
- name: csi-attacher
image: quay.io/k8scsi/csi-attacher:canary
imagePullPolicy: Always
- name: csi-resizer
image: quay.io/k8scsi/csi-resizer:canary
imagePullPolicy: Always
- name: csi-provisioner
image: quay.io/k8scsi/csi-provisioner:canary
imagePullPolicy: Always
- name: hcloud-csi-driver
image: hetznercloud/hcloud-csi-driver:latest
imagePullPolicy: Always
- name: liveness-probe
image: quay.io/k8scsi/livenessprobe:canary
imagePullPolicy: Always
volumes:
- name: socket-dir
emptyDir: {}
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: hcloud-csi-node
namespace: kube-system
labels:
app: hcloud-csi
spec:
selector:
matchLabels:
app: hcloud-csi
template:
spec:
containers:
- name: csi-node-driver-registrar
image: quay.io/k8scsi/csi-node-driver-registrar:canary
imagePullPolicy: Always
- name: hcloud-csi-driver
image: hetznercloud/hcloud-csi-driver:latest
imagePullPolicy: Always
- name: liveness-probe
image: quay.io/k8scsi/livenessprobe:canary
imagePullPolicy: Always

0
kured/patch.yaml → patches/kured.yaml
View File

35
servers.tf
View File

@ -15,31 +15,24 @@ resource "hcloud_server" "control_planes" {
"engine" = "k3s",
}
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
provisioner "file" {
content = templatefile("${path.module}/templates/config.ign.tpl", {
name = self.name
ssh_public_key = local.ssh_public_key
})
destination = "/root/config.ign"
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
# Install MicroOS
provisioner "remote-exec" {
inline = local.MicroOS_install_commands
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
# Issue a reboot command
@ -75,13 +68,6 @@ resource "hcloud_server" "control_planes" {
node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
})
destination = "/etc/rancher/k3s/config.yaml"
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
# Run an other control plane server
@ -102,13 +88,6 @@ resource "hcloud_server" "control_planes" {
done
EOT
]
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
}
network {

8
templates/hetzner_ccm.yaml.tpl
View File

@ -1,8 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${ccm_version}/ccm-networks.yaml"
patchesStrategicMerge:
- ${patch_name}.yaml

10
templates/hetzner_csi.yaml.tpl
View File

@ -1,10 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- "https://raw.githubusercontent.com/hetznercloud/csi-driver/${csi_version}/deploy/kubernetes/hcloud-csi.yml"
%{ if patch_name != "" }
patchesStrategicMerge:
- ${patch_name}.yaml
%{ endif }

8
templates/kured.yaml.tpl
View File

@ -1,8 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- "https://github.com/weaveworks/kured/releases/download/${version}/kured-${version}-dockerhub.yaml"
patchesStrategicMerge:
- patch.yaml

4
versions.tf
View File

@ -12,5 +12,9 @@ terraform {
source = "hashicorp/local"
version = ">= 2.0.0, < 3.0.0"
}
remote = {
source = "tenstad/remote"
version = "~> 0.0.23"
}
}
}