228 lines
8.5 KiB
HCL
228 lines
8.5 KiB
HCL
resource "null_resource" "first_control_plane" {
|
|
connection {
|
|
user = "root"
|
|
private_key = local.ssh_private_key
|
|
agent_identity = local.ssh_identity
|
|
host = module.control_planes[keys(module.control_planes)[0]].ipv4_address
|
|
}
|
|
|
|
# Generating k3s master config file
|
|
provisioner "file" {
|
|
content = yamlencode(merge({
|
|
node-name = module.control_planes[keys(module.control_planes)[0]].name
|
|
token = random_password.k3s_token.result
|
|
cluster-init = true
|
|
disable-cloud-controller = true
|
|
disable = local.disable_extras
|
|
flannel-iface = "eth1"
|
|
kubelet-arg = ["cloud-provider=external", "volume-plugin-dir=/var/lib/kubelet/volumeplugins"]
|
|
kube-controller-manager-arg = "flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins"
|
|
node-ip = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
|
|
advertise-address = module.control_planes[keys(module.control_planes)[0]].private_ipv4_address
|
|
node-taint = local.control_plane_nodes[keys(module.control_planes)[0]].taints
|
|
node-label = local.control_plane_nodes[keys(module.control_planes)[0]].labels
|
|
disable-network-policy = var.cni_plugin == "calico" ? true : var.disable_network_policy
|
|
},
|
|
var.cni_plugin == "calico" ? {
|
|
flannel-backend = "none"
|
|
} : {}))
|
|
|
|
destination = "/tmp/config.yaml"
|
|
}
|
|
|
|
# Install k3s server
|
|
provisioner "remote-exec" {
|
|
inline = local.install_k3s_server
|
|
}
|
|
|
|
# Upon reboot start k3s and wait for it to be ready to receive commands
|
|
provisioner "remote-exec" {
|
|
inline = [
|
|
"systemctl start k3s",
|
|
# prepare the post_install directory
|
|
"mkdir -p /var/post_install",
|
|
# wait for k3s to become ready
|
|
<<-EOT
|
|
timeout 120 bash <<EOF
|
|
until systemctl status k3s > /dev/null; do
|
|
systemctl start k3s
|
|
echo "Waiting for the k3s server to start..."
|
|
sleep 2
|
|
done
|
|
until [ -e /etc/rancher/k3s/k3s.yaml ]; do
|
|
echo "Waiting for kubectl config..."
|
|
sleep 2
|
|
done
|
|
until [[ "\$(kubectl get --raw='/readyz' 2> /dev/null)" == "ok" ]]; do
|
|
echo "Waiting for the cluster to become ready..."
|
|
sleep 2
|
|
done
|
|
EOF
|
|
EOT
|
|
]
|
|
}
|
|
|
|
depends_on = [
|
|
hcloud_network_subnet.control_plane
|
|
]
|
|
}
|
|
|
|
resource "null_resource" "kustomization" {
|
|
connection {
|
|
user = "root"
|
|
private_key = local.ssh_private_key
|
|
agent_identity = local.ssh_identity
|
|
host = module.control_planes[keys(module.control_planes)[0]].ipv4_address
|
|
}
|
|
|
|
# Upload kustomization.yaml, containing Hetzner CSI & CSM, as well as kured.
|
|
provisioner "file" {
|
|
content = yamlencode({
|
|
apiVersion = "kustomize.config.k8s.io/v1beta1"
|
|
kind = "Kustomization"
|
|
|
|
resources = concat(
|
|
[
|
|
"https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${local.ccm_version}/ccm-networks.yaml",
|
|
"https://github.com/weaveworks/kured/releases/download/${local.kured_version}/kured-${local.kured_version}-dockerhub.yaml",
|
|
"https://raw.githubusercontent.com/rancher/system-upgrade-controller/master/manifests/system-upgrade-controller.yaml",
|
|
],
|
|
var.disable_hetzner_csi ? [] : ["https://raw.githubusercontent.com/hetznercloud/csi-driver/${local.csi_version}/deploy/kubernetes/hcloud-csi.yml"],
|
|
local.is_single_node_cluster ? [] : var.traefik_enabled ? ["traefik_config.yaml"] : [],
|
|
var.cni_plugin == "calico" ? ["https://projectcalico.docs.tigera.io/manifests/calico.yaml"] : [],
|
|
var.enable_longhorn ? ["longhorn.yaml"] : [],
|
|
var.enable_cert_manager || var.enable_rancher ? ["cert-manager.yaml"] : [],
|
|
var.enable_rancher ? ["rancher.yaml"] : [],
|
|
),
|
|
patchesStrategicMerge = concat(
|
|
[
|
|
file("${path.module}/kustomize/kured.yaml"),
|
|
file("${path.module}/kustomize/system-upgrade-controller.yaml"),
|
|
"ccm.yaml",
|
|
],
|
|
var.cni_plugin == "calico" ? ["calico.yaml"] : []
|
|
)
|
|
})
|
|
destination = "/var/post_install/kustomization.yaml"
|
|
}
|
|
|
|
# Upload traefik config
|
|
provisioner "file" {
|
|
content = local.is_single_node_cluster || var.traefik_enabled == false ? "" : templatefile(
|
|
"${path.module}/templates/traefik_config.yaml.tpl",
|
|
{
|
|
name = "${var.cluster_name}-traefik"
|
|
load_balancer_disable_ipv6 = var.load_balancer_disable_ipv6
|
|
load_balancer_type = var.load_balancer_type
|
|
location = var.load_balancer_location
|
|
traefik_acme_tls = var.traefik_acme_tls
|
|
traefik_acme_email = var.traefik_acme_email
|
|
traefik_additional_options = var.traefik_additional_options
|
|
})
|
|
destination = "/var/post_install/traefik_config.yaml"
|
|
}
|
|
|
|
# Upload the CCM patch config
|
|
provisioner "file" {
|
|
content = templatefile(
|
|
"${path.module}/templates/ccm.yaml.tpl",
|
|
{
|
|
cluster_cidr_ipv4 = local.cluster_cidr_ipv4
|
|
})
|
|
destination = "/var/post_install/ccm.yaml"
|
|
}
|
|
|
|
# Upload the calico patch config
|
|
provisioner "file" {
|
|
content = templatefile(
|
|
"${path.module}/templates/calico.yaml.tpl",
|
|
{
|
|
cluster_cidr_ipv4 = local.cluster_cidr_ipv4
|
|
})
|
|
destination = "/var/post_install/calico.yaml"
|
|
}
|
|
|
|
# Upload the system upgrade controller plans config
|
|
provisioner "file" {
|
|
content = templatefile(
|
|
"${path.module}/templates/plans.yaml.tpl",
|
|
{
|
|
channel = var.initial_k3s_channel
|
|
})
|
|
destination = "/var/post_install/plans.yaml"
|
|
}
|
|
|
|
# Upload the Longhorn config
|
|
provisioner "file" {
|
|
content = templatefile(
|
|
"${path.module}/templates/longhorn.yaml.tpl",
|
|
{
|
|
disable_hetzner_csi = var.disable_hetzner_csi
|
|
})
|
|
destination = "/var/post_install/longhorn.yaml"
|
|
}
|
|
|
|
# Upload the cert-manager config
|
|
provisioner "file" {
|
|
content = templatefile(
|
|
"${path.module}/templates/cert-manager.yaml.tpl",
|
|
{})
|
|
destination = "/var/post_install/cert-manager.yaml"
|
|
}
|
|
|
|
# Upload the Rancher config
|
|
provisioner "file" {
|
|
content = templatefile(
|
|
"${path.module}/templates/rancher.yaml.tpl",
|
|
{
|
|
rancher_install_channel = var.rancher_install_channel
|
|
rancher_hostname = var.rancher_hostname
|
|
number_control_plane_nodes = length(local.control_plane_nodes)
|
|
})
|
|
destination = "/var/post_install/rancher.yaml"
|
|
}
|
|
|
|
# Deploy secrets, logging is automatically disabled due to sensitive variables
|
|
provisioner "remote-exec" {
|
|
inline = [
|
|
"set -ex",
|
|
"kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --dry-run=client -o yaml | kubectl apply -f -",
|
|
"kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --dry-run=client -o yaml | kubectl apply -f -",
|
|
]
|
|
}
|
|
|
|
# Deploy our post-installation kustomization
|
|
provisioner "remote-exec" {
|
|
inline = concat([
|
|
"set -ex",
|
|
# This ugly hack is here, because terraform serializes the
|
|
# embedded yaml files with "- |2", when there is more than
|
|
# one yamldocument in the embedded file. Kustomize does not understand
|
|
# that syntax and tries to parse the blocks content as a file, resulting
|
|
# in weird errors. so gnu sed with funny escaping is used to
|
|
# replace lines like "- |3" by "- |" (yaml block syntax).
|
|
# due to indendation this should not changes the embedded
|
|
# manifests themselves
|
|
"sed -i 's/^- |[0-9]\\+$/- |/g' /var/post_install/kustomization.yaml",
|
|
"kubectl apply -k /var/post_install",
|
|
"echo 'Waiting for the system-upgrade-controller deployment to become available...'",
|
|
"kubectl -n system-upgrade wait --for=condition=available --timeout=120s deployment/system-upgrade-controller",
|
|
"kubectl -n system-upgrade apply -f /var/post_install/plans.yaml"
|
|
],
|
|
local.is_single_node_cluster || var.traefik_enabled == false ? [] : [<<-EOT
|
|
timeout 120 bash <<EOF
|
|
until [ -n "\$(kubectl get -n kube-system service/traefik --output=jsonpath='{.status.loadBalancer.ingress[0].ip}' 2> /dev/null)" ]; do
|
|
echo "Waiting for load-balancer to get an IP..."
|
|
sleep 2
|
|
done
|
|
EOF
|
|
EOT
|
|
])
|
|
}
|
|
|
|
depends_on = [
|
|
null_resource.first_control_plane,
|
|
local_sensitive_file.kubeconfig
|
|
]
|
|
}
|