diff --git a/agents.tf b/agents.tf index 2944711..4f6e7e5 100644 --- a/agents.tf +++ b/agents.tf @@ -1,62 +1,46 @@ -resource "hcloud_server" "agents" { +module "agents" { + source = "./modules/host" + count = var.agents_num name = "k3s-agent-${count.index}" - image = data.hcloud_image.linux.name - rescue = "linux64" - server_type = var.agent_server_type - location = var.location - ssh_keys = [hcloud_ssh_key.k3s.id] - firewall_ids = [hcloud_firewall.k3s.id] - placement_group_id = hcloud_placement_group.k3s.id - + ssh_keys = [hcloud_ssh_key.k3s.id] + public_key = var.public_key + private_key = var.private_key + additional_public_keys = var.additional_public_keys + firewall_ids = [hcloud_firewall.k3s.id] + placement_group_id = hcloud_placement_group.k3s.id + location = var.location + network_id = hcloud_network.k3s.id + ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index) + server_type = var.control_plane_server_type labels = { "provisioner" = "terraform", - "engine" = "k3s", + "engine" = "k3s" + } + + hcloud_token = var.hcloud_token +} + +resource "null_resource" "agents" { + count = var.agents_num + + triggers = { + agent_id = module.agents[count.index].id } connection { user = "root" private_key = local.ssh_private_key agent_identity = local.ssh_identity - host = self.ipv4_address - } - - provisioner "file" { - content = local.ignition_config - destination = "/root/config.ign" - } - - # Combustion script file to install k3s-selinux - provisioner "file" { - content = local.combustion_script - destination = "/root/script" - } - - # Install MicroOS - provisioner "remote-exec" { - inline = local.microOS_install_commands - } - - # Issue a reboot command and wait for the node to reboot - provisioner "local-exec" { - command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3" - } - provisioner "local-exec" { - command = <<-EOT - until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null - do - echo "Waiting for MicroOS to reboot and become available..." - sleep 3 - done - EOT + host = module.agents[count.index].ipv4_address } # Generating k3s agent config file provisioner "file" { content = yamlencode({ - node-name = self.name + node-name = module.agents[count.index].name server = "https://${local.first_control_plane_network_ip}:6443" token = random_password.k3s_token.result kubelet-arg = "cloud-provider=external" @@ -88,13 +72,8 @@ resource "hcloud_server" "agents" { ] } - network { - network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index) - } - depends_on = [ - hcloud_server.first_control_plane, + null_resource.first_control_plane, hcloud_network_subnet.k3s ] } diff --git a/data.tf b/data.tf index 20447c5..41d1cff 100644 --- a/data.tf +++ b/data.tf @@ -15,8 +15,4 @@ data "github_release" "kured" { repository = "kured" owner = "weaveworks" retrieve_by = "latest" -} - -data "hcloud_image" "linux" { - name = local.hcloud_image_name -} +} \ No newline at end of file diff --git a/kubeconfig.tf b/kubeconfig.tf index 07db0d2..ce92d94 100644 --- a/kubeconfig.tf +++ b/kubeconfig.tf @@ -1,17 +1,19 @@ data "remote_file" "kubeconfig" { conn { - host = hcloud_server.first_control_plane.ipv4_address + host = module.first_control_plane.ipv4_address port = 22 user = "root" private_key = local.ssh_private_key agent = var.private_key == null } path = "/etc/rancher/k3s/k3s.yaml" + + depends_on = [null_resource.first_control_plane] } locals { - kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", hcloud_server.first_control_plane.ipv4_address) + kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", module.first_control_plane.ipv4_address) kubeconfig_parsed = yamldecode(local.kubeconfig_external) kubeconfig_data = { host = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"] diff --git a/locals.tf b/locals.tf index 9ccfbc0..0aa10e8 100644 --- a/locals.tf +++ b/locals.tf @@ -1,6 +1,5 @@ locals { first_control_plane_network_ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 257) - hcloud_image_name = "ubuntu-20.04" ssh_public_key = trimspace(file(var.public_key)) # ssh_private_key is either the contents of var.private_key or null to use a ssh agent. @@ -18,67 +17,8 @@ locals { csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag kured_version = data.github_release.kured.release_tag - microOS_install_commands = [ - "set -ex", - "apt-get update", - "apt-get install -y aria2", - "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-kvm-and-xen.qcow2.meta4", - "qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda", - "sgdisk -e /dev/sda", - "parted -s /dev/sda resizepart 4 99%", - "parted -s /dev/sda mkpart primary ext2 99% 100%", - "partprobe /dev/sda && udevadm settle && fdisk -l /dev/sda", - "mount /dev/sda4 /mnt/ && btrfs filesystem resize max /mnt && umount /mnt", - "mke2fs -L ignition /dev/sda5", - "mount /dev/sda5 /mnt", - "mkdir /mnt/ignition", - "cp /root/config.ign /mnt/ignition/config.ign", - "mkdir /mnt/combustion", - "cp /root/script /mnt/combustion/script", - "umount /mnt" - ] - - ignition_config = jsonencode({ - ignition = { - version = "3.0.0" - } - passwd = { - users = [{ - name = "root" - sshAuthorizedKeys = concat([local.ssh_public_key], var.additional_public_keys) - }] - } - storage = { - files = [ - { - path = "/etc/sysconfig/network/ifcfg-eth1" - mode = 420 - overwrite = true - contents = { "source" = "data:,BOOTPROTO%3D%27dhcp%27%0ASTARTMODE%3D%27auto%27" } - }, - { - path = "/etc/ssh/sshd_config.d/kube-hetzner.conf" - mode = 420 - overwrite = true - contents = { "source" = "data:,PasswordAuthentication%20no%0AX11Forwarding%20no%0AMaxAuthTries%202%0AAllowTcpForwarding%20no%0AAllowAgentForwarding%20no%0AAuthorizedKeysFile%20.ssh%2Fauthorized_keys" } - } - ] - } - }) - - combustion_script = < /etc/transactional-update.conf", # prepare the k3s config directory "mkdir -p /etc/rancher/k3s", # move the config file into place @@ -88,5 +28,4 @@ udevadm settle install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_EXEC=server sh -"]) install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_EXEC=agent sh -"]) - } diff --git a/master.tf b/master.tf index a0015dd..cfd3283 100644 --- a/master.tf +++ b/master.tf @@ -1,60 +1,46 @@ -resource "hcloud_server" "first_control_plane" { +module "first_control_plane" { + source = "./modules/host" + name = "k3s-control-plane-0" - image = data.hcloud_image.linux.name - rescue = "linux64" - server_type = var.control_plane_server_type - location = var.location - ssh_keys = [hcloud_ssh_key.k3s.id] - firewall_ids = [hcloud_firewall.k3s.id] - placement_group_id = hcloud_placement_group.k3s.id + ssh_keys = [hcloud_ssh_key.k3s.id] + public_key = var.public_key + private_key = var.private_key + additional_public_keys = var.additional_public_keys + firewall_ids = [hcloud_firewall.k3s.id] + placement_group_id = hcloud_placement_group.k3s.id + location = var.location + network_id = hcloud_network.k3s.id + ip = local.first_control_plane_network_ip + server_type = var.control_plane_server_type labels = { "provisioner" = "terraform", "engine" = "k3s" } + hcloud_token = var.hcloud_token +} + +resource "null_resource" "first_control_plane" { + + triggers = { + first_control_plane_id = module.first_control_plane.id + } + + connection { user = "root" private_key = local.ssh_private_key agent_identity = local.ssh_identity - host = self.ipv4_address - } - - provisioner "file" { - content = local.ignition_config - destination = "/root/config.ign" - } - - # Combustion script file to install k3s-selinux - provisioner "file" { - content = local.combustion_script - destination = "/root/script" - } - - # Install MicroOS - provisioner "remote-exec" { - inline = local.microOS_install_commands - } - - # Issue a reboot command and wait for the node to reboot - provisioner "local-exec" { - command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3" - } - provisioner "local-exec" { - command = <<-EOT - until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null - do - echo "Waiting for MicroOS to reboot and become available..." - sleep 3 - done - EOT + host = module.first_control_plane.ipv4_address } # Generating k3s master config file provisioner "file" { content = yamlencode({ - node-name = self.name + node-name = module.first_control_plane.name + token = random_password.k3s_token.result cluster-init = true disable-cloud-controller = true disable = ["servicelb", "local-storage"] @@ -62,7 +48,7 @@ resource "hcloud_server" "first_control_plane" { kubelet-arg = "cloud-provider=external" node-ip = local.first_control_plane_network_ip advertise-address = local.first_control_plane_network_ip - token = random_password.k3s_token.result + tls-san = local.first_control_plane_network_ip node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"] node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] }) @@ -175,11 +161,6 @@ resource "hcloud_server" "first_control_plane" { ] } - network { - network_id = hcloud_network.k3s.id - ip = local.first_control_plane_network_ip - } - depends_on = [ hcloud_network_subnet.k3s, hcloud_firewall.k3s diff --git a/modules/host/locals.tf b/modules/host/locals.tf new file mode 100644 index 0000000..5a61b2a --- /dev/null +++ b/modules/host/locals.tf @@ -0,0 +1,71 @@ +locals { + ssh_public_key = trimspace(file(var.public_key)) + # ssh_private_key is either the contents of var.private_key or null to use a ssh agent. + ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key)) + # ssh_identity is not set if the private key is passed directly, but if ssh agent is used, the public key tells ssh agent which private key to use. + # For terraforms provisioner.connection.agent_identity, we need the public key as a string. + ssh_identity = var.private_key == null ? local.ssh_public_key : null + # ssh_identity_file is used for ssh "-i" flag, its the private key if that is set, or a public key file + # if an ssh agent is used. + ssh_identity_file = var.private_key == null ? var.public_key : var.private_key + # shared flags for ssh to ignore host keys, to use our ssh identity file for all connections during provisioning. + ssh_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${local.ssh_identity_file}" + + microOS_install_commands = [ + "set -ex", + "apt-get update", + "apt-get install -y aria2", + "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-kvm-and-xen.qcow2.meta4", + "qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda", + "sgdisk -e /dev/sda", + "parted -s /dev/sda resizepart 4 99%", + "parted -s /dev/sda mkpart primary ext2 99% 100%", + "partprobe /dev/sda && udevadm settle && fdisk -l /dev/sda", + "mount /dev/sda4 /mnt/ && btrfs filesystem resize max /mnt && umount /mnt", + "mke2fs -L ignition /dev/sda5", + "mount /dev/sda5 /mnt", + "mkdir /mnt/ignition", + "cp /root/config.ign /mnt/ignition/config.ign", + "mkdir /mnt/combustion", + "cp /root/script /mnt/combustion/script", + "umount /mnt" + ] + + ignition_config = jsonencode({ + ignition = { + version = "3.0.0" + } + passwd = { + users = [{ + name = "root" + sshAuthorizedKeys = concat([local.ssh_public_key], var.additional_public_keys) + }] + } + storage = { + files = [ + { + path = "/etc/sysconfig/network/ifcfg-eth1" + mode = 420 + overwrite = true + contents = { "source" = "data:,BOOTPROTO%3D%27dhcp%27%0ASTARTMODE%3D%27auto%27" } + }, + { + path = "/etc/ssh/sshd_config.d/kube-hetzner.conf" + mode = 420 + overwrite = true + contents = { "source" = "data:,PasswordAuthentication%20no%0AX11Forwarding%20no%0AMaxAuthTries%202%0AAllowTcpForwarding%20no%0AAllowAgentForwarding%20no%0AAuthorizedKeysFile%20.ssh%2Fauthorized_keys" } + } + ] + } + }) + + combustion_script = < /dev/null + do + echo "Waiting for MicroOS to reboot and become available..." + sleep 3 + done + EOT + } + + # Run the agent + provisioner "remote-exec" { + inline = [ + # set the hostname in a persistent fashion + "hostnamectl set-hostname ${self.name}", + # Disable automatic reboot (after transactional updates), and configure the reboot method as kured + "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf" + ] + } +} diff --git a/modules/host/out.tf b/modules/host/out.tf new file mode 100644 index 0000000..905ddff --- /dev/null +++ b/modules/host/out.tf @@ -0,0 +1,12 @@ +output "ipv4_address" { + value = hcloud_server.server.ipv4_address +} + + +output "name" { + value = hcloud_server.server.name +} + +output "id" { + value = hcloud_server.server.id +} diff --git a/modules/host/variables.tf b/modules/host/variables.tf new file mode 100644 index 0000000..b336fc5 --- /dev/null +++ b/modules/host/variables.tf @@ -0,0 +1,71 @@ +variable "hcloud_token" { + description = "Hetzner Cloud API Token" + type = string + sensitive = true +} + +variable "name" { + description = "Host name" + type = string +} + +variable "public_key" { + description = "SSH public Key." + type = string +} + +variable "private_key" { + description = "SSH private Key." + type = string +} + +variable "additional_public_keys" { + description = "Additional SSH public Keys. Use them to grant other team members root access to your cluster nodes" + type = list(string) + default = [] +} + +variable "ssh_keys" { + description = "List of SSH key IDs" + type = list(string) + nullable = true +} + +variable "firewall_ids" { + description = "Set of firewal IDs" + type = set(number) + nullable = true +} + +variable "placement_group_id" { + description = "Placement group ID" + type = number + nullable = true +} + +variable "labels" { + description = "Labels" + type = map(any) + nullable = true +} + +variable "location" { + description = "The server location" + type = string +} + +variable "network_id" { + description = "The network or subnet id" + type = number +} + +variable "ip" { + description = "The IP" + type = string + nullable = true +} + +variable "server_type" { + description = "The server type" + type = string +} diff --git a/modules/host/versions.tf b/modules/host/versions.tf new file mode 100644 index 0000000..fe79022 --- /dev/null +++ b/modules/host/versions.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + hcloud = { + source = "hetznercloud/hcloud" + version = ">= 1.0.0, < 2.0.0" + } + local = { + source = "hashicorp/local" + version = ">= 2.0.0, < 3.0.0" + } + remote = { + source = "tenstad/remote" + version = "~> 0.0.23" + } + } +} diff --git a/output.tf b/output.tf index 330d587..8abb440 100644 --- a/output.tf +++ b/output.tf @@ -1,10 +1,10 @@ output "controlplanes_public_ip" { - value = concat([hcloud_server.first_control_plane.ipv4_address], hcloud_server.control_planes.*.ipv4_address) + value = concat([module.first_control_plane.ipv4_address], module.control_planes.*.ipv4_address) description = "The public IP addresses of the controlplane server." } output "agents_public_ip" { - value = hcloud_server.agents.*.ipv4_address + value = module.agents.*.ipv4_address description = "The public IP addresses of the agent server." } diff --git a/servers.tf b/servers.tf index e1d069b..5bc9ba4 100644 --- a/servers.tf +++ b/servers.tf @@ -1,66 +1,51 @@ -resource "hcloud_server" "control_planes" { +module "control_planes" { + source = "./modules/host" + count = var.servers_num - 1 name = "k3s-control-plane-${count.index + 1}" - image = data.hcloud_image.linux.name - rescue = "linux64" - server_type = var.control_plane_server_type - location = var.location - ssh_keys = [hcloud_ssh_key.k3s.id] - firewall_ids = [hcloud_firewall.k3s.id] - placement_group_id = hcloud_placement_group.k3s.id + ssh_keys = [hcloud_ssh_key.k3s.id] + public_key = var.public_key + private_key = var.private_key + additional_public_keys = var.additional_public_keys + firewall_ids = [hcloud_firewall.k3s.id] + placement_group_id = hcloud_placement_group.k3s.id + location = var.location + network_id = hcloud_network.k3s.id + ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 258 + count.index) + server_type = var.control_plane_server_type labels = { "provisioner" = "terraform", - "engine" = "k3s", + "engine" = "k3s" + } + + hcloud_token = var.hcloud_token +} + +resource "null_resource" "control_planes" { + count = var.servers_num - 1 + + triggers = { + control_plane_id = module.control_planes[count.index].id } connection { user = "root" private_key = local.ssh_private_key agent_identity = local.ssh_identity - host = self.ipv4_address - } - - provisioner "file" { - content = local.ignition_config - destination = "/root/config.ign" - } - - # Combustion script file to install k3s-selinux - provisioner "file" { - content = local.combustion_script - destination = "/root/script" - } - - # Install MicroOS - provisioner "remote-exec" { - inline = local.microOS_install_commands - } - - # Issue a reboot command and wait for the node to reboot - provisioner "local-exec" { - command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3" - } - provisioner "local-exec" { - command = <<-EOT - until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null - do - echo "Waiting for MicroOS to reboot and become available..." - sleep 3 - done - EOT + host = module.control_planes[count.index].ipv4_address } # Generating k3s server config file provisioner "file" { content = yamlencode({ - node-name = self.name + node-name = module.control_planes[count.index].name server = "https://${local.first_control_plane_network_ip}:6443" token = random_password.k3s_token.result cluster-init = true disable-cloud-controller = true - disable = "servicelb, local-storage" + disable = ["servicelb", "local-storage"] flannel-iface = "eth1" kubelet-arg = "cloud-provider=external" node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 258 + count.index) @@ -93,13 +78,8 @@ resource "hcloud_server" "control_planes" { ] } - network { - network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 258 + count.index) - } - depends_on = [ - hcloud_server.first_control_plane, + null_resource.first_control_plane, hcloud_network_subnet.k3s ] }