kjuulh/terraform-hcloud-kube-hetzner
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Fist attempt to factor microOS install

Browse Source
This commit is contained in:
Marco Nenciarini 2022-02-19 13:38:24 +01:00
parent 2a47ad6b89
commit 0090c1a6f1
No known key found for this signature in database
GPG Key ID: 589F03F01BA55038
13 changed files with 315 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
agents.tf
View File

@ -1,65 +1,45 @@
resource "hcloud_server" "agents" {
module "agents" {
source = "./modules/host"
count = var.agents_num
name = "k3s-agent-${count.index}"
image = data.hcloud_image.linux.name
rescue = "linux64"
server_type = var.agent_server_type
location = var.location
ssh_keys = [hcloud_ssh_key.k3s.id]
public_key = var.public_key
private_key = var.private_key
firewall_ids = [hcloud_firewall.k3s.id]
placement_group_id = hcloud_placement_group.k3s.id
location = var.location
network_id = hcloud_network.k3s.id
ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index)
server_type = var.control_plane_server_type
labels = {
"provisioner" = "terraform",
"engine" = "k3s",
"engine" = "k3s"
}
hcloud_token = var.hcloud_token
}
resource "null_resource" "agents" {
count = var.agents_num
triggers = {
agent_id = module.agents[count.index].id
}
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
provisioner "file" {
content = templatefile("${path.module}/templates/config.ign.tpl", {
name = self.name
ssh_public_key = local.ssh_public_key
})
destination = "/root/config.ign"
}
# Combustion script file to install k3s-selinux
provisioner "file" {
content = local.combustion_script
destination = "/root/script"
}
# Install MicroOS
provisioner "remote-exec" {
inline = local.microOS_install_commands
}
# Issue a reboot command and wait for the node to reboot
provisioner "local-exec" {
command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
}
provisioner "local-exec" {
command = <<-EOT
until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
do
echo "Waiting for MicroOS to reboot and become available..."
sleep 3
done
EOT
host = module.agents[count.index].ipv4_address
}
# Generating k3s agent config file
provisioner "file" {
content = yamlencode({
node-name = self.name
node-name = module.agents[count.index].name
server = "https://${local.first_control_plane_network_ip}:6443"
token = random_password.k3s_token.result
kubelet-arg = "cloud-provider=external"
@ -89,13 +69,8 @@ resource "hcloud_server" "agents" {
]
}
network {
network_id = hcloud_network.k3s.id
ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 513 + count.index)
}
depends_on = [
hcloud_server.first_control_plane,
null_resource.first_control_plane,
hcloud_network_subnet.k3s
]
}

4
data.tf
View File

@ -16,7 +16,3 @@ data "github_release" "kured" {
owner = "weaveworks"
retrieve_by = "latest"
}
data "hcloud_image" "linux" {
name = local.hcloud_image_name
}

6
kubeconfig.tf
View File

@ -1,17 +1,19 @@
data "remote_file" "kubeconfig" {
conn {
host = hcloud_server.first_control_plane.ipv4_address
host = module.first_control_plane.ipv4_address
port = 22
user = "root"
private_key = local.ssh_private_key
agent = var.private_key == null
}
path = "/etc/rancher/k3s/k3s.yaml"
depends_on = [null_resource.first_control_plane]
}
locals {
kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", hcloud_server.first_control_plane.ipv4_address)
kubeconfig_external = replace(data.remote_file.kubeconfig.content, "127.0.0.1", module.first_control_plane.ipv4_address)
kubeconfig_parsed = yamldecode(local.kubeconfig_external)
kubeconfig_data = {
host = local.kubeconfig_parsed["clusters"][0]["cluster"]["server"]

31
locals.tf
View File

@ -1,6 +1,5 @@
locals {
first_control_plane_network_ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 257)
hcloud_image_name = "ubuntu-20.04"
ssh_public_key = trimspace(file(var.public_key))
# ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
@ -18,39 +17,9 @@ locals {
csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag
kured_version = data.github_release.kured.release_tag
microOS_install_commands = [
"set -ex",
"apt-get update",
"apt-get install -y aria2",
"aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-kvm-and-xen.qcow2.meta4",
"qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda",
"sgdisk -e /dev/sda",
"parted -s /dev/sda resizepart 4 99%",
"parted -s /dev/sda mkpart primary ext2 99% 100%",
"partprobe /dev/sda && udevadm settle && fdisk -l /dev/sda",
"mount /dev/sda4 /mnt/ && btrfs filesystem resize max /mnt && umount /mnt",
"mke2fs -L ignition /dev/sda5",
"mount /dev/sda5 /mnt",
"mkdir /mnt/ignition",
"cp /root/config.ign /mnt/ignition/config.ign",
"mkdir /mnt/combustion",
"cp /root/script /mnt/combustion/script",
"umount /mnt"
]
combustion_script = <<EOF
#!/bin/bash
# combustion: network
rpm --import https://rpm.rancher.io/public.key
zypper refresh
zypper --gpg-auto-import-keys install -y https://rpm.rancher.io/k3s/stable/common/microos/noarch/k3s-selinux-0.4-1.sle.noarch.rpm
udevadm settle
EOF
common_commands_install_k3s = [
"set -ex",
# first we disable automatic reboot (after transactional updates), and configure the reboot method as kured
"rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf",
# prepare the k3s config directory
"mkdir -p /etc/rancher/k3s",
# move the config file into place

69
master.tf
View File

@ -1,63 +1,45 @@
resource "hcloud_server" "first_control_plane" {
module "first_control_plane" {
source = "./modules/host"
name = "k3s-control-plane-0"
image = data.hcloud_image.linux.name
rescue = "linux64"
server_type = var.control_plane_server_type
location = var.location
ssh_keys = [hcloud_ssh_key.k3s.id]
public_key = var.public_key
private_key = var.private_key
firewall_ids = [hcloud_firewall.k3s.id]
placement_group_id = hcloud_placement_group.k3s.id
location = var.location
network_id = hcloud_network.k3s.id
ip = local.first_control_plane_network_ip
server_type = var.control_plane_server_type
labels = {
"provisioner" = "terraform",
"engine" = "k3s"
}
hcloud_token = var.hcloud_token
}
resource "null_resource" "first_control_plane" {
triggers = {
first_control_plane_id = module.first_control_plane.id
}
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
provisioner "file" {
content = templatefile("${path.module}/templates/config.ign.tpl", {
name = self.name
ssh_public_key = local.ssh_public_key
})
destination = "/root/config.ign"
}
# Combustion script file to install k3s-selinux
provisioner "file" {
content = local.combustion_script
destination = "/root/script"
}
# Install MicroOS
provisioner "remote-exec" {
inline = local.microOS_install_commands
}
# Issue a reboot command and wait for the node to reboot
provisioner "local-exec" {
command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
}
provisioner "local-exec" {
command = <<-EOT
until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
do
echo "Waiting for MicroOS to reboot and become available..."
sleep 3
done
EOT
host = module.first_control_plane.ipv4_address
}
# Generating k3s master config file
provisioner "file" {
content = yamlencode({
node-name = self.name
node-name = module.first_control_plane.name
token = random_password.k3s_token.result
cluster-init = true
disable-cloud-controller = true
disable = ["servicelb", "local-storage"]
@ -65,7 +47,7 @@ resource "hcloud_server" "first_control_plane" {
kubelet-arg = "cloud-provider=external"
node-ip = local.first_control_plane_network_ip
advertise-address = local.first_control_plane_network_ip
token = random_password.k3s_token.result
tls-san = local.first_control_plane_network_ip
node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"]
node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : []
})
@ -177,11 +159,6 @@ resource "hcloud_server" "first_control_plane" {
]
}
network {
network_id = hcloud_network.k3s.id
ip = local.first_control_plane_network_ip
}
depends_on = [
hcloud_network_subnet.k3s,
hcloud_firewall.k3s

43
modules/host/locals.tf Normal file
View File

@ -0,0 +1,43 @@
locals {
ssh_public_key = trimspace(file(var.public_key))
# ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key))
# ssh_identity is not set if the private key is passed directly, but if ssh agent is used, the public key tells ssh agent which private key to use.
# For terraforms provisioner.connection.agent_identity, we need the public key as a string.
ssh_identity = var.private_key == null ? local.ssh_public_key : null
# ssh_identity_file is used for ssh "-i" flag, its the private key if that is set, or a public key file
# if an ssh agent is used.
ssh_identity_file = var.private_key == null ? var.public_key : var.private_key
# shared flags for ssh to ignore host keys, to use our ssh identity file for all connections during provisioning.
ssh_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${local.ssh_identity_file}"
microOS_install_commands = [
"set -ex",
"apt-get update",
"apt-get install -y aria2",
"aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-kvm-and-xen.qcow2.meta4",
"qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda",
"sgdisk -e /dev/sda",
"parted -s /dev/sda resizepart 4 99%",
"parted -s /dev/sda mkpart primary ext2 99% 100%",
"partprobe /dev/sda && udevadm settle && fdisk -l /dev/sda",
"mount /dev/sda4 /mnt/ && btrfs filesystem resize max /mnt && umount /mnt",
"mke2fs -L ignition /dev/sda5",
"mount /dev/sda5 /mnt",
"mkdir /mnt/ignition",
"cp /root/config.ign /mnt/ignition/config.ign",
"mkdir /mnt/combustion",
"cp /root/script /mnt/combustion/script",
"umount /mnt"
]
combustion_script = <<EOF
#!/bin/bash
# combustion: network
rpm --import https://rpm.rancher.io/public.key
zypper refresh
zypper --gpg-auto-import-keys install -y https://rpm.rancher.io/k3s/stable/common/microos/noarch/k3s-selinux-0.4-1.sle.noarch.rpm
udevadm settle
EOF
}

71
modules/host/main.tf Normal file
View File

@ -0,0 +1,71 @@
resource "hcloud_server" "server" {
name = var.name
image = "ubuntu-20.04"
rescue = "linux64"
server_type = var.server_type
location = var.location
ssh_keys = var.ssh_keys
firewall_ids = var.firewall_ids
placement_group_id = var.placement_group_id
labels = var.labels
network {
network_id = var.network_id
ip = var.ip
}
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
provisioner "file" {
content = templatefile("${path.module}/templates/config.ign.tpl", {
name = self.name
ssh_public_key = local.ssh_public_key
})
destination = "/root/config.ign"
}
# Combustion script file to install k3s-selinux
provisioner "file" {
content = local.combustion_script
destination = "/root/script"
}
# Install MicroOS
provisioner "remote-exec" {
inline = local.microOS_install_commands
}
# Issue a reboot command
provisioner "local-exec" {
command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
}
# Wait for MicroOS to reboot and be ready
provisioner "local-exec" {
command = <<-EOT
until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
do
echo "Waiting for MicroOS to reboot and become available..."
sleep 2
done
EOT
}
# Run the agent
provisioner "remote-exec" {
inline = [
# set the hostname in a persistent fashion
"hostnamectl set-hostname ${self.name}",
# Disable automatic reboot (after transactional updates), and configure the reboot method as kured
"rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf"
]
}
}

12
modules/host/out.tf Normal file
View File

@ -0,0 +1,12 @@
output "ipv4_address" {
value = hcloud_server.server.ipv4_address
}
output "name" {
value = hcloud_server.server.name
}
output "id" {
value = hcloud_server.server.id
}

31
modules/host/templates/config.ign.tpl Normal file
View File

@ -0,0 +1,31 @@
{
"ignition": {
"version": "3.0.0"
},
"passwd": {
"users": [
{
"name": "root",
"sshAuthorizedKeys": [
"${ssh_public_key}"
]
}
]
},
"storage": {
"files": [
{
"path": "/etc/sysconfig/network/ifcfg-eth1",
"mode": 420,
"overwrite": true,
"contents": { "source": "data:,BOOTPROTO%3D%27dhcp%27%0ASTARTMODE%3D%27auto%27" }
},
{
"path": "/etc/ssh/sshd_config.d/kube-hetzner.conf",
"mode": 420,
"overwrite": true,
"contents": { "source": "data:,PasswordAuthentication%20no%0AX11Forwarding%20no%0AMaxAuthTries%202%0AAllowTcpForwarding%20no%0AAllowAgentForwarding%20no%0AAuthorizedKeysFile%20.ssh%2Fauthorized_keys" }
}
]
}
}

65
modules/host/variables.tf Normal file
View File

@ -0,0 +1,65 @@
variable "hcloud_token" {
description = "Hetzner Cloud API Token"
type = string
sensitive = true
}
variable "name" {
description = "Host name"
type = string
}
variable "public_key" {
description = "SSH public Key."
type = string
}
variable "private_key" {
description = "SSH private Key."
type = string
}
variable "ssh_keys" {
description = "List of SSH key IDs"
type = list(string)
nullable = true
}
variable "firewall_ids" {
description = "Set of firewal IDs"
type = set(number)
nullable = true
}
variable "placement_group_id" {
description = "Placement group ID"
type = number
nullable = true
}
variable "labels" {
description = "Labels"
type = map(any)
nullable = true
}
variable "location" {
description = "The server location"
type = string
}
variable "network_id" {
description = "The network or subnet id"
type = number
}
variable "ip" {
description = "The IP"
type = string
nullable = true
}
variable "server_type" {
description = "The server type"
type = string
}

16
modules/host/versions.tf Normal file
View File

@ -0,0 +1,16 @@
terraform {
required_providers {
hcloud = {
source = "hetznercloud/hcloud"
version = ">= 1.0.0, < 2.0.0"
}
local = {
source = "hashicorp/local"
version = ">= 2.0.0, < 3.0.0"
}
remote = {
source = "tenstad/remote"
version = "~> 0.0.23"
}
}
}

4
output.tf
View File

@ -1,10 +1,10 @@
output "controlplanes_public_ip" {
value = concat([hcloud_server.first_control_plane.ipv4_address], hcloud_server.control_planes.*.ipv4_address)
value = concat([module.first_control_plane.ipv4_address], module.control_planes.*.ipv4_address)
description = "The public IP addresses of the controlplane server."
}
output "agents_public_ip" {
value = hcloud_server.agents.*.ipv4_address
value = module.agents.*.ipv4_address
description = "The public IP addresses of the agent server."
}

72
servers.tf
View File

@ -1,69 +1,50 @@
resource "hcloud_server" "control_planes" {
module "control_planes" {
source = "./modules/host"
count = var.servers_num - 1
name = "k3s-control-plane-${count.index + 1}"
image = data.hcloud_image.linux.name
rescue = "linux64"
server_type = var.control_plane_server_type
location = var.location
ssh_keys = [hcloud_ssh_key.k3s.id]
public_key = var.public_key
private_key = var.private_key
firewall_ids = [hcloud_firewall.k3s.id]
placement_group_id = hcloud_placement_group.k3s.id
location = var.location
network_id = hcloud_network.k3s.id
ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 258 + count.index)
server_type = var.control_plane_server_type
labels = {
"provisioner" = "terraform",
"engine" = "k3s",
"engine" = "k3s"
}
hcloud_token = var.hcloud_token
}
resource "null_resource" "control_planes" {
count = var.servers_num - 1
triggers = {
control_plane_id = module.control_planes[count.index].id
}
connection {
user = "root"
private_key = local.ssh_private_key
agent_identity = local.ssh_identity
host = self.ipv4_address
}
provisioner "file" {
content = templatefile("${path.module}/templates/config.ign.tpl", {
name = self.name
ssh_public_key = local.ssh_public_key
})
destination = "/root/config.ign"
}
# Combustion script file to install k3s-selinux
provisioner "file" {
content = local.combustion_script
destination = "/root/script"
}
# Install MicroOS
provisioner "remote-exec" {
inline = local.microOS_install_commands
}
# Issue a reboot command and wait for the node to reboot
provisioner "local-exec" {
command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3"
}
provisioner "local-exec" {
command = <<-EOT
until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null
do
echo "Waiting for MicroOS to reboot and become available..."
sleep 3
done
EOT
host = module.control_planes[count.index].ipv4_address
}
# Generating k3s server config file
provisioner "file" {
content = yamlencode({
node-name = self.name
node-name = module.control_planes[count.index].name
server = "https://${local.first_control_plane_network_ip}:6443"
token = random_password.k3s_token.result
cluster-init = true
disable-cloud-controller = true
disable = "servicelb, local-storage"
disable = ["servicelb", "local-storage"]
flannel-iface = "eth1"
kubelet-arg = "cloud-provider=external"
node-ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 258 + count.index)
@ -95,13 +76,8 @@ resource "hcloud_server" "control_planes" {
]
}
network {
network_id = hcloud_network.k3s.id
ip = cidrhost(hcloud_network_subnet.k3s.ip_range, 258 + count.index)
}
depends_on = [
hcloud_server.first_control_plane,
null_resource.first_control_plane,
hcloud_network_subnet.k3s
]
}