From dbc6dbf0aaf0bc82335d81af1cf383d664e3f303 Mon Sep 17 00:00:00 2001 From: Marco Nenciarini Date: Wed, 23 Feb 2022 19:36:03 +0100 Subject: [PATCH 01/24] Switch to cloud-init for host initialization --- locals.tf | 4 +- modules/host/locals.tf | 55 +------------------- modules/host/main.tf | 64 +++++++++++++++--------- modules/host/templates/boothook.sh.tpl | 5 ++ modules/host/templates/userdata.yaml.tpl | 45 +++++++++++++++++ modules/host/versions.tf | 4 ++ 6 files changed, 98 insertions(+), 79 deletions(-) create mode 100644 modules/host/templates/boothook.sh.tpl create mode 100644 modules/host/templates/userdata.yaml.tpl diff --git a/locals.tf b/locals.tf index 0fe2a8c..d4fec46 100644 --- a/locals.tf +++ b/locals.tf @@ -145,9 +145,9 @@ locals { "[ -e /etc/rancher/k3s/k3s.yaml ] && exit 0", ] - install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"]) + install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"]) - install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"]) + install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"]) agent_nodepools = merge([ for nodepool_name, nodepool_obj in var.agent_nodepools : { diff --git a/modules/host/locals.tf b/modules/host/locals.tf index 08306d6..2da653d 100644 --- a/modules/host/locals.tf +++ b/modules/host/locals.tf @@ -15,60 +15,7 @@ locals { "set -ex", "apt-get update", "apt-get install -y aria2", - "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-kvm-and-xen.qcow2.meta4", + "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2.meta4", "qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda", - "sgdisk -e /dev/sda", - "parted -s /dev/sda resizepart 4 99%", - "parted -s /dev/sda mkpart primary ext2 99% 100%", - "partprobe /dev/sda && udevadm settle && fdisk -l /dev/sda", - "mount /dev/sda4 /mnt/ && btrfs filesystem resize max /mnt && umount /mnt", - "mke2fs -L ignition /dev/sda5", - "mount /dev/sda5 /mnt", - "mkdir /mnt/ignition", - "cp /root/config.ign /mnt/ignition/config.ign", - "mkdir /mnt/combustion", - "cp /root/script /mnt/combustion/script", - "umount /mnt" ] - - ignition_config = jsonencode({ - ignition = { - version = "3.0.0" - } - passwd = { - users = [{ - name = "root" - sshAuthorizedKeys = concat([local.ssh_public_key], var.additional_public_keys) - }] - } - storage = { - files = [ - { - path = "/etc/sysconfig/network/ifcfg-eth1" - mode = 420 - overwrite = true - contents = { "source" = "data:,BOOTPROTO%3D%27dhcp%27%0ASTARTMODE%3D%27auto%27" } - }, - { - path = "/etc/ssh/sshd_config.d/kube-hetzner.conf" - mode = 420 - overwrite = true - contents = { "source" = "data:,PasswordAuthentication%20no%0AX11Forwarding%20no%0AMaxAuthTries%202%0AAllowTcpForwarding%20no%0AAllowAgentForwarding%20no%0AAuthorizedKeysFile%20.ssh%2Fauthorized_keys" } - } - ] - } - }) - - combustion_script = < /etc/transactional-update.conf", - # set the hostname - "hostnamectl set-hostname ${self.name}" - ] - } } resource "hcloud_server_network" "server" { @@ -68,3 +56,33 @@ resource "hcloud_server_network" "server" { server_id = hcloud_server.server.id subnet_id = var.ipv4_subnet_id } + +data "template_cloudinit_config" "config" { + gzip = true + base64_encode = true + + # Main cloud-config configuration file. + part { + filename = "init.cfg" + content_type = "text/cloud-config" + content = templatefile( + "${path.module}/templates/userdata.yaml.tpl", + { + hostname = var.name + sshAuthorizedKeys = concat([local.ssh_public_key], var.additional_public_keys) + } + ) + } + + # Initialization script (runs at every reboot) + part { + content_type = "text/cloud-boothook" + filename = "boothook.sh" + content = templatefile( + "${path.module}/templates/boothook.sh.tpl", + { + hostname = var.name + } + ) + } +} diff --git a/modules/host/templates/boothook.sh.tpl b/modules/host/templates/boothook.sh.tpl new file mode 100644 index 0000000..0b1f461 --- /dev/null +++ b/modules/host/templates/boothook.sh.tpl @@ -0,0 +1,5 @@ +#!/bin/sh +#cloud-boothook + +# Fix hostname after reboot +hostnamectl hostname "${hostname}" diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl new file mode 100644 index 0000000..c685b6e --- /dev/null +++ b/modules/host/templates/userdata.yaml.tpl @@ -0,0 +1,45 @@ +#cloud-config +write_files: + +# Configure private network +- content: | + BOOTPROTO='dhcp' + STARTMODE='auto' + path: /etc/sysconfig/network/ifcfg-eth1 + +# Disable ssh password authentication +- content: | + PasswordAuthentication no + X11Forwarding no + MaxAuthTries 2 + AllowTcpForwarding no + AllowAgentForwarding no + AuthorizedKeysFile .ssh/authorized_keys + path: /etc/ssh/sshd_config.d/kube-hetzner.conf + +# Setting the right reboot mode +- content: | + REBOOT_METHOD=kured + path: /etc/transactional-update.conf + append: true + +# Add ssh authorized keys +ssh_authorized_keys: +%{ for key in sshAuthorizedKeys ~} + - ${key} +%{ endfor ~} + +runcmd: + +# Activate the private network +- systemctl reload network + +# Activate ssh configuration +- systemctl reload sshd + +# Fix hostname (during first boot) +- hostnamectl hostname ${hostname} + +# Finishing automatic reboot via Kured setup +- systemctl reload transactional-update +- rebootmgrctl set-strategy off \ No newline at end of file diff --git a/modules/host/versions.tf b/modules/host/versions.tf index fe79022..7c8da9d 100644 --- a/modules/host/versions.tf +++ b/modules/host/versions.tf @@ -12,5 +12,9 @@ terraform { source = "tenstad/remote" version = "~> 0.0.23" } + template = { + source = "hashicorp/template" + version = "~> 2.2.0" + } } } From a6bd1a6a4895a6d59f301228d737e4896a0bf39c Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Mon, 28 Feb 2022 17:57:28 +0100 Subject: [PATCH 02/24] trying the reboot-sentinel route --- kustomize/kured.yaml | 1 + modules/host/templates/userdata.yaml.tpl | 9 +-------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/kustomize/kured.yaml b/kustomize/kured.yaml index bf72a0c..b4abdea 100644 --- a/kustomize/kured.yaml +++ b/kustomize/kured.yaml @@ -18,3 +18,4 @@ spec: command: - /usr/bin/kured - --reboot-command=/usr/bin/systemctl reboot + - --reboot-sentinel=/var/run/reboot-needed diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index c685b6e..ad7587e 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -17,12 +17,6 @@ write_files: AuthorizedKeysFile .ssh/authorized_keys path: /etc/ssh/sshd_config.d/kube-hetzner.conf -# Setting the right reboot mode -- content: | - REBOOT_METHOD=kured - path: /etc/transactional-update.conf - append: true - # Add ssh authorized keys ssh_authorized_keys: %{ for key in sshAuthorizedKeys ~} @@ -40,6 +34,5 @@ runcmd: # Fix hostname (during first boot) - hostnamectl hostname ${hostname} -# Finishing automatic reboot via Kured setup -- systemctl reload transactional-update +# We are going to let kured do the reboot - rebootmgrctl set-strategy off \ No newline at end of file From ecbd79743e754ac5301c3e37bccb10532fd49f1c Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Mon, 28 Feb 2022 18:45:27 +0100 Subject: [PATCH 03/24] trying the reboot-sentinel route --- modules/host/templates/userdata.yaml.tpl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index ad7587e..37b80ee 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -17,6 +17,11 @@ write_files: AuthorizedKeysFile .ssh/authorized_keys path: /etc/ssh/sshd_config.d/kube-hetzner.conf +# Setting the right reboot mode +- content: | + REBOOT_METHOD=rebootmgr + path: /etc/transactional-update.conf + # Add ssh authorized keys ssh_authorized_keys: %{ for key in sshAuthorizedKeys ~} @@ -34,5 +39,5 @@ runcmd: # Fix hostname (during first boot) - hostnamectl hostname ${hostname} -# We are going to let kured do the reboot +# Finishing automatic reboot via Kured setup - rebootmgrctl set-strategy off \ No newline at end of file From e5404b6adc9b363b78a24bc2340cb66124c4b352 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Tue, 1 Mar 2022 21:57:57 +0100 Subject: [PATCH 04/24] proper application of selinux --- locals.tf | 5 ++-- modules/host/locals.tf | 8 ------- modules/host/main.tf | 30 +++++++++++++++++++++++- modules/host/templates/boothook.sh.tpl | 2 +- modules/host/templates/userdata.yaml.tpl | 17 +++++++------- 5 files changed, 41 insertions(+), 21 deletions(-) diff --git a/locals.tf b/locals.tf index d4fec46..61a8fff 100644 --- a/locals.tf +++ b/locals.tf @@ -145,9 +145,10 @@ locals { "[ -e /etc/rancher/k3s/k3s.yaml ] && exit 0", ] - install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"]) + k3s_selinux_apply = ["chcon -u system_u -r object_r -t container_runtime_exec_t /usr/local/bin/k3s"] - install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"]) + install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"], local.k3s_selinux_apply) + install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"], local.k3s_selinux_apply) agent_nodepools = merge([ for nodepool_name, nodepool_obj in var.agent_nodepools : { diff --git a/modules/host/locals.tf b/modules/host/locals.tf index 2da653d..1fcef4d 100644 --- a/modules/host/locals.tf +++ b/modules/host/locals.tf @@ -10,12 +10,4 @@ locals { ssh_identity_file = var.private_key == null ? var.public_key : var.private_key # shared flags for ssh to ignore host keys, to use our ssh identity file for all connections during provisioning. ssh_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${local.ssh_identity_file}" - - microOS_install_commands = [ - "set -ex", - "apt-get update", - "apt-get install -y aria2", - "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2.meta4", - "qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda", - ] } diff --git a/modules/host/main.tf b/modules/host/main.tf index 99d4069..6f17ec9 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -31,14 +31,42 @@ resource "hcloud_server" "server" { # Install MicroOS provisioner "remote-exec" { - inline = local.microOS_install_commands + inline = [ + "set -ex", + "apt-get update", + "apt-get install -y aria2", + "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2.meta4", + "qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda", + ] } # Issue a reboot command provisioner "local-exec" { command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3" } + # Wait for MicroOS to reboot and be ready + provisioner "local-exec" { + command = <<-EOT + until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null + do + echo "Waiting for MicroOS to reboot and become available..." + sleep 3 + done + EOT + } + # We've rebooted into MicroOS, now we install the k3s-selinux RPM + provisioner "remote-exec" { + inline = [ + "set -ex", + "transactional-update pkg install -y k3s-selinux" + ] + } + + # Issue a reboot command + provisioner "local-exec" { + command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3" + } # Wait for MicroOS to reboot and be ready provisioner "local-exec" { command = <<-EOT diff --git a/modules/host/templates/boothook.sh.tpl b/modules/host/templates/boothook.sh.tpl index 0b1f461..96b4a3c 100644 --- a/modules/host/templates/boothook.sh.tpl +++ b/modules/host/templates/boothook.sh.tpl @@ -2,4 +2,4 @@ #cloud-boothook # Fix hostname after reboot -hostnamectl hostname "${hostname}" +hostnamectl hostname "${hostname}" \ No newline at end of file diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index 37b80ee..1ec848a 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -29,15 +29,14 @@ ssh_authorized_keys: %{ endfor ~} runcmd: + # Activate the private network + - systemctl reload network -# Activate the private network -- systemctl reload network + # Activate ssh configuration + - systemctl reload sshd -# Activate ssh configuration -- systemctl reload sshd + # Fix hostname (during first boot) + - hostnamectl hostname ${hostname} -# Fix hostname (during first boot) -- hostnamectl hostname ${hostname} - -# Finishing automatic reboot via Kured setup -- rebootmgrctl set-strategy off \ No newline at end of file + # Finishing automatic reboot via Kured setup + - rebootmgrctl set-strategy off \ No newline at end of file From c321dbb7ccb5889e19f5c59b8326c77ecb02d67a Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Tue, 1 Mar 2022 22:13:41 +0100 Subject: [PATCH 05/24] added new lines at the end of cloud init templates --- modules/host/templates/boothook.sh.tpl | 2 +- modules/host/templates/userdata.yaml.tpl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/host/templates/boothook.sh.tpl b/modules/host/templates/boothook.sh.tpl index 96b4a3c..0b1f461 100644 --- a/modules/host/templates/boothook.sh.tpl +++ b/modules/host/templates/boothook.sh.tpl @@ -2,4 +2,4 @@ #cloud-boothook # Fix hostname after reboot -hostnamectl hostname "${hostname}" \ No newline at end of file +hostnamectl hostname "${hostname}" diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index 1ec848a..e9d9845 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -39,4 +39,5 @@ runcmd: - hostnamectl hostname ${hostname} # Finishing automatic reboot via Kured setup - - rebootmgrctl set-strategy off \ No newline at end of file + - rebootmgrctl set-strategy off + \ No newline at end of file From e2ca54c6fe78a63cb11def37010615f785a2b288 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 2 Mar 2022 02:06:31 +0100 Subject: [PATCH 06/24] delegated back the k3s selinux RPM to the install script with subsequent reboot --- agents.tf | 13 +++++++++++++ control_planes.tf | 13 +++++++++++++ init.tf | 15 ++++++++++++++- kustomize/kured.yaml | 2 +- locals.tf | 6 ++---- modules/host/main.tf | 32 +++----------------------------- 6 files changed, 46 insertions(+), 35 deletions(-) diff --git a/agents.tf b/agents.tf index da4c4f5..e129650 100644 --- a/agents.tf +++ b/agents.tf @@ -59,6 +59,19 @@ resource "null_resource" "agents" { inline = local.install_k3s_agent } + # Issue a reboot command and wait for MicroOS to reboot and be ready + # so that the new snapshot with k3s-selinux kicks in + provisioner "local-exec" { + command = <<-EOT + ssh ${local.ssh_args} root@${module.agents[each.key].ipv4_address} '(sleep 2; reboot)&'; sleep 3 + until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${module.agents[each.key].ipv4_address} true 2> /dev/null + do + echo "Waiting for MicroOS to reboot and become available..." + sleep 3 + done + EOT + } + # Start the k3s agent and wait for it to have started provisioner "remote-exec" { inline = [ diff --git a/control_planes.tf b/control_planes.tf index 7c31d4b..1ecd24f 100644 --- a/control_planes.tf +++ b/control_planes.tf @@ -65,6 +65,19 @@ resource "null_resource" "control_planes" { inline = local.install_k3s_server } + # Issue a reboot command and wait for MicroOS to reboot and be ready, + # so that the new snapshot with k3s-selinux kicks in, only if k3s has never been initialized on the node + provisioner "local-exec" { + command = <<-EOT + ssh ${local.ssh_args} root@${module.control_planes[count.index].ipv4_address} '[[ ! -f /etc/rancher/k3s/k3s.yaml ]] && (sleep 2; reboot)&'; sleep 3 + until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${module.control_planes[count.index].ipv4_address} true 2> /dev/null + do + echo "Waiting for MicroOS to reboot and become available..." + sleep 3 + done + EOT + } + # Start the k3s server and wait for it to have started correctly provisioner "remote-exec" { inline = [ diff --git a/init.tf b/init.tf index 7f7e021..81eb32b 100644 --- a/init.tf +++ b/init.tf @@ -30,7 +30,20 @@ resource "null_resource" "first_control_plane" { inline = local.install_k3s_server } - # Upon reboot verify that the k3s server is starts, and wait for k3s to be ready to receive commands + # so that the new snapshot with k3s-selinux kicks in + # Issue a reboot command and wait for MicroOS to reboot and be ready + provisioner "local-exec" { + command = <<-EOT + ssh ${local.ssh_args} root@${module.control_planes[0].ipv4_address} '(sleep 2; reboot)&'; sleep 3 + until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${module.control_planes[0].ipv4_address} true 2> /dev/null + do + echo "Waiting for MicroOS to reboot and become available..." + sleep 3 + done + EOT + } + + # Upon reboot verify start k3s and wait for it to be ready to receive commands provisioner "remote-exec" { inline = [ "systemctl start k3s", diff --git a/kustomize/kured.yaml b/kustomize/kured.yaml index b4abdea..3425a49 100644 --- a/kustomize/kured.yaml +++ b/kustomize/kured.yaml @@ -18,4 +18,4 @@ spec: command: - /usr/bin/kured - --reboot-command=/usr/bin/systemctl reboot - - --reboot-sentinel=/var/run/reboot-needed + - --reboot-sentinel=/var/run/reboot-needed \ No newline at end of file diff --git a/locals.tf b/locals.tf index 61a8fff..7acb380 100644 --- a/locals.tf +++ b/locals.tf @@ -145,10 +145,8 @@ locals { "[ -e /etc/rancher/k3s/k3s.yaml ] && exit 0", ] - k3s_selinux_apply = ["chcon -u system_u -r object_r -t container_runtime_exec_t /usr/local/bin/k3s"] - - install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"], local.k3s_selinux_apply) - install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"], local.k3s_selinux_apply) + install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"]) + install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"]) agent_nodepools = merge([ for nodepool_name, nodepool_obj in var.agent_nodepools : { diff --git a/modules/host/main.tf b/modules/host/main.tf index 6f17ec9..b8c4d17 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -35,41 +35,15 @@ resource "hcloud_server" "server" { "set -ex", "apt-get update", "apt-get install -y aria2", - "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2.meta4", + "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-k3s-OpenStack-Cloud.qcow2.meta4", "qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda", ] } - # Issue a reboot command - provisioner "local-exec" { - command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3" - } - # Wait for MicroOS to reboot and be ready - provisioner "local-exec" { - command = <<-EOT - until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null - do - echo "Waiting for MicroOS to reboot and become available..." - sleep 3 - done - EOT - } - - # We've rebooted into MicroOS, now we install the k3s-selinux RPM - provisioner "remote-exec" { - inline = [ - "set -ex", - "transactional-update pkg install -y k3s-selinux" - ] - } - - # Issue a reboot command - provisioner "local-exec" { - command = "ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3" - } - # Wait for MicroOS to reboot and be ready + # Issue a reboot command and wait for MicroOS to reboot and be ready provisioner "local-exec" { command = <<-EOT + ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3 until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null do echo "Waiting for MicroOS to reboot and become available..." From 52adcd3f40df7c8b64cede675526eb4fe682e0cd Mon Sep 17 00:00:00 2001 From: phaer Date: Wed, 2 Mar 2022 14:30:19 +0100 Subject: [PATCH 07/24] resize /var, not /... because cloud-inits growpart can only resize the last partition --- modules/host/templates/userdata.yaml.tpl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index e9d9845..5ac25d9 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -1,4 +1,9 @@ #cloud-config + +# Resize /var, not /, as that's the last partition in MicroOS image. +growpart: + devices: ["/var"] + write_files: # Configure private network @@ -40,4 +45,3 @@ runcmd: # Finishing automatic reboot via Kured setup - rebootmgrctl set-strategy off - \ No newline at end of file From cd0702514737bbb3ffee390b015a408bd3f0ad06 Mon Sep 17 00:00:00 2001 From: Paul Haerle Date: Wed, 2 Mar 2022 20:54:29 +0100 Subject: [PATCH 08/24] add note regarding lifecycle values --- terraform.tfvars.example | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 047c8bc..d424058 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -1,4 +1,10 @@ -# You need to replace these +# Values in this file serve as an example only, +# you are expected to adapt them to your needs. +# +# Some of them, notably "location" and "public_key" have no effect after the initial cluster has been setup. +# This is in order to keep terraform from re-provsioning all nodes at once which would loose data. You should +# instead change the value here and then manually re-provsion each node one-by-one. Grep for "lifecycle". + hcloud_token = "xxxxxxxxxxxxxxxxxxYYYYYYYYYYYYYYYYYYYzzzzzzzzzzzzzzzzzzzzz" public_key = "/home/username/.ssh/id_ed25519.pub" # Must be "private_key = null" when you want to use ssh-agent, for a Yubikey like device auth or an SSH key-pair with passphrase From c5c8004928d7b1efed716d0c84e83b5529426bb1 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 2 Mar 2022 22:19:58 +0100 Subject: [PATCH 09/24] tweaked userdata --- modules/host/main.tf | 12 ----- modules/host/templates/boothook.sh.tpl | 5 -- modules/host/templates/userdata.yaml.tpl | 62 +++++++++++++----------- 3 files changed, 35 insertions(+), 44 deletions(-) delete mode 100644 modules/host/templates/boothook.sh.tpl diff --git a/modules/host/main.tf b/modules/host/main.tf index b8c4d17..869fc8a 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -75,16 +75,4 @@ data "template_cloudinit_config" "config" { } ) } - - # Initialization script (runs at every reboot) - part { - content_type = "text/cloud-boothook" - filename = "boothook.sh" - content = templatefile( - "${path.module}/templates/boothook.sh.tpl", - { - hostname = var.name - } - ) - } } diff --git a/modules/host/templates/boothook.sh.tpl b/modules/host/templates/boothook.sh.tpl deleted file mode 100644 index 0b1f461..0000000 --- a/modules/host/templates/boothook.sh.tpl +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -#cloud-boothook - -# Fix hostname after reboot -hostnamectl hostname "${hostname}" diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index e9d9845..64d91c5 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -1,32 +1,38 @@ #cloud-config + write_files: + # Configure private network + - content: | + BOOTPROTO='dhcp' + STARTMODE='auto' + path: /etc/sysconfig/network/ifcfg-eth1 -# Configure private network -- content: | - BOOTPROTO='dhcp' - STARTMODE='auto' - path: /etc/sysconfig/network/ifcfg-eth1 + # Disable ssh password authentication + - content: | + PasswordAuthentication no + X11Forwarding no + MaxAuthTries 2 + AllowTcpForwarding no + AllowAgentForwarding no + AuthorizedKeysFile .ssh/authorized_keys + path: /etc/ssh/sshd_config.d/kube-hetzner.conf -# Disable ssh password authentication -- content: | - PasswordAuthentication no - X11Forwarding no - MaxAuthTries 2 - AllowTcpForwarding no - AllowAgentForwarding no - AuthorizedKeysFile .ssh/authorized_keys - path: /etc/ssh/sshd_config.d/kube-hetzner.conf + # Setting the right reboot mode + - content: | + REBOOT_METHOD=rebootmgr + path: /etc/transactional-update.conf -# Setting the right reboot mode -- content: | - REBOOT_METHOD=rebootmgr - path: /etc/transactional-update.conf + # Add ssh authorized keys + ssh_authorized_keys: + %{ for key in sshAuthorizedKeys ~} + - ${key} + %{ endfor ~} -# Add ssh authorized keys -ssh_authorized_keys: -%{ for key in sshAuthorizedKeys ~} - - ${key} -%{ endfor ~} +# Making sure the hostname is set correctly +manage_etc_hosts: "localhost" +preserve_hostname: true +prefer_fqdn_over_hostname: false +hostname: ${hostname} runcmd: # Activate the private network @@ -35,9 +41,11 @@ runcmd: # Activate ssh configuration - systemctl reload sshd - # Fix hostname (during first boot) - - hostnamectl hostname ${hostname} - # Finishing automatic reboot via Kured setup - rebootmgrctl set-strategy off - \ No newline at end of file + + # Reduce the default number of snapshots from 2-10 number limit, to 4 + # And from 4-10 number limit important, to 2 + - snapper -c root set-config "NUMBER_LIMIT=4" + - snapper -c root set-config "NUMBER_LIMIT_IMPORTANT=2" + From ddcaf0fb668c0309f0353feb43c7bdac202f930b Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 2 Mar 2022 22:32:34 +0100 Subject: [PATCH 10/24] tweaked tfvars.example --- terraform.tfvars.example | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/terraform.tfvars.example b/terraform.tfvars.example index d424058..8d3ecab 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -1,13 +1,15 @@ -# Values in this file serve as an example only, -# you are expected to adapt them to your needs. +# Only the first values starting with a * are obligatory, the rest can remain with their default values, but Values or you want +# could adapt them to your needs. # -# Some of them, notably "location" and "public_key" have no effect after the initial cluster has been setup. -# This is in order to keep terraform from re-provsioning all nodes at once which would loose data. You should -# instead change the value here and then manually re-provsion each node one-by-one. Grep for "lifecycle". +# Note that some values, notably "location" and "public_key" have no effect after the initial cluster has been setup. +# This is in order to keep terraform from re-provisioning all nodes at once which would loose data. If you want to update, +# those, you should instead change the value here and then manually re-provision each node one-by-one. Grep for "lifecycle". +# * Your Hetzner project API token hcloud_token = "xxxxxxxxxxxxxxxxxxYYYYYYYYYYYYYYYYYYYzzzzzzzzzzzzzzzzzzzzz" +# * Your public key public_key = "/home/username/.ssh/id_ed25519.pub" -# Must be "private_key = null" when you want to use ssh-agent, for a Yubikey like device auth or an SSH key-pair with passphrase +# * Your private key, must be "private_key = null" when you want to use ssh-agent, for a Yubikey like device auth or an SSH key-pair with passphrase private_key = "/home/username/.ssh/id_ed25519" # These can be customized, or left with the default values @@ -16,9 +18,6 @@ private_key = "/home/username/.ssh/id_ed25519" location = "fsn1" # change to `ash` for us-east Ashburn, Virginia location network_region = "eu-central" # change to `us-east` if location is ash -# It's best to leave the network range as is, unless you know what you are doing. The default is "10.0.0.0/8". -# network_ipv4_range = "10.0.0.0/8" - # You can have up to as many subnets as you want (preferably if the form of 10.X.0.0/16), # their primary use is to logically separate the nodes. # The control_plane network is mandatory. @@ -31,7 +30,7 @@ network_ipv4_subnets = { # At least 3 server nodes is recommended for HA, otherwise you need to turn off automatic upgrade (see ReadMe). # As per rancher docs, it must be always an odd number, never even! See https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/ # For instance, 1 is ok (non-HA), 2 not ok, 3 is ok (becomes HA). -control_plane_count = 3 +control_plane_count = 3 # The type of control plane nodes, see https://www.hetzner.com/cloud, the minimum instance supported is cpx11 (just a few cents more than cx11) control_plane_server_type = "cpx11" @@ -54,6 +53,11 @@ agent_nodepools = { # That will depend on how much load you want it to handle, see https://www.hetzner.com/cloud/load-balancer load_balancer_type = "lb11" +### The following values are fully optional + +# It's best to leave the network range as is, unless you know what you are doing. The default is "10.0.0.0/8". +# network_ipv4_range = "10.0.0.0/8" + # If you want to use a specific Hetzner CCM and CSI version, set them below, otherwise leave as is for the latest versions # hetzner_ccm_version = "" # hetzner_csi_version = "" From 117daeb9c34a7533bf3fc55f98863cd8319477c9 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Thu, 3 Mar 2022 01:28:51 +0100 Subject: [PATCH 11/24] tweaked userdata and tfvars.example --- modules/host/templates/userdata.yaml.tpl | 94 +++++++++++++----------- terraform.tfvars.example | 2 +- 2 files changed, 51 insertions(+), 45 deletions(-) diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index 67349d2..4878c6b 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -1,57 +1,63 @@ #cloud-config +write_files: + +# Configure the private network interface +- content: | + BOOTPROTO='dhcp' + STARTMODE='auto' + path: /etc/sysconfig/network/ifcfg-eth1 + +# Disable ssh password authentication +- content: | + PasswordAuthentication no + X11Forwarding no + MaxAuthTries 2 + AllowTcpForwarding no + AllowAgentForwarding no + AuthorizedKeysFile .ssh/authorized_keys + path: /etc/ssh/sshd_config.d/kube-hetzner.conf + +# Setting the right reboot mode +- content: | + REBOOT_METHOD=rebootmgr + path: /etc/transactional-update.conf + +# Add ssh authorized keys +ssh_authorized_keys: +%{ for key in sshAuthorizedKeys ~} + - ${key} +%{ endfor ~} + # Resize /var, not /, as that's the last partition in MicroOS image. growpart: devices: ["/var"] -write_files: - # Configure private network - - content: | - BOOTPROTO='dhcp' - STARTMODE='auto' - path: /etc/sysconfig/network/ifcfg-eth1 - - # Disable ssh password authentication - - content: | - PasswordAuthentication no - X11Forwarding no - MaxAuthTries 2 - AllowTcpForwarding no - AllowAgentForwarding no - AuthorizedKeysFile .ssh/authorized_keys - path: /etc/ssh/sshd_config.d/kube-hetzner.conf - - # Setting the right reboot mode - - content: | - REBOOT_METHOD=rebootmgr - path: /etc/transactional-update.conf - - # Add ssh authorized keys - ssh_authorized_keys: - %{ for key in sshAuthorizedKeys ~} - - ${key} - %{ endfor ~} - # Make sure the hostname is set correctly -manage_etc_hosts: "localhost" -preserve_hostname: true -prefer_fqdn_over_hostname: false hostname: ${hostname} +preserve_hostname: true +manage_etc_hosts: "localhost" runcmd: - # As above, make sure the hostname is not reset - - sed -i 's#NETCONFIG_NIS_SETDOMAINNAME="yes"#NETCONFIG_NIS_SETDOMAINNAME="no"#g' /etc/sysconfig/network/config - - # Activate the private network - - systemctl reload network - # Activate ssh configuration - - systemctl reload sshd +# As above, make sure the hostname is not reset +- [ sed, -i, 's#preserve_hostname: false#preserve_hostname: true#g', /etc/cloud/cloud.cfg] +- [ sed, -i, 's#NETCONFIG_NIS_SETDOMAINNAME="yes"#NETCONFIG_NIS_SETDOMAINNAME="no"#g', /etc/sysconfig/network/config] +- [ sed, -i, 's#DHCLIENT_SET_HOSTNAME="yes"#DHCLIENT_SET_HOSTNAME="no"#g', /etc/sysconfig/network/dhcp] - # Finishing automatic reboot via Kured setup - - rebootmgrctl set-strategy off +# We set Google DNS servers +- [ sed, -i, 's#NETCONFIG_DNS_STATIC_SERVERS=""#NETCONFIG_DNS_STATIC_SERVERS="8.8.8.8 8.8.4.4"#g', /etc/sysconfig/network/config] - # Reduce the default number of snapshots from 2-10 number limit, to 4 - # And from 4-10 number limit important, to 2 - - snapper -c root set-config "NUMBER_LIMIT=4" - - snapper -c root set-config "NUMBER_LIMIT_IMPORTANT=2" +# Activate the private network +- systemctl reload network + +# Activate ssh configuration +- systemctl reload sshd + +# Finishing automatic reboot via Kured setup +- rebootmgrctl set-strategy off + +# Reduce the default number of snapshots from 2-10 number limit, to 4 +# And from 4-10 number limit important, to 2 +- snapper -c root set-config "NUMBER_LIMIT=4" +- snapper -c root set-config "NUMBER_LIMIT_IMPORTANT=2" diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 8d3ecab..2710ac1 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -1,4 +1,4 @@ -# Only the first values starting with a * are obligatory, the rest can remain with their default values, but Values or you want +# Only the first values starting with a * are obligatory, the rest can remain with their default values, or you # could adapt them to your needs. # # Note that some values, notably "location" and "public_key" have no effect after the initial cluster has been setup. From a0d3cb2ffdd9ff2ec8c3c6f48d6207e7abe2d91f Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Thu, 3 Mar 2022 01:56:04 +0100 Subject: [PATCH 12/24] fixed k3s selinux --- agents.tf | 13 ------------- control_planes.tf | 13 ------------- init.tf | 13 ------------- locals.tf | 6 ++++-- modules/host/main.tf | 20 ++++++++++++++++++++ 5 files changed, 24 insertions(+), 41 deletions(-) diff --git a/agents.tf b/agents.tf index e129650..da4c4f5 100644 --- a/agents.tf +++ b/agents.tf @@ -59,19 +59,6 @@ resource "null_resource" "agents" { inline = local.install_k3s_agent } - # Issue a reboot command and wait for MicroOS to reboot and be ready - # so that the new snapshot with k3s-selinux kicks in - provisioner "local-exec" { - command = <<-EOT - ssh ${local.ssh_args} root@${module.agents[each.key].ipv4_address} '(sleep 2; reboot)&'; sleep 3 - until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${module.agents[each.key].ipv4_address} true 2> /dev/null - do - echo "Waiting for MicroOS to reboot and become available..." - sleep 3 - done - EOT - } - # Start the k3s agent and wait for it to have started provisioner "remote-exec" { inline = [ diff --git a/control_planes.tf b/control_planes.tf index 1ecd24f..7c31d4b 100644 --- a/control_planes.tf +++ b/control_planes.tf @@ -65,19 +65,6 @@ resource "null_resource" "control_planes" { inline = local.install_k3s_server } - # Issue a reboot command and wait for MicroOS to reboot and be ready, - # so that the new snapshot with k3s-selinux kicks in, only if k3s has never been initialized on the node - provisioner "local-exec" { - command = <<-EOT - ssh ${local.ssh_args} root@${module.control_planes[count.index].ipv4_address} '[[ ! -f /etc/rancher/k3s/k3s.yaml ]] && (sleep 2; reboot)&'; sleep 3 - until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${module.control_planes[count.index].ipv4_address} true 2> /dev/null - do - echo "Waiting for MicroOS to reboot and become available..." - sleep 3 - done - EOT - } - # Start the k3s server and wait for it to have started correctly provisioner "remote-exec" { inline = [ diff --git a/init.tf b/init.tf index 81eb32b..e15b054 100644 --- a/init.tf +++ b/init.tf @@ -30,19 +30,6 @@ resource "null_resource" "first_control_plane" { inline = local.install_k3s_server } - # so that the new snapshot with k3s-selinux kicks in - # Issue a reboot command and wait for MicroOS to reboot and be ready - provisioner "local-exec" { - command = <<-EOT - ssh ${local.ssh_args} root@${module.control_planes[0].ipv4_address} '(sleep 2; reboot)&'; sleep 3 - until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${module.control_planes[0].ipv4_address} true 2> /dev/null - do - echo "Waiting for MicroOS to reboot and become available..." - sleep 3 - done - EOT - } - # Upon reboot verify start k3s and wait for it to be ready to receive commands provisioner "remote-exec" { inline = [ diff --git a/locals.tf b/locals.tf index 7acb380..6ebbe16 100644 --- a/locals.tf +++ b/locals.tf @@ -145,8 +145,10 @@ locals { "[ -e /etc/rancher/k3s/k3s.yaml ] && exit 0", ] - install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"]) - install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"]) + apply_k3s_selinux = ["/sbin/semodule -v -i /usr/share/selinux/packages/k3s.pp"] + + install_k3s_server = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=server sh -"], local.apply_k3s_selinux) + install_k3s_agent = concat(local.common_commands_install_k3s, ["curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true INSTALL_K3S_SKIP_SELINUX_RPM=true INSTALL_K3S_CHANNEL=${var.initial_k3s_channel} INSTALL_K3S_EXEC=agent sh -"], local.apply_k3s_selinux) agent_nodepools = merge([ for nodepool_name, nodepool_obj in var.agent_nodepools : { diff --git a/modules/host/main.tf b/modules/host/main.tf index 869fc8a..8eae7d6 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -51,6 +51,26 @@ resource "hcloud_server" "server" { done EOT } + + # Install k3s-selinux (compatible version) + provisioner "remote-exec" { + inline = [ + "set -ex", + "transactional-update pkg install -y k3s-selinux" + ] + } + + # Issue a reboot command and wait for MicroOS to reboot and be ready + provisioner "local-exec" { + command = <<-EOT + ssh ${local.ssh_args} root@${self.ipv4_address} '(sleep 2; reboot)&'; sleep 3 + until ssh ${local.ssh_args} -o ConnectTimeout=2 root@${self.ipv4_address} true 2> /dev/null + do + echo "Waiting for MicroOS to reboot and become available..." + sleep 3 + done + EOT + } } resource "hcloud_server_network" "server" { From 7bdd555e1a84e96fbd235f184d6fa24857cdbfa9 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Thu, 3 Mar 2022 02:06:29 +0100 Subject: [PATCH 13/24] small tweaks --- init.tf | 2 +- kustomize/kured.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/init.tf b/init.tf index e15b054..ee9134b 100644 --- a/init.tf +++ b/init.tf @@ -30,7 +30,7 @@ resource "null_resource" "first_control_plane" { inline = local.install_k3s_server } - # Upon reboot verify start k3s and wait for it to be ready to receive commands + # Upon reboot start k3s and wait for it to be ready to receive commands provisioner "remote-exec" { inline = [ "systemctl start k3s", diff --git a/kustomize/kured.yaml b/kustomize/kured.yaml index 3425a49..b4abdea 100644 --- a/kustomize/kured.yaml +++ b/kustomize/kured.yaml @@ -18,4 +18,4 @@ spec: command: - /usr/bin/kured - --reboot-command=/usr/bin/systemctl reboot - - --reboot-sentinel=/var/run/reboot-needed \ No newline at end of file + - --reboot-sentinel=/var/run/reboot-needed From f6fe3c24840e892ef451f164065f93f4b81328e4 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Thu, 3 Mar 2022 02:47:14 +0100 Subject: [PATCH 14/24] returned to the normal openstack image --- modules/host/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/host/main.tf b/modules/host/main.tf index 8eae7d6..e085883 100644 --- a/modules/host/main.tf +++ b/modules/host/main.tf @@ -35,7 +35,7 @@ resource "hcloud_server" "server" { "set -ex", "apt-get update", "apt-get install -y aria2", - "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-k3s-OpenStack-Cloud.qcow2.meta4", + "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2.meta4", "qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*qcow2$') /dev/sda", ] } From 52bb5f663f8c4c71595f14ca94450e38831be725 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Thu, 3 Mar 2022 03:02:24 +0100 Subject: [PATCH 15/24] removed explicit reboot method flag --- modules/host/templates/userdata.yaml.tpl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index 4878c6b..b704130 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -18,11 +18,6 @@ write_files: AuthorizedKeysFile .ssh/authorized_keys path: /etc/ssh/sshd_config.d/kube-hetzner.conf -# Setting the right reboot mode -- content: | - REBOOT_METHOD=rebootmgr - path: /etc/transactional-update.conf - # Add ssh authorized keys ssh_authorized_keys: %{ for key in sshAuthorizedKeys ~} From c92868c054dc03fd1f3a4e7c9f6bc2687ae2c142 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Thu, 3 Mar 2022 19:08:12 +0100 Subject: [PATCH 16/24] fixed TLS handshake error on agents --- .gitignore | 1 + agents.tf | 4 ++-- control_planes.tf | 1 - init.tf | 1 - locals.tf | 2 -- 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 16bc502..458f192 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ kubeconfig.yaml-e terraform.tfvars plans.yaml traefik_config.yaml +kured.yaml diff --git a/agents.tf b/agents.tf index da4c4f5..10912ec 100644 --- a/agents.tf +++ b/agents.tf @@ -44,11 +44,11 @@ resource "null_resource" "agents" { provisioner "file" { content = yamlencode({ node-name = module.agents[each.key].name - server = "https://${local.first_control_plane_network_ipv4}:6443" + server = "https://${module.control_planes[0].private_ipv4_address}:6443" token = random_password.k3s_token.result kubelet-arg = "cloud-provider=external" flannel-iface = "eth1" - node-ip = module.agents[each.key].ipv4_address + node-ip = module.agents[each.key].private_ipv4_address node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] }) destination = "/tmp/config.yaml" diff --git a/control_planes.tf b/control_planes.tf index 7c31d4b..34cf4bf 100644 --- a/control_planes.tf +++ b/control_planes.tf @@ -53,7 +53,6 @@ resource "null_resource" "control_planes" { kubelet-arg = "cloud-provider=external" node-ip = module.control_planes[count.index].private_ipv4_address advertise-address = module.control_planes[count.index].private_ipv4_address - tls-san = module.control_planes[count.index].private_ipv4_address node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"] node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] }) diff --git a/init.tf b/init.tf index ee9134b..87e624e 100644 --- a/init.tf +++ b/init.tf @@ -18,7 +18,6 @@ resource "null_resource" "first_control_plane" { kubelet-arg = "cloud-provider=external" node-ip = module.control_planes[0].private_ipv4_address advertise-address = module.control_planes[0].private_ipv4_address - tls-san = module.control_planes[0].private_ipv4_address node-taint = var.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/master:NoSchedule"] node-label = var.automatically_upgrade_k3s ? ["k3s_upgrade=true"] : [] }) diff --git a/locals.tf b/locals.tf index 6ebbe16..1e77fa2 100644 --- a/locals.tf +++ b/locals.tf @@ -1,6 +1,4 @@ locals { - first_control_plane_network_ipv4 = module.control_planes[0].private_ipv4_address - ssh_public_key = trimspace(file(var.public_key)) # ssh_private_key is either the contents of var.private_key or null to use a ssh agent. ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key)) From 5a3b8afd287fd35a83e4760db09d9929ce97c00d Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Thu, 3 Mar 2022 19:25:58 +0100 Subject: [PATCH 17/24] bounded the logs size and age --- modules/host/templates/userdata.yaml.tpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index b704130..6720ca5 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -43,6 +43,10 @@ runcmd: # We set Google DNS servers - [ sed, -i, 's#NETCONFIG_DNS_STATIC_SERVERS=""#NETCONFIG_DNS_STATIC_SERVERS="8.8.8.8 8.8.4.4"#g', /etc/sysconfig/network/config] +# Bound the amount of logs that can survive on the system +- [ sed, -i, 's/#SystemMaxUse=/SystemMaxUse=3G/g', /etc/systemd/journald.conf] +- [ sed, -i, 's/#MaxRetentionSec=/MaxRetentionSec=1week/g', /etc/systemd/journald.conf] + # Activate the private network - systemctl reload network From ae3313c6ffddfc5d0351ab0cabd38721ddc6f65a Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Fri, 4 Mar 2022 13:40:10 +0100 Subject: [PATCH 18/24] fixed gitignore --- .gitignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 458f192..6c2542f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,6 @@ crash.log kubeconfig.yaml kubeconfig.yaml-e terraform.tfvars -plans.yaml -traefik_config.yaml -kured.yaml +plans-custom.yaml +traefik-custom.yaml +kured-custom.yaml From d54a50d54446df0a9caed3e24d571e376bcf7563 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Fri, 4 Mar 2022 13:46:01 +0100 Subject: [PATCH 19/24] added cloudflare dns first --- modules/host/templates/userdata.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index 6720ca5..254abc3 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -41,7 +41,7 @@ runcmd: - [ sed, -i, 's#DHCLIENT_SET_HOSTNAME="yes"#DHCLIENT_SET_HOSTNAME="no"#g', /etc/sysconfig/network/dhcp] # We set Google DNS servers -- [ sed, -i, 's#NETCONFIG_DNS_STATIC_SERVERS=""#NETCONFIG_DNS_STATIC_SERVERS="8.8.8.8 8.8.4.4"#g', /etc/sysconfig/network/config] +- [ sed, -i, 's#NETCONFIG_DNS_STATIC_SERVERS=""#NETCONFIG_DNS_STATIC_SERVERS="1.1.1.1 1.0.0.1 8.8.8.8 8.8.4.4"#g', /etc/sysconfig/network/config] # Bound the amount of logs that can survive on the system - [ sed, -i, 's/#SystemMaxUse=/SystemMaxUse=3G/g', /etc/systemd/journald.conf] From c14d85300e49db6e3bd79aee7c325643b5d0f857 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Fri, 4 Mar 2022 13:46:52 +0100 Subject: [PATCH 20/24] switch back to the standard kured reboot method --- kustomize/kured.yaml | 1 - modules/host/templates/userdata.yaml.tpl | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/kured.yaml b/kustomize/kured.yaml index b4abdea..bf72a0c 100644 --- a/kustomize/kured.yaml +++ b/kustomize/kured.yaml @@ -18,4 +18,3 @@ spec: command: - /usr/bin/kured - --reboot-command=/usr/bin/systemctl reboot - - --reboot-sentinel=/var/run/reboot-needed diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index 254abc3..39bf5f3 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -54,6 +54,7 @@ runcmd: - systemctl reload sshd # Finishing automatic reboot via Kured setup +- echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf - rebootmgrctl set-strategy off # Reduce the default number of snapshots from 2-10 number limit, to 4 From 1cddf1288651fc5ebaaa30b876f403af5f680ade Mon Sep 17 00:00:00 2001 From: Marco Nenciarini Date: Fri, 4 Mar 2022 13:53:45 +0100 Subject: [PATCH 21/24] Return to default kured reboot method --- .gitignore | 1 - kustomize/kured.yaml | 1 - modules/host/templates/userdata.yaml.tpl | 6 ++++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 458f192..16bc502 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,3 @@ kubeconfig.yaml-e terraform.tfvars plans.yaml traefik_config.yaml -kured.yaml diff --git a/kustomize/kured.yaml b/kustomize/kured.yaml index b4abdea..bf72a0c 100644 --- a/kustomize/kured.yaml +++ b/kustomize/kured.yaml @@ -18,4 +18,3 @@ spec: command: - /usr/bin/kured - --reboot-command=/usr/bin/systemctl reboot - - --reboot-sentinel=/var/run/reboot-needed diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index 6720ca5..4b20b5a 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -18,6 +18,12 @@ write_files: AuthorizedKeysFile .ssh/authorized_keys path: /etc/ssh/sshd_config.d/kube-hetzner.conf +# Setting the right reboot mode +- content: | + REBOOT_METHOD=kured + path: /etc/transactional-update.conf + append: true + # Add ssh authorized keys ssh_authorized_keys: %{ for key in sshAuthorizedKeys ~} From 32d43b50f40b30cba207fb5946dee422825d93ef Mon Sep 17 00:00:00 2001 From: Marco Nenciarini Date: Fri, 4 Mar 2022 15:11:07 +0100 Subject: [PATCH 22/24] Use Cloudflare DNS --- modules/host/templates/userdata.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index 4b20b5a..a4b06ea 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -47,7 +47,7 @@ runcmd: - [ sed, -i, 's#DHCLIENT_SET_HOSTNAME="yes"#DHCLIENT_SET_HOSTNAME="no"#g', /etc/sysconfig/network/dhcp] # We set Google DNS servers -- [ sed, -i, 's#NETCONFIG_DNS_STATIC_SERVERS=""#NETCONFIG_DNS_STATIC_SERVERS="8.8.8.8 8.8.4.4"#g', /etc/sysconfig/network/config] +- [ sed, -i, 's#NETCONFIG_DNS_STATIC_SERVERS=""#NETCONFIG_DNS_STATIC_SERVERS="1.1.1.1 1.0.0.1"#g', /etc/sysconfig/network/config] # Bound the amount of logs that can survive on the system - [ sed, -i, 's/#SystemMaxUse=/SystemMaxUse=3G/g', /etc/systemd/journald.conf] From fecd312cdd41d159997db9d88db0fee30d0a567b Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Fri, 4 Mar 2022 15:46:15 +0100 Subject: [PATCH 23/24] userdata tweaks --- modules/host/templates/userdata.yaml.tpl | 39 +++++++++++------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index 39bf5f3..7b027dd 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -18,6 +18,11 @@ write_files: AuthorizedKeysFile .ssh/authorized_keys path: /etc/ssh/sshd_config.d/kube-hetzner.conf +# Set reboot method as "kured" +- content: | + REBOOT_METHOD=kured + path: /etc/transactional-update.conf + # Add ssh authorized keys ssh_authorized_keys: %{ for key in sshAuthorizedKeys ~} @@ -31,33 +36,23 @@ growpart: # Make sure the hostname is set correctly hostname: ${hostname} preserve_hostname: true -manage_etc_hosts: "localhost" runcmd: # As above, make sure the hostname is not reset -- [ sed, -i, 's#preserve_hostname: false#preserve_hostname: true#g', /etc/cloud/cloud.cfg] -- [ sed, -i, 's#NETCONFIG_NIS_SETDOMAINNAME="yes"#NETCONFIG_NIS_SETDOMAINNAME="no"#g', /etc/sysconfig/network/config] -- [ sed, -i, 's#DHCLIENT_SET_HOSTNAME="yes"#DHCLIENT_SET_HOSTNAME="no"#g', /etc/sysconfig/network/dhcp] +- [sed, -i, 's#NETCONFIG_NIS_SETDOMAINNAME="yes"#NETCONFIG_NIS_SETDOMAINNAME="no"#g', /etc/sysconfig/network/config] +- [sed, -i, 's#DHCLIENT_SET_HOSTNAME="yes"#DHCLIENT_SET_HOSTNAME="no"#g', /etc/sysconfig/network/dhcp] -# We set Google DNS servers -- [ sed, -i, 's#NETCONFIG_DNS_STATIC_SERVERS=""#NETCONFIG_DNS_STATIC_SERVERS="1.1.1.1 1.0.0.1 8.8.8.8 8.8.4.4"#g', /etc/sysconfig/network/config] +# We set Cloudflare DNS servers, followed by Google as a backup +- [sed, -i, 's#NETCONFIG_DNS_STATIC_SERVERS=""#NETCONFIG_DNS_STATIC_SERVERS="1.1.1.1 1.0.0.1 8.8.8.8"#g', /etc/sysconfig/network/config] -# Bound the amount of logs that can survive on the system -- [ sed, -i, 's/#SystemMaxUse=/SystemMaxUse=3G/g', /etc/systemd/journald.conf] -- [ sed, -i, 's/#MaxRetentionSec=/MaxRetentionSec=1week/g', /etc/systemd/journald.conf] +# Bounds the amount of logs that can survive on the system +- [sed, -i, 's/#SystemMaxUse=/SystemMaxUse=3G/g', /etc/systemd/journald.conf] +- [sed, -i, 's/#MaxRetentionSec=/MaxRetentionSec=1week/g', /etc/systemd/journald.conf] -# Activate the private network -- systemctl reload network +# Disables unneeded services +- [systemctl, disable, '--now', 'rebootmgr.service'] -# Activate ssh configuration -- systemctl reload sshd - -# Finishing automatic reboot via Kured setup -- echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf -- rebootmgrctl set-strategy off - -# Reduce the default number of snapshots from 2-10 number limit, to 4 -# And from 4-10 number limit important, to 2 -- snapper -c root set-config "NUMBER_LIMIT=4" -- snapper -c root set-config "NUMBER_LIMIT_IMPORTANT=2" +# Reduces the default number of snapshots from 2-10 number limit, to 4 and from 4-10 number limit important, to 2 +- [snapper, -c, root, 'set-config', 'NUMBER_LIMIT=4'] +- [snapper, -c, root, 'set-config', 'NUMBER_LIMIT_IMPORTANT=2'] From 9f7c1fc8113a8281ff7c8357eeca7476943dc137 Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Fri, 4 Mar 2022 16:08:05 +0100 Subject: [PATCH 24/24] fixed snapper config not holding --- modules/host/templates/userdata.yaml.tpl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/host/templates/userdata.yaml.tpl b/modules/host/templates/userdata.yaml.tpl index 7b027dd..51ff107 100644 --- a/modules/host/templates/userdata.yaml.tpl +++ b/modules/host/templates/userdata.yaml.tpl @@ -40,19 +40,19 @@ preserve_hostname: true runcmd: # As above, make sure the hostname is not reset -- [sed, -i, 's#NETCONFIG_NIS_SETDOMAINNAME="yes"#NETCONFIG_NIS_SETDOMAINNAME="no"#g', /etc/sysconfig/network/config] -- [sed, -i, 's#DHCLIENT_SET_HOSTNAME="yes"#DHCLIENT_SET_HOSTNAME="no"#g', /etc/sysconfig/network/dhcp] +- [sed, '-i', 's/NETCONFIG_NIS_SETDOMAINNAME="yes"/NETCONFIG_NIS_SETDOMAINNAME="no"/g', /etc/sysconfig/network/config] +- [sed, '-i', 's/DHCLIENT_SET_HOSTNAME="yes"/DHCLIENT_SET_HOSTNAME="no"/g', /etc/sysconfig/network/dhcp] # We set Cloudflare DNS servers, followed by Google as a backup -- [sed, -i, 's#NETCONFIG_DNS_STATIC_SERVERS=""#NETCONFIG_DNS_STATIC_SERVERS="1.1.1.1 1.0.0.1 8.8.8.8"#g', /etc/sysconfig/network/config] +- [sed, '-i', 's/NETCONFIG_DNS_STATIC_SERVERS=""/NETCONFIG_DNS_STATIC_SERVERS="1.1.1.1 1.0.0.1 8.8.8.8"/g', /etc/sysconfig/network/config] # Bounds the amount of logs that can survive on the system -- [sed, -i, 's/#SystemMaxUse=/SystemMaxUse=3G/g', /etc/systemd/journald.conf] -- [sed, -i, 's/#MaxRetentionSec=/MaxRetentionSec=1week/g', /etc/systemd/journald.conf] +- [sed, '-i', 's/#SystemMaxUse=/SystemMaxUse=3G/g', /etc/systemd/journald.conf] +- [sed, '-i', 's/#MaxRetentionSec=/MaxRetentionSec=1week/g', /etc/systemd/journald.conf] + +# Reduces the default number of snapshots from 2-10 number limit, to 4 and from 4-10 number limit important, to 2 +- [sed, '-i', 's/NUMBER_LIMIT="2-10"/NUMBER_LIMIT="4"/g', /etc/snapper/configs/root] +- [sed, '-i', 's/NUMBER_LIMIT_IMPORTANT="4-10"/NUMBER_LIMIT_IMPORTANT="3"/g', /etc/snapper/configs/root] # Disables unneeded services - [systemctl, disable, '--now', 'rebootmgr.service'] - -# Reduces the default number of snapshots from 2-10 number limit, to 4 and from 4-10 number limit important, to 2 -- [snapper, -c, root, 'set-config', 'NUMBER_LIMIT=4'] -- [snapper, -c, root, 'set-config', 'NUMBER_LIMIT_IMPORTANT=2']