From 8113016f86f24b8a9ff083cf211305b3467e174a Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Fri, 3 Dec 2021 02:11:52 +0100 Subject: [PATCH] k3os ok --- agents.tf | 60 ++++++++++++++++++++++++++++++++++++++++++++ main.tf | 27 ++++++++++---------- master.tf | 20 +++++++-------- output.tf | 9 +++++++ servers.tf | 59 +++++++++++++++++++++++++++++++++++++++++++ templates/agent.tpl | 36 ++++++++++++++------------ templates/master.tpl | 8 +++--- templates/server.tpl | 41 ++++++++++++++++++++++++++++++ 8 files changed, 217 insertions(+), 43 deletions(-) create mode 100644 agents.tf create mode 100644 output.tf create mode 100644 servers.tf create mode 100644 templates/server.tpl diff --git a/agents.tf b/agents.tf new file mode 100644 index 0000000..33b4a1e --- /dev/null +++ b/agents.tf @@ -0,0 +1,60 @@ +resource "hcloud_server" "agents" { + count = var.agents_num + name = "k3s-agent-${count.index}" + + image = data.hcloud_image.linux.name + rescue = "linux64" + server_type = var.agent_server_type + location = var.location + ssh_keys = [hcloud_ssh_key.default.id] + firewall_ids = [hcloud_firewall.k3s.id] + + + labels = { + "provisioner" = "terraform", + "engine" = "k3s", + "k3s_upgrade" = "true" + } + + provisioner "file" { + content = templatefile("${path.module}/templates/agent.tpl", { + name = self.name + ssh_public_key = local.ssh_public_key + k3s_token = random_password.k3s_token.result + master_ip = local.first_control_plane_network_ip + node_ip = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index) + }) + destination = "/tmp/config.yaml" + + connection { + user = "root" + private_key = file(var.private_key) + host = self.ipv4_address + } + } + + + provisioner "remote-exec" { + inline = local.k3os_install_commands + + connection { + user = "root" + private_key = file(var.private_key) + host = self.ipv4_address + } + } + + provisioner "local-exec" { + command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 60" + } + + network { + network_id = hcloud_network.k3s.id + ip = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index) + } + + depends_on = [ + hcloud_server.first_control_plane, + hcloud_network_subnet.k3s + ] +} diff --git a/main.tf b/main.tf index 722cd0c..b3899fb 100644 --- a/main.tf +++ b/main.tf @@ -88,23 +88,24 @@ resource "hcloud_firewall" "k3s" { } } -data "hcloud_image" "linux" { - name = "ubuntu-20.04" -} locals { first_control_plane_network_ip = cidrhost(hcloud_network.k3s.ip_range, 2) - name_master = "k3s-control-plane-0" ssh_public_key = trimspace(file(var.public_key)) + hcloud_image_name = "ubuntu-20.04" + + k3os_install_commands = [ + "apt install -y grub-efi grub-pc-bin mtools xorriso", + "latest=$(curl -s https://api.github.com/repos/rancher/k3os/releases | jq '.[0].tag_name')", + "curl -Lo ./install.sh https://raw.githubusercontent.com/rancher/k3os/$(echo $latest | xargs)/install.sh", + "chmod +x ./install.sh", + "./install.sh --config /tmp/config.yaml /dev/sda https://github.com/rancher/k3os/releases/download/$(echo $latest | xargs)/k3os-amd64.iso", + "shutdown -r +1", + "sleep 3", + "exit 0" + ] } -data "template_file" "master" { - template = file("${path.module}/templates/master.tpl") - - vars = { - name = local.name_master - ssh_public_key = local.ssh_public_key - k3s_token = random_password.k3s_token.result - ip = local.first_control_plane_network_ip - } +data "hcloud_image" "linux" { + name = local.hcloud_image_name } diff --git a/master.tf b/master.tf index d137431..d6ffade 100644 --- a/master.tf +++ b/master.tf @@ -1,5 +1,5 @@ resource "hcloud_server" "first_control_plane" { - name = local.name_master + name = "k3s-control-plane-0" image = data.hcloud_image.linux.name rescue = "linux64" @@ -14,7 +14,12 @@ resource "hcloud_server" "first_control_plane" { } provisioner "file" { - content = data.template_file.master.rendered + content = templatefile("${path.module}/templates/master.tpl", { + name = self.name + ssh_public_key = local.ssh_public_key + k3s_token = random_password.k3s_token.result + master_ip = local.first_control_plane_network_ip + }) destination = "/tmp/config.yaml" connection { @@ -26,14 +31,7 @@ resource "hcloud_server" "first_control_plane" { provisioner "remote-exec" { - inline = [ - "apt install -y grub-efi grub-pc-bin mtools xorriso", - "latest=$(curl -s https://api.github.com/repos/rancher/k3os/releases | jq '.[0].tag_name')", - "curl -Lo ./install.sh https://raw.githubusercontent.com/rancher/k3os/$(echo $latest | xargs)/install.sh", - "chmod +x ./install.sh", - "./install.sh --config /tmp/config.yaml /dev/sda https://github.com/rancher/k3os/releases/download/$(echo $latest | xargs)/k3os-amd64.iso", - "shutdown -r now" - ] + inline = local.k3os_install_commands connection { user = "root" @@ -44,7 +42,7 @@ resource "hcloud_server" "first_control_plane" { provisioner "local-exec" { command = <<-EOT - ping ${self.ipv4_address} | grep --line-buffered "bytes from" | head -1 && sleep 60 && scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.private_key} rancher@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml + sleep 60 && ping ${self.ipv4_address} | grep --line-buffered "bytes from" | head -1 && sleep 60 && scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.private_key} rancher@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml EOT } diff --git a/output.tf b/output.tf new file mode 100644 index 0000000..928445d --- /dev/null +++ b/output.tf @@ -0,0 +1,9 @@ +output "controlplanes_public_ip" { + value = concat([hcloud_server.first_control_plane.ipv4_address], hcloud_server.control_planes.*.ipv4_address) + description = "The public IP addresses of the controlplane server." +} + +output "agents_public_ip" { + value = hcloud_server.agents.*.ipv4_address + description = "The public IP addresses of the agent server." +} diff --git a/servers.tf b/servers.tf new file mode 100644 index 0000000..4e4c903 --- /dev/null +++ b/servers.tf @@ -0,0 +1,59 @@ +resource "hcloud_server" "control_planes" { + count = var.servers_num - 1 + name = "k3s-control-plane-${count.index + 1}" + + image = data.hcloud_image.linux.name + rescue = "linux64" + server_type = var.control_plane_server_type + location = var.location + ssh_keys = [hcloud_ssh_key.default.id] + firewall_ids = [hcloud_firewall.k3s.id] + + labels = { + "provisioner" = "terraform", + "engine" = "k3s", + "k3s_upgrade" = "true" + } + + provisioner "file" { + content = templatefile("${path.module}/templates/server.tpl", { + name = self.name + ssh_public_key = local.ssh_public_key + k3s_token = random_password.k3s_token.result + master_ip = local.first_control_plane_network_ip + node_ip = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) + }) + destination = "/tmp/config.yaml" + + connection { + user = "root" + private_key = file(var.private_key) + host = self.ipv4_address + } + } + + + provisioner "remote-exec" { + inline = local.k3os_install_commands + + connection { + user = "root" + private_key = file(var.private_key) + host = self.ipv4_address + } + } + + provisioner "local-exec" { + command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 60" + } + + network { + network_id = hcloud_network.k3s.id + ip = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) + } + + depends_on = [ + hcloud_server.first_control_plane, + hcloud_network_subnet.k3s + ] +} diff --git a/templates/agent.tpl b/templates/agent.tpl index 3c52250..c9df5d6 100644 --- a/templates/agent.tpl +++ b/templates/agent.tpl @@ -1,19 +1,27 @@ ssh_authorized_keys: - ${ssh_public_key} hostname: ${name} +boot_cmd: +- | + echo 'auto eth0 + iface eth0 inet dhcp + auto eth1 + iface eth1 inet dhcp' > /etc/network/interfaces +- rc-update del connman boot +- rc-update add networking boot +- rc-update add ntpd default +run_cmd: +- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1" k3os: k3s_args: - - server - --node-ip=${ip} - --advertise-address=${ip} - --bind-address=${ip} - --tls-san=${ip} - --disable-cloud-controller - --disable-network-policy - --disable=traefik - --disable=servicelb - --disable='local-storage' - --kubelet-arg='cloud-provider=external' + - agent + - "--server" + - "https://${master_ip}:6443" + - "--node-ip" + - "${node_ip}" + - "--kubelet-arg" + - "cloud-provider=external" + - "--flannel-iface=eth1" token: ${k3s_token} ntp_servers: - 0.de.pool.ntp.org @@ -21,9 +29,5 @@ k3os: dns_nameservers: - 8.8.8.8 - 1.1.1.1 - - 8.8.4.4 - - 1.0.0.1 - 2001:4860:4860::8888 - - 2606:4700:4700::1111 - - 2001:4860:4860::8844 - - 2606:4700:4700::1001 + - 2606:4700:4700::1111 \ No newline at end of file diff --git a/templates/master.tpl b/templates/master.tpl index 9edb377..05ac341 100644 --- a/templates/master.tpl +++ b/templates/master.tpl @@ -10,6 +10,8 @@ boot_cmd: - rc-update del connman boot - rc-update add networking boot - rc-update add ntpd default +run_cmd: +- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1" k3os: k3s_args: - server @@ -20,11 +22,11 @@ k3os: - "--disable=local-storage" - "--flannel-iface=eth1" - "--node-ip" - - "${ip}" + - "${master_ip}" - "--advertise-address" - - "${ip}" + - "${master_ip}" - "--tls-san" - - "${ip}" + - "${master_ip}" - "--kubelet-arg" - "cloud-provider=external" token: ${k3s_token} diff --git a/templates/server.tpl b/templates/server.tpl new file mode 100644 index 0000000..abd127b --- /dev/null +++ b/templates/server.tpl @@ -0,0 +1,41 @@ +ssh_authorized_keys: +- ${ssh_public_key} +hostname: ${name} +boot_cmd: +- | + echo 'auto eth0 + iface eth0 inet dhcp + auto eth1 + iface eth1 inet dhcp' > /etc/network/interfaces +- rc-update del connman boot +- rc-update add networking boot +- rc-update add ntpd default +run_cmd: +- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1" +k3os: + k3s_args: + - server + - "--server" + - "https://${master_ip}:6443" + - "--disable-cloud-controller" + - "--disable=traefik" + - "--disable=servicelb" + - "--disable=local-storage" + - "--flannel-iface=eth1" + - "--node-ip" + - "${node_ip}" + - "--advertise-address" + - "${node_ip}" + - "--tls-san" + - "${node_ip}" + - "--kubelet-arg" + - "cloud-provider=external" + token: ${k3s_token} + ntp_servers: + - 0.de.pool.ntp.org + - 1.de.pool.ntp.org + dns_nameservers: + - 8.8.8.8 + - 1.1.1.1 + - 2001:4860:4860::8888 + - 2606:4700:4700::1111 \ No newline at end of file