diff --git a/agents.tf b/agents.tf new file mode 100644 index 0000000..ab856fd --- /dev/null +++ b/agents.tf @@ -0,0 +1,110 @@ +resource "hcloud_server" "agents" { + count = var.agents_num + name = "k3s-agent-${count.index}" + + image = data.hcloud_image.linux.name + rescue = "linux64" + server_type = var.agent_server_type + location = var.location + ssh_keys = [hcloud_ssh_key.k3s.id] + firewall_ids = [hcloud_firewall.k3s.id] + placement_group_id = hcloud_placement_group.k3s_placement_group.id + + + labels = { + "provisioner" = "terraform", + "engine" = "k3s", + } + + provisioner "file" { + content = templatefile("${path.module}/templates/config.ign.tpl", { + name = self.name + ssh_public_key = local.ssh_public_key + }) + destination = "/root/config.ign" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Install MicroOS + provisioner "remote-exec" { + inline = local.MicroOS_install_commands + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Wait for MicroOS to reboot and be ready + provisioner "local-exec" { + command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 30" + } + + # Generating and uploading the angent.conf file + provisioner "file" { + content = templatefile("${path.module}/templates/agent.conf.tpl", { + server_url = "https://${local.first_control_plane_network_ip}:6443" + node_token = random_password.k3s_token.result + }) + destination = "/etc/rancher/k3s/agent.conf" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Generating k3s server config file + provisioner "file" { + content = templatefile("${path.module}/templates/agent_config.yaml.tpl", { + node_ip = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index) + node_name = self.name + }) + destination = "/etc/rancher/k3s/config.yaml" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Run the agent + provisioner "remote-exec" { + inline = [ + "set -ex", + # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured + "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf", + # then turn on k3s and join the cluster + "systemctl --now enable k3s-agent", + ] + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + network { + network_id = hcloud_network.k3s.id + ip = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index) + } + + depends_on = [ + hcloud_server.first_control_plane, + hcloud_network_subnet.k3s + ] +} diff --git a/locals.tf b/locals.tf index a5cab15..6d6427c 100644 --- a/locals.tf +++ b/locals.tf @@ -11,10 +11,10 @@ locals { # if an ssh agent is used. ssh_identity_file = var.private_key == null ? var.public_key : var.private_key - microOS_install_commands = [ + MicroOS_install_commands = [ "set -ex", "apt-get install -y aria2", - "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-k3s-kvm-and-xen.qcow2.meta4", + "aria2c --follow-metalink=mem https://download.opensuse.org/tumbleweed/appliances/openSUSE-MicroOS.x86_64-k3s-kvm-and-xen.qcow2.meta4", "qemu-img convert -p -f qcow2 -O host_device $(ls -a | grep -ie '^opensuse.*microos.*k3s.*qcow2$') /dev/sda", "sgdisk -e /dev/sda", "partprobe /dev/sda", diff --git a/master.tf b/master.tf index e2f7a76..64c4456 100644 --- a/master.tf +++ b/master.tf @@ -14,23 +14,6 @@ resource "hcloud_server" "first_control_plane" { "engine" = "k3s" } - provisioner "file" { - content = templatefile("${path.module}/templates/master.tpl", { - name = self.name - ssh_public_key = local.ssh_public_key - k3s_token = random_password.k3s_token.result - master_ip = local.first_control_plane_network_ip - }) - destination = "/tmp/config.yaml" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } - } - provisioner "file" { content = templatefile("${path.module}/templates/config.ign.tpl", { name = self.name @@ -46,9 +29,9 @@ resource "hcloud_server" "first_control_plane" { } } - # Install k3os + # Install MicroOS provisioner "remote-exec" { - inline = local.microOS_install_commands + inline = local.MicroOS_install_commands connection { user = "root" @@ -57,18 +40,61 @@ resource "hcloud_server" "first_control_plane" { host = self.ipv4_address } } - /* - # Wait for MicroOS to be ready and fetch kubeconfig.yaml + + # Wait for MicroOS to reboot and be ready + provisioner "local-exec" { + command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 30" + } + + # Generating k3s master config file + provisioner "file" { + content = templatefile("${path.module}/templates/master_config.yaml.tpl", { + node_ip = local.first_control_plane_network_ip + token = random_password.k3s_token.result + node_name = self.name + }) + destination = "/etc/rancher/k3s/config.yaml" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + # Run the first control plane + provisioner "remote-exec" { + inline = [ + "set -ex", + # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured + "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf", + # then we initiate the cluster + "systemctl --now enable k3s-server", + ] + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Get the Kubeconfig, and wait for the node to be available provisioner "local-exec" { command = <<-EOT - sleep 60 && ping ${self.ipv4_address} | grep --line-buffered "bytes from" | head -1 && sleep 100 && scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${local.ssh_identity_file} rancher@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml + set -ex + sleep 30 + scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${local.ssh_identity_file} root@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml + sleep 30 EOT } - # Install Hetzner CCM and CSI + # Install the Hetzner CCM and CSI provisioner "local-exec" { command = <<-EOT + set -ex kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --kubeconfig ${path.module}/kubeconfig.yaml kubectl apply -k ${dirname(local_file.hetzner_ccm_config.filename)} --kubeconfig ${path.module}/kubeconfig.yaml kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --kubeconfig ${path.module}/kubeconfig.yaml @@ -80,7 +106,7 @@ resource "hcloud_server" "first_control_plane" { provisioner "local-exec" { command = "kubectl apply -f ${local_file.traefik_config.filename} --kubeconfig ${path.module}/kubeconfig.yaml" } -*/ + network { network_id = hcloud_network.k3s.id ip = local.first_control_plane_network_ip diff --git a/temp/output.tf b/output.tf similarity index 100% rename from temp/output.tf rename to output.tf diff --git a/servers.tf b/servers.tf new file mode 100644 index 0000000..a0e90e7 --- /dev/null +++ b/servers.tf @@ -0,0 +1,95 @@ +resource "hcloud_server" "control_planes" { + count = var.servers_num - 1 + name = "k3s-control-plane-${count.index + 1}" + + image = data.hcloud_image.linux.name + rescue = "linux64" + server_type = var.control_plane_server_type + location = var.location + ssh_keys = [hcloud_ssh_key.k3s.id] + firewall_ids = [hcloud_firewall.k3s.id] + placement_group_id = hcloud_placement_group.k3s_placement_group.id + + labels = { + "provisioner" = "terraform", + "engine" = "k3s", + } + + provisioner "file" { + content = templatefile("${path.module}/templates/config.ign.tpl", { + name = self.name + ssh_public_key = local.ssh_public_key + }) + destination = "/root/config.ign" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Install MicroOS + provisioner "remote-exec" { + inline = local.MicroOS_install_commands + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Wait for MicroOS to reboot and be ready + provisioner "local-exec" { + command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 30" + } + + # Generating k3s server config file + provisioner "file" { + content = templatefile("${path.module}/templates/server_config.yaml.tpl", { + first_control_plane_url = "https://${local.first_control_plane_network_ip}:6443" + node_ip = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) + token = random_password.k3s_token.result + node_name = self.name + }) + destination = "/etc/rancher/k3s/config.yaml" + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + # Run the other control plane + provisioner "remote-exec" { + inline = [ + "set -ex", + # first we disable automatic reboot (after transactional updates), and configure the reboot method as kured + "rebootmgrctl set-strategy off && echo 'REBOOT_METHOD=kured' > /etc/transactional-update.conf", + # then we initiate the cluster + "systemctl --now enable k3s-server", + ] + + connection { + user = "root" + private_key = local.ssh_private_key + agent_identity = local.ssh_identity + host = self.ipv4_address + } + } + + network { + network_id = hcloud_network.k3s.id + ip = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) + } + + depends_on = [ + hcloud_server.first_control_plane, + hcloud_network_subnet.k3s + ] +} diff --git a/temp/agents.tf b/temp/agents.tf deleted file mode 100644 index 290b8a4..0000000 --- a/temp/agents.tf +++ /dev/null @@ -1,63 +0,0 @@ -resource "hcloud_server" "agents" { - count = var.agents_num - name = "k3s-agent-${count.index}" - - image = data.hcloud_image.linux.name - rescue = "linux64" - server_type = var.agent_server_type - location = var.location - ssh_keys = [hcloud_ssh_key.k3s.id] - firewall_ids = [hcloud_firewall.k3s.id] - placement_group_id = hcloud_placement_group.k3s_placement_group.id - - - labels = { - "provisioner" = "terraform", - "engine" = "k3s", - "k3s_upgrade" = "true" - } - - provisioner "file" { - content = templatefile("${path.module}/templates/agent.tpl", { - name = self.name - ssh_public_key = local.ssh_public_key - k3s_token = random_password.k3s_token.result - master_ip = local.first_control_plane_network_ip - node_ip = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index) - }) - destination = "/tmp/config.yaml" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } - } - - - provisioner "remote-exec" { - inline = local.microOS_install_commands - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } - } - - provisioner "local-exec" { - command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 100" - } - - network { - network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index) - } - - depends_on = [ - hcloud_server.first_control_plane, - hcloud_network_subnet.k3s - ] -} diff --git a/temp/servers.tf b/temp/servers.tf deleted file mode 100644 index a774bd2..0000000 --- a/temp/servers.tf +++ /dev/null @@ -1,62 +0,0 @@ -resource "hcloud_server" "control_planes" { - count = var.servers_num - 1 - name = "k3s-control-plane-${count.index + 1}" - - image = data.hcloud_image.linux.name - rescue = "linux64" - server_type = var.control_plane_server_type - location = var.location - ssh_keys = [hcloud_ssh_key.k3s.id] - firewall_ids = [hcloud_firewall.k3s.id] - placement_group_id = hcloud_placement_group.k3s_placement_group.id - - labels = { - "provisioner" = "terraform", - "engine" = "k3s", - "k3s_upgrade" = "true" - } - - provisioner "file" { - content = templatefile("${path.module}/templates/server.tpl", { - name = self.name - ssh_public_key = local.ssh_public_key - k3s_token = random_password.k3s_token.result - master_ip = local.first_control_plane_network_ip - node_ip = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) - }) - destination = "/tmp/config.yaml" - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } - } - - - provisioner "remote-exec" { - inline = local.microOS_install_commands - - connection { - user = "root" - private_key = local.ssh_private_key - agent_identity = local.ssh_identity - host = self.ipv4_address - } - } - - provisioner "local-exec" { - command = "sleep 60 && ping ${self.ipv4_address} | grep --line-buffered 'bytes from' | head -1 && sleep 100" - } - - network { - network_id = hcloud_network.k3s.id - ip = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index) - } - - depends_on = [ - hcloud_server.first_control_plane, - hcloud_network_subnet.k3s - ] -} diff --git a/templates/agent.conf.tpl b/templates/agent.conf.tpl new file mode 100644 index 0000000..047b9cf --- /dev/null +++ b/templates/agent.conf.tpl @@ -0,0 +1,3 @@ +SERVER_URL="${server_url}" +NODE_TOKEN="${node_token}" +AGENT_OPTS= \ No newline at end of file diff --git a/templates/agent.tpl b/templates/agent.tpl deleted file mode 100644 index c9df5d6..0000000 --- a/templates/agent.tpl +++ /dev/null @@ -1,33 +0,0 @@ -ssh_authorized_keys: -- ${ssh_public_key} -hostname: ${name} -boot_cmd: -- | - echo 'auto eth0 - iface eth0 inet dhcp - auto eth1 - iface eth1 inet dhcp' > /etc/network/interfaces -- rc-update del connman boot -- rc-update add networking boot -- rc-update add ntpd default -run_cmd: -- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1" -k3os: - k3s_args: - - agent - - "--server" - - "https://${master_ip}:6443" - - "--node-ip" - - "${node_ip}" - - "--kubelet-arg" - - "cloud-provider=external" - - "--flannel-iface=eth1" - token: ${k3s_token} - ntp_servers: - - 0.de.pool.ntp.org - - 1.de.pool.ntp.org - dns_nameservers: - - 8.8.8.8 - - 1.1.1.1 - - 2001:4860:4860::8888 - - 2606:4700:4700::1111 \ No newline at end of file diff --git a/templates/agent_config.yaml.tpl b/templates/agent_config.yaml.tpl new file mode 100644 index 0000000..8f13611 --- /dev/null +++ b/templates/agent_config.yaml.tpl @@ -0,0 +1,4 @@ +node-ip: ${node_ip} +kubelet-arg: "cloud-provider=external" +flannel-iface: eth1 +node-name: ${node_name} \ No newline at end of file diff --git a/templates/master.tpl b/templates/master.tpl deleted file mode 100644 index ec9ca70..0000000 --- a/templates/master.tpl +++ /dev/null @@ -1,39 +0,0 @@ -ssh_authorized_keys: -- ${ssh_public_key} -hostname: ${name} -boot_cmd: -- | - echo 'auto eth0 - iface eth0 inet dhcp - auto eth1 - iface eth1 inet dhcp' > /etc/network/interfaces -- rc-update del connman boot -- rc-update add networking boot -- rc-update add ntpd default -run_cmd: -- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1" -k3os: - k3s_args: - - server - - "--cluster-init" - - "--disable-cloud-controller" - - "--disable=servicelb" - - "--disable=local-storage" - - "--flannel-iface=eth1" - - "--node-ip" - - "${master_ip}" - - "--advertise-address" - - "${master_ip}" - - "--tls-san" - - "${master_ip}" - - "--kubelet-arg" - - "cloud-provider=external" - token: ${k3s_token} - ntp_servers: - - 0.de.pool.ntp.org - - 1.de.pool.ntp.org - dns_nameservers: - - 8.8.8.8 - - 1.1.1.1 - - 2001:4860:4860::8888 - - 2606:4700:4700::1111 diff --git a/templates/master_config.yaml.tpl b/templates/master_config.yaml.tpl new file mode 100644 index 0000000..bef3ff1 --- /dev/null +++ b/templates/master_config.yaml.tpl @@ -0,0 +1,11 @@ +cluster-init: true +disable-cloud-controller: true +disable: servicelb +disable: local-storage +flannel-iface: eth1 +node-ip: ${node_ip} +advertise-address: ${node_ip} +tls-san: ${node_ip} +kubelet-arg: "cloud-provider=external" +token: ${token} +node-name: ${node_name} \ No newline at end of file diff --git a/templates/server.tpl b/templates/server.tpl deleted file mode 100644 index 079bb84..0000000 --- a/templates/server.tpl +++ /dev/null @@ -1,42 +0,0 @@ -ssh_authorized_keys: -- ${ssh_public_key} -hostname: ${name} -boot_cmd: -- | - echo 'auto eth0 - iface eth0 inet dhcp - auto eth1 - iface eth1 inet dhcp' > /etc/network/interfaces -- rc-update del connman boot -- rc-update add networking boot -- rc-update add ntpd default -run_cmd: -- sh -c "ip route add 10.0.0.0/16 via 10.0.0.1 dev eth1" -k3os: - k3s_args: - - server - - "--server" - - "https://${master_ip}:6443" - - "--disable-cloud-controller" - - "--disable-network-policy" - - "--disable=traefik" - - "--disable=servicelb" - - "--disable=local-storage" - - "--flannel-iface=eth1" - - "--node-ip" - - "${node_ip}" - - "--advertise-address" - - "${node_ip}" - - "--tls-san" - - "${node_ip}" - - "--kubelet-arg" - - "cloud-provider=external" - token: ${k3s_token} - ntp_servers: - - 0.de.pool.ntp.org - - 1.de.pool.ntp.org - dns_nameservers: - - 8.8.8.8 - - 1.1.1.1 - - 2001:4860:4860::8888 - - 2606:4700:4700::1111 \ No newline at end of file diff --git a/templates/server_config.yaml.tpl b/templates/server_config.yaml.tpl new file mode 100644 index 0000000..eafbe99 --- /dev/null +++ b/templates/server_config.yaml.tpl @@ -0,0 +1,11 @@ +server: ${first_control_plane_url} +disable-cloud-controller: true +disable: servicelb +disable: local-storage +flannel-iface: eth1 +node-ip: ${node_ip} +advertise-address: ${node_ip} +tls-san: ${node_ip} +kubelet-arg: "cloud-provider=external" +token: ${token} +node-name: ${node_name} \ No newline at end of file