k3os master ok

2021-11-30 23:09:34 +01:00 · 2021-11-30 23:09:34 +01:00 · 61f8093951
commit 61f8093951
parent f308220bfe
9 changed files with 114 additions and 234 deletions
--- a/agents.tf
+++ b/agents.tf
@ -1,51 +0,0 @@
-resource "hcloud_server" "agents" {
-  count = var.agents_num
-  name  = "k3s-agent-${count.index}"
-
-  image        = data.hcloud_image.linux.name
-  server_type  = var.agent_server_type
-  location     = var.location
-  ssh_keys     = [hcloud_ssh_key.default.id]
-  firewall_ids = [hcloud_firewall.k3s.id]
-
-
-  labels = {
-    "provisioner" = "terraform",
-    "engine"      = "k3s",
-    "k3s_upgrade" = "true"
-  }
-
-  user_data = data.template_cloudinit_config.init_cfg.rendered
-
-  provisioner "remote-exec" {
-    inline = var.initial_commands
-
-    connection {
-      user        = "root"
-      private_key = file(var.private_key)
-      host        = self.ipv4_address
-    }
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - agent --server https://${local.first_control_plane_network_ip}:6443 --node-ip=${cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index)} ${var.k3s_agent_flags}"
-    ]
-
-    connection {
-      user        = "root"
-      private_key = file(var.private_key)
-      host        = self.ipv4_address
-    }
-  }
-
-  network {
-    network_id = hcloud_network.k3s.id
-    ip         = cidrhost(hcloud_network.k3s.ip_range, 2 + var.servers_num + count.index)
-  }
-
-  depends_on = [
-    hcloud_server.first_control_plane,
-    hcloud_network_subnet.k3s
-  ]
-}
--- a/init.cfg
+++ b/init.cfg
@ -1,42 +0,0 @@
-  #cloud-config
-  write_files:
-    - path: /etc/sysctl.d/k8s.conf
-      content: |
-        net.bridge.bridge-nf-call-ip6tables = 1
-        net.bridge.bridge-nf-call-iptables = 1
-    - path: /etc/sysctl.d/99-override_cilium_rp_filter.conf
-      content: |
-        net.ipv4.conf.lxc*.rp_filter = 0
-    - path: /etc/fail2ban/jail.local
-      content: |
-        [sshd]
-        enabled = true
-        banaction = iptables-multiport
-    - path: /etc/yum.repos.d/rpm-rancher-io.repo
-      content: |
-        [rancher] 
-        name=Rancher 
-        baseurl=https://rpm.rancher.io/k3s/stable/common/centos/8/noarch
-        enabled=1 
-        gpgcheck=1 
-        gpgkey=https://rpm.rancher.io/public.key
-    - path: /etc/dnf/automatic.conf
-      content: |
-        [commands]
-        upgrade_type = default
-        random_sleep = 10000
-        network_online_timeout = 60
-        download_updates = yes
-        apply_updates = yes
-        [emitters]
-        emit_via = stdio
-        [base]
-        debuglevel = 1
-
-  runcmd:
-    - sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
-    - sed -i -e '/^X11Forwarding/s/^.*$/X11Forwarding no/' /etc/ssh/sshd_config
-    - sed -i -e '/^#MaxAuthTries/s/^.*$/MaxAuthTries 2/' /etc/ssh/sshd_config
-    - sed -i -e '/^#AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/sshd_config
-    - sed -i -e '/^#AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config
-    - sed -i -e '/^#AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config
--- a/main.tf
+++ b/main.tf
@ -1,11 +1,11 @@
-resource "random_password" "k3s_cluster_secret" {
+resource "random_password" "k3s_token" {
  length  = 48
  special = false
 }

 resource "hcloud_ssh_key" "default" {
  name       = "K3S terraform module - Provisioning SSH key"
-  public_key = file(var.public_key)
+  public_key = local.ssh_public_key
 }

 resource "hcloud_network" "k3s" {
@ -77,43 +77,34 @@ resource "hcloud_firewall" "k3s" {
      "0.0.0.0/0"
    ]
  }
-}

-data "hcloud_image" "linux" {
-  name = "fedora-34"
-}
-
-data "template_file" "init_cfg" {
-  template = file("${path.module}/init.cfg")
-}
-
-# Render a multi-part cloud-init config making use of the part
-# above, and other source files
-data "template_cloudinit_config" "init_cfg" {
-  gzip          = true
-  base64_encode = true
-
-  # Main cloud-config configuration file.
-  part {
-    filename     = "init.cfg"
-    content_type = "text/cloud-config"
-    content      = data.template_file.init_cfg.rendered
+  # Allow ping on ipv4
+  rule {
+    direction = "in"
+    protocol  = "icmp"
+    source_ips = [
+      "0.0.0.0/0"
+    ]
  }
 }

-data "template_file" "ccm" {
-  template = file("${path.module}/manifests/hcloud-ccm-net.yaml")
+data "hcloud_image" "linux" {
+  name = "ubuntu-20.04"
 }

-data "template_file" "plans" {
-  template = file("${path.module}/manifests/upgrade/plans.yaml")
-}
-
-data "template_file" "kured" {
-  template = file("${path.module}/manifests/upgrade/kured.yaml")
-}
-
-
 locals {
  first_control_plane_network_ip = cidrhost(hcloud_network.k3s.ip_range, 2)
+  name_master                    = "k3s-control-plane-0"
+  ssh_public_key                 = trimspace(file(var.public_key))
+}
+
+data "template_file" "master" {
+  template = file("${path.module}/templates/master.tpl")
+
+  vars = {
+    name           = local.name_master
+    ssh_public_key = local.ssh_public_key
+    k3s_token      = random_password.k3s_token.result
+    ip             = local.first_control_plane_network_ip
+  }
 }
--- a/master.tf
+++ b/master.tf
@ -1,23 +1,21 @@
 resource "hcloud_server" "first_control_plane" {
-  name = "k3s-control-plane-0"
+  name = local.name_master

  image        = data.hcloud_image.linux.name
+  rescue       = "linux64"
  server_type  = var.control_plane_server_type
  location     = var.location
  ssh_keys     = [hcloud_ssh_key.default.id]
  firewall_ids = [hcloud_firewall.k3s.id]

-
  labels = {
    "provisioner" = "terraform",
-    "engine"      = "k3s",
-    "k3s_upgrade" = "true"
+    "engine"      = "k3s"
  }

-  user_data = data.template_cloudinit_config.init_cfg.rendered
-
-  provisioner "remote-exec" {
-    inline = var.initial_commands
+  provisioner "file" {
+    content     = data.template_file.master.rendered
+    destination = "/tmp/config.yaml"

    connection {
      user        = "root"
@ -26,19 +24,15 @@ resource "hcloud_server" "first_control_plane" {
    }
  }

+
  provisioner "remote-exec" {
    inline = [
-      "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --cluster-init --node-ip=${local.first_control_plane_network_ip} --advertise-address=${local.first_control_plane_network_ip} --tls-san=${local.first_control_plane_network_ip} ${var.k3s_server_flags}",
-      "until systemctl is-active --quiet k3s.service; do sleep 1; done",
-      "until kubectl get node ${self.name}; do sleep 1; done",
-      "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name}",
-      "kubectl apply -f -<<EOF\n${data.template_file.ccm.rendered}\nEOF",
-      "kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token}",
-      "kubectl apply -f https://raw.githubusercontent.com/hetznercloud/csi-driver/master/deploy/kubernetes/hcloud-csi.yml",
-      "kubectl apply -f https://raw.githubusercontent.com/rancher/system-upgrade-controller/master/manifests/system-upgrade-controller.yaml",
-      "sleep 33",
-      "kubectl apply -f -<<EOF\n${data.template_file.plans.rendered}\nEOF",
-      "kubectl apply -f -<<EOF\n${data.template_file.kured.rendered}\nEOF",
+      "apt install -y grub-efi grub-pc-bin mtools xorriso",
+      "latest=$(curl -s https://api.github.com/repos/rancher/k3os/releases | jq '.[0].tag_name')",
+      "curl -Lo ./install.sh https://raw.githubusercontent.com/rancher/k3os/$(echo $latest | xargs)/install.sh",
+      "chmod +x ./install.sh",
+      "./install.sh --config /tmp/config.yaml /dev/sda https://github.com/rancher/k3os/releases/download/$(echo $latest | xargs)/k3os-amd64.iso",
+      "shutdown -r now"
    ]

    connection {
@ -49,15 +43,19 @@ resource "hcloud_server" "first_control_plane" {
  }

  provisioner "local-exec" {
-    command = "scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.private_key} root@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml"
+    command = <<-EOT
+      ping ${self.ipv4_address} | grep --line-buffered "bytes from" | head -1 && sleep 60 && scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.private_key} rancher@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml
+      sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml
+    EOT
  }

  provisioner "local-exec" {
-    command = "sed -i -e 's/127.0.0.1/${self.ipv4_address}/g' ${path.module}/kubeconfig.yaml"
-  }
-
-  provisioner "local-exec" {
-    command = "helm repo add cilium https://helm.cilium.io/ --kubeconfig ${path.module}/kubeconfig.yaml; helm repo update --kubeconfig ${path.module}; helm install --values=manifests/helm/cilium/values.yaml cilium cilium/cilium -n kube-system --kubeconfig ${path.module}/kubeconfig.yaml"
+    command = <<-EOT
+      kubectl -n kube-system create secret generic hcloud --from-literal=token=${random_password.k3s_token.result} --from-literal=network=${hcloud_network.k3s.name} --kubeconfig ${path.module}/kubeconfig.yaml
+      kubectl apply -f ${path.module}/manifests/hcloud-ccm-net.yaml --kubeconfig ${path.module}/kubeconfig.yaml
+      kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${random_password.k3s_token.result} --kubeconfig ${path.module}/kubeconfig.yaml
+      kubectl apply -f https://raw.githubusercontent.com/hetznercloud/csi-driver/master/deploy/kubernetes/hcloud-csi.yml --kubeconfig ${path.module}/kubeconfig.yaml
+    EOT
  }

  network {
--- a/output.tf
+++ b/output.tf
@ -1,9 +0,0 @@
-output "controlplanes_public_ip" {
-  value       = concat([hcloud_server.first_control_plane.ipv4_address], hcloud_server.control_planes.*.ipv4_address)
-  description = "The public IP addresses of the controlplane server."
-}
-
-output "agents_public_ip" {
-  value       = hcloud_server.agents.*.ipv4_address
-  description = "The public IP addresses of the agent server."
-}
--- a/servers.tf
+++ b/servers.tf
@ -1,51 +0,0 @@
-resource "hcloud_server" "control_planes" {
-  count = var.servers_num - 1
-  name  = "k3s-control-plane-${count.index + 1}"
-
-  image        = data.hcloud_image.linux.name
-  server_type  = var.control_plane_server_type
-  location     = var.location
-  ssh_keys     = [hcloud_ssh_key.default.id]
-  firewall_ids = [hcloud_firewall.k3s.id]
-
-
-  labels = {
-    "provisioner" = "terraform",
-    "engine"      = "k3s",
-    "k3s_upgrade" = "true"
-  }
-
-  user_data = data.template_cloudinit_config.init_cfg.rendered
-
-  provisioner "remote-exec" {
-    inline = var.initial_commands
-
-    connection {
-      user        = "root"
-      private_key = file(var.private_key)
-      host        = self.ipv4_address
-    }
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "curl -sfL https://get.k3s.io | K3S_TOKEN=${random_password.k3s_cluster_secret.result} sh -s - server --server https://${local.first_control_plane_network_ip}:6443 --node-ip=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} --advertise-address=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} --tls-san=${cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)} ${var.k3s_server_flags}",
-    ]
-
-    connection {
-      user        = "root"
-      private_key = file(var.private_key)
-      host        = self.ipv4_address
-    }
-  }
-
-  network {
-    network_id = hcloud_network.k3s.id
-    ip         = cidrhost(hcloud_network.k3s.ip_range, 3 + count.index)
-  }
-
-  depends_on = [
-    hcloud_server.first_control_plane,
-    hcloud_network_subnet.k3s
-  ]
-}
--- a/templates/agent.tpl
+++ b/templates/agent.tpl
@ -0,0 +1,29 @@
+ssh_authorized_keys:
+- ${ssh_public_key}
+hostname: ${name}
+k3os:
+  k3s_args:
+  - server
+  --node-ip=${ip} 
+  --advertise-address=${ip} 
+  --bind-address=${ip} 
+  --tls-san=${ip}
+  --disable-cloud-controller
+  --disable-network-policy
+  --disable=traefik
+  --disable=servicelb
+  --disable='local-storage'
+  --kubelet-arg='cloud-provider=external'
+  token: ${k3s_token}
+  ntp_servers:
+  - 0.de.pool.ntp.org
+  - 1.de.pool.ntp.org
+  dns_nameservers:
+  - 8.8.8.8
+  - 1.1.1.1
+  - 8.8.4.4
+  - 1.0.0.1
+  - 2001:4860:4860::8888
+  - 2606:4700:4700::1111
+  - 2001:4860:4860::8844
+  - 2606:4700:4700::1001
--- a/templates/master.tpl
+++ b/templates/master.tpl
@ -0,0 +1,38 @@
+ssh_authorized_keys:
+- ${ssh_public_key}
+hostname: ${name}
+boot_cmd:
+- |
+  echo 'auto eth0
+  iface eth0 inet dhcp
+  auto eth1
+  iface eth1 inet dhcp' > /etc/network/interfaces
+- rc-update del connman boot
+- rc-update add networking boot
+- rc-update add ntpd default
+k3os:
+  k3s_args:
+  - server
+  - "--cluster-init"
+  - "--disable-cloud-controller"
+  - "--disable=traefik"
+  - "--disable=servicelb"
+  - "--disable=local-storage"
+  - "--flannel-iface=eth1"
+  - "--node-ip"
+  - "${ip}"
+  - "--advertise-address"
+  - "${ip}"
+  - "--tls-san"
+  - "${ip}"
+  - "--kubelet-arg"
+  - "cloud-provider=external"
+  token: ${k3s_token}
+  ntp_servers:
+  - 0.de.pool.ntp.org
+  - 1.de.pool.ntp.org
+  dns_nameservers:
+  - 8.8.8.8
+  - 1.1.1.1
+  - 2001:4860:4860::8888
+  - 2606:4700:4700::1111
--- a/variables.tf
+++ b/variables.tf
@ -43,26 +43,3 @@ variable "agent_server_type" {
  description = "Default agent server type"
  default     = "cx21"
 }
-
-variable "k3s_server_flags" {
-  description = "Important flags to make our setup work"
-  default     = "--disable-cloud-controller --disable-network-policy --disable=traefik --disable=servicelb --disable='local-storage' --kubelet-arg='cloud-provider=external' --flannel-backend=none"
-}
-
-variable "k3s_agent_flags" {
-  description = "Important flags to make our setup work"
-  default     = "--kubelet-arg='cloud-provider=external'"
-}
-
-variable "initial_commands" {
-  description = "Initial commands to run on each machines."
-  default = [
-    "dnf upgrade -y",
-    "dnf install -y container-selinux selinux-policy-base fail2ban k3s-selinux dnf-automatic jq dnf-utils",
-    "systemctl enable --now fail2ban",
-    "systemctl enable --now dnf-automatic.timer",
-    "systemctl disable firewalld",
-    "grubby --args='systemd.unified_cgroup_hierarchy=0' --update-kernel=ALL",
-    "sleep 11; shutdown -r +0"
-  ]
-}