From 76a1e0080503d44819741620f17d91b37a2ec57f Mon Sep 17 00:00:00 2001 From: Karim Naufal Date: Wed, 5 Jan 2022 15:04:22 +0100 Subject: [PATCH] added traefik --- README.md | 44 ++----------- hetzner/ccm/kustomization.yaml | 8 +++ hetzner/ccm/patch.yaml | 19 ++++++ hetzner/csi/kustomization.yaml | 8 +++ hetzner/csi/patch.yaml | 54 ++++++++++++++++ main.tf | 65 +++++++++++++++++++ manifests/hcloud-ccm-net.yaml | 87 -------------------------- manifests/helm/nginx/values.yaml | 8 --- manifests/upgrade/plans.yaml | 50 --------------- master.tf | 14 +++-- templates/rendered/traefik_config.yaml | 21 +++++++ templates/traefik_config.yaml.tpl | 21 +++++++ variables.tf | 5 ++ 13 files changed, 217 insertions(+), 187 deletions(-) create mode 100644 hetzner/ccm/kustomization.yaml create mode 100644 hetzner/ccm/patch.yaml create mode 100644 hetzner/csi/kustomization.yaml create mode 100644 hetzner/csi/patch.yaml delete mode 100644 manifests/hcloud-ccm-net.yaml delete mode 100644 manifests/helm/nginx/values.yaml delete mode 100644 manifests/upgrade/plans.yaml create mode 100644 templates/rendered/traefik_config.yaml create mode 100644 templates/traefik_config.yaml.tpl diff --git a/README.md b/README.md index 062c4fc..58d48e6 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ _Please note that we are not affiliated to Hetzner, this is just an open source - Lightweight and resource-efficient Kubernetes powered by [k3s](https://github.com/k3s-io/k3s) on [k3os](https://github.com/rancher/k3os) nodes. - Automatic HA with the default setting of two control-plane and agents nodes. - Add or remove as many nodes as you want while the cluster stays running (just change the number instances and run terraform apply again). -- (Optional) [Nginx ingress controller](https://kubernetes.github.io/ingress-nginx/) that will automatically use Hetzner's private network to allocate a Hetzner load balancer. +- Automatic Traefik ingress controller and with a hetzner load balancer and proxy protocol turned on. _It uses Terraform to deploy as it's easy to use, and Hetzner provides a great [Hetzner Terraform Provider](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs)._ @@ -72,6 +72,7 @@ agents_num = 2 location = "fsn1" agent_server_type = "cpx21" control_plane_server_type = "cpx11" +lb_server_type = "lb11" ``` ### Installation @@ -100,18 +101,6 @@ servers_num = 2 agents_num = 3 ``` -### Ingress Controller (Optional) - -When using Kubernetes, it is ideal to have an ingress controller to expose services to the outside world. And it turns out that the Hetzner Cloud Controller allows us to automatically deploy a Hetzner Load Balancer that the ingress controller can use. You can install the Nginx ingress controller with the following command: - -```sh -helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx -helm repo update -helm install --values=manifests/helm/nginx/values.yaml ingress-nginx ingress-nginx/ingress-nginx -n kube-system --kubeconfig kubeconfig.yaml -``` - -_Please note that the load balancer's geographic location and instance type are editable in [values.yaml](manifests/helm/nginx/values.yaml)._ - ## Usage @@ -147,37 +136,16 @@ By default, k3os and its embedded k3s instance get upgraded automatically on eac kubectl label node 'k3os.io/upgrade'- --kubeconfig kubeconfig.yaml ``` -### Individual components upgrade - -To upgrade individual components, you can use the following commands: - -- Hetzner CCM and CSI - -```sh -kubectl apply -f https://raw.githubusercontent.com/mysticaltech/kube-hetzner/master/manifests/hcloud-ccm-net.yaml --kubeconfig kubeconfig.yaml -kubectl apply -f https://raw.githubusercontent.com/hetznercloud/csi-driver/master/deploy/kubernetes/hcloud-csi.yml --kubeconfig kubeconfig.yaml -``` - -- (Optional, if installed) Nginx ingress controller - -```sh -helm repo update -helm upgrade --values=manifests/helm/nginx/values.yaml ingress-nginx ingress-nginx/ingress-nginx -n kube-system --kubeconfig kubeconfig.yaml -``` +As for the Hetzner CCM and CSI, their container images are set to latest and with and imagePullPolicy of "Always". This means that when the nodes upgrade, they will be automatically upgraded too. ## Takedown -If you choose to install the Nginx ingress controller, you need to delete it first to release the load balancer, as follows: - -```sh -helm delete ingress-nginx -n kube-system --kubeconfig kubeconfig.yaml -``` - Then you can proceed to take down the rest of the cluster with: ```sh -kubectl delete -f https://raw.githubusercontent.com/mysticaltech/kube-hetzner/master/manifests/hcloud-ccm-net.yaml --kubeconfig kubeconfig.yaml -kubectl delete -f https://raw.githubusercontent.com/hetznercloud/csi-driver/master/deploy/kubernetes/hcloud-csi.yml --kubeconfig kubeconfig.yaml +kubectl delete -k hetzer/csi --kubeconfig kubeconfig.yaml +kubectl delete -k hetzer/ccm --kubeconfig kubeconfig.yaml +hcloud load-balancer delete traefik terraform destroy -auto-approve ``` diff --git a/hetzner/ccm/kustomization.yaml b/hetzner/ccm/kustomization.yaml new file mode 100644 index 0000000..8e6eab1 --- /dev/null +++ b/hetzner/ccm/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml + +patchesStrategicMerge: +- patch.yaml \ No newline at end of file diff --git a/hetzner/ccm/patch.yaml b/hetzner/ccm/patch.yaml new file mode 100644 index 0000000..a631620 --- /dev/null +++ b/hetzner/ccm/patch.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system +spec: + template: + spec: + containers: + - image: hetznercloud/hcloud-cloud-controller-manager:latest + imagePullPolicy: Always + name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=true" + - "--cluster-cidr=10.42.0.0/16" \ No newline at end of file diff --git a/hetzner/csi/kustomization.yaml b/hetzner/csi/kustomization.yaml new file mode 100644 index 0000000..78ad622 --- /dev/null +++ b/hetzner/csi/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- https://raw.githubusercontent.com/hetznercloud/csi-driver/master/deploy/kubernetes/hcloud-csi.yml + +patchesStrategicMerge: +- patch.yaml \ No newline at end of file diff --git a/hetzner/csi/patch.yaml b/hetzner/csi/patch.yaml new file mode 100644 index 0000000..743d655 --- /dev/null +++ b/hetzner/csi/patch.yaml @@ -0,0 +1,54 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: hcloud-csi-controller + namespace: kube-system +spec: + template: + metadata: + labels: + app: hcloud-csi-controller + spec: + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:canary + imagePullPolicy: Always + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:canary + imagePullPolicy: Always + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always + volumes: + - name: socket-dir + emptyDir: {} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: hcloud-csi-node + namespace: kube-system + labels: + app: hcloud-csi +spec: + selector: + matchLabels: + app: hcloud-csi + template: + spec: + containers: + - name: csi-node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:canary + imagePullPolicy: Always + - name: hcloud-csi-driver + image: hetznercloud/hcloud-csi-driver:latest + imagePullPolicy: Always + - name: liveness-probe + image: quay.io/k8scsi/livenessprobe:canary + imagePullPolicy: Always \ No newline at end of file diff --git a/main.tf b/main.tf index aab2dcd..dbfc114 100644 --- a/main.tf +++ b/main.tf @@ -85,6 +85,63 @@ resource "hcloud_firewall" "k3s" { "0.0.0.0/0" ] } + + # Allow basic out traffic + # ICMP to ping outside services + rule { + direction = "out" + protocol = "icmp" + destination_ips = [ + "0.0.0.0/0" + ] + } + + # DNS + rule { + direction = "out" + protocol = "tcp" + port = "53" + destination_ips = [ + "0.0.0.0/0" + ] + } + rule { + direction = "out" + protocol = "udp" + port = "53" + destination_ips = [ + "0.0.0.0/0" + ] + } + + # HTTP(s) + rule { + direction = "out" + protocol = "tcp" + port = "80" + destination_ips = [ + "0.0.0.0/0" + ] + } + rule { + direction = "out" + protocol = "tcp" + port = "443" + destination_ips = [ + "0.0.0.0/0" + ] + } + + #NTP + rule { + direction = "out" + protocol = "udp" + port = "123" + destination_ips = [ + "0.0.0.0/0" + ] + } + } @@ -108,3 +165,11 @@ locals { data "hcloud_image" "linux" { name = local.hcloud_image_name } + +resource "local_file" "traefik_config" { + content = templatefile("${path.module}/templates/traefik_config.yaml.tpl", { + lb_server_type = var.lb_server_type + location = var.location + }) + filename = "${path.module}/templates/rendered/traefik_config.yaml" +} diff --git a/manifests/hcloud-ccm-net.yaml b/manifests/hcloud-ccm-net.yaml deleted file mode 100644 index 7269ff7..0000000 --- a/manifests/hcloud-ccm-net.yaml +++ /dev/null @@ -1,87 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cloud-controller-manager - namespace: kube-system ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:cloud-controller-manager -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hcloud-cloud-controller-manager - namespace: kube-system -spec: - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - app: hcloud-cloud-controller-manager - template: - metadata: - labels: - app: hcloud-cloud-controller-manager - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - spec: - serviceAccountName: cloud-controller-manager - dnsPolicy: Default - tolerations: - # this taint is set by all kubelets running `--cloud-provider=external` - # so we should tolerate it to schedule the cloud controller manager - - key: "node.cloudprovider.kubernetes.io/uninitialized" - value: "true" - effect: "NoSchedule" - - key: "CriticalAddonsOnly" - operator: "Exists" - # cloud controller manages should be able to run on masters - - key: "node-role.kubernetes.io/master" - effect: NoSchedule - - key: "node-role.kubernetes.io/control-plane" - effect: NoSchedule - - key: "node.kubernetes.io/not-ready" - effect: "NoSchedule" - hostNetwork: true - containers: - - image: hetznercloud/hcloud-cloud-controller-manager:v1.12.0 - name: hcloud-cloud-controller-manager - command: - - "/bin/hcloud-cloud-controller-manager" - - "--cloud-provider=hcloud" - - "--leader-elect=false" - - "--allow-untagged-cloud" - - "--allocate-node-cidrs=true" - - "--cluster-cidr=10.42.0.0/16" - resources: - requests: - cpu: 100m - memory: 50Mi - limits: - cpu: 500m - memory: 500Mi - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: HCLOUD_TOKEN - valueFrom: - secretKeyRef: - name: hcloud - key: token - - name: HCLOUD_NETWORK - valueFrom: - secretKeyRef: - name: hcloud - key: network \ No newline at end of file diff --git a/manifests/helm/nginx/values.yaml b/manifests/helm/nginx/values.yaml deleted file mode 100644 index b83fa93..0000000 --- a/manifests/helm/nginx/values.yaml +++ /dev/null @@ -1,8 +0,0 @@ -controller: - service: - type: LoadBalancer - annotations: - load-balancer.hetzner.cloud/name: nginx-ingress - load-balancer.hetzner.cloud/use-private-ip: true - load-balancer.hetzner.cloud/location: fsn1 - load-balancer.hetzner.cloud/type: lb11 diff --git a/manifests/upgrade/plans.yaml b/manifests/upgrade/plans.yaml deleted file mode 100644 index 8da7e2c..0000000 --- a/manifests/upgrade/plans.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Doc: https://rancher.com/docs/k3s/latest/en/upgrades/automated/ -# agent plan -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: k3s-agent - namespace: system-upgrade - labels: - k3s_upgrade: agent -spec: - concurrency: 1 - channel: https://update.k3s.io/v1-release/channels/stable - nodeSelector: - matchExpressions: - - {key: k3s_upgrade, operator: Exists} - - {key: k3s_upgrade, operator: NotIn, values: ["disabled", "false"]} - - {key: node-role.kubernetes.io/master, operator: NotIn, values: ["true"]} - serviceAccountName: system-upgrade - prepare: - image: rancher/k3s-upgrade - args: ["prepare", "k3s-server"] - drain: - force: true - skipWaitForDeleteTimeout: 60 - upgrade: - image: rancher/k3s-upgrade ---- -# server plan -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: k3s-server - namespace: system-upgrade - labels: - k3s_upgrade: server -spec: - concurrency: 1 - channel: https://update.k3s.io/v1-release/channels/stable - nodeSelector: - matchExpressions: - - {key: k3s_upgrade, operator: Exists} - - {key: k3s_upgrade, operator: NotIn, values: ["disabled", "false"]} - - {key: node-role.kubernetes.io/master, operator: In, values: ["true"]} - tolerations: - - {key: node-role.kubernetes.io/master, effect: NoSchedule, operator: Exists} - - {key: CriticalAddonsOnly, effect: NoExecute, operator: Exists} - serviceAccountName: system-upgrade - cordon: true - upgrade: - image: rancher/k3s-upgrade \ No newline at end of file diff --git a/master.tf b/master.tf index 5d162f5..ea1b58f 100644 --- a/master.tf +++ b/master.tf @@ -29,7 +29,7 @@ resource "hcloud_server" "first_control_plane" { } } - + # Install k3os provisioner "remote-exec" { inline = local.k3os_install_commands @@ -40,6 +40,7 @@ resource "hcloud_server" "first_control_plane" { } } + # Wait for k3os to be ready and fetch kubeconfig.yaml provisioner "local-exec" { command = <<-EOT sleep 60 && ping ${self.ipv4_address} | grep --line-buffered "bytes from" | head -1 && sleep 60 && scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${var.private_key} rancher@${self.ipv4_address}:/etc/rancher/k3s/k3s.yaml ${path.module}/kubeconfig.yaml @@ -47,16 +48,21 @@ resource "hcloud_server" "first_control_plane" { EOT } - # Install the Hetzner Cloud cloud controller and cloud storage interface + # Install Hetzner CCM and CSI provisioner "local-exec" { command = <<-EOT kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl apply -f ${path.module}/manifests/hcloud-ccm-net.yaml --kubeconfig ${path.module}/kubeconfig.yaml + kubectl apply -k ${path.module}/hetzner/ccm --kubeconfig ${path.module}/kubeconfig.yaml kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --kubeconfig ${path.module}/kubeconfig.yaml - kubectl apply -f https://raw.githubusercontent.com/hetznercloud/csi-driver/master/deploy/kubernetes/hcloud-csi.yml --kubeconfig ${path.module}/kubeconfig.yaml + kubectl apply -k ${path.module}/hetzner/csi --kubeconfig ${path.module}/kubeconfig.yaml EOT } + # Configure the Traefik ingress controller + provisioner "local-exec" { + command = "kubectl apply -f ${local_file.traefik_config.filename} --kubeconfig ${path.module}/kubeconfig.yaml" + } + network { network_id = hcloud_network.k3s.id ip = local.first_control_plane_network_ip diff --git a/templates/rendered/traefik_config.yaml b/templates/rendered/traefik_config.yaml new file mode 100644 index 0000000..e66390a --- /dev/null +++ b/templates/rendered/traefik_config.yaml @@ -0,0 +1,21 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: traefik + namespace: kube-system +spec: + valuesContent: |- + service: + enabled: true + type: LoadBalancer + annotations: + "load-balancer.hetzner.cloud/name": "traefik" + "load-balancer.hetzner.cloud/use-private-ip": "true" + "load-balancer.hetzner.cloud/location": "fsn1" + "load-balancer.hetzner.cloud/type": "lb11" + "load-balancer.hetzner.cloud/uses-proxyprotocol": "true" + additionalArguments: + - "--entryPoints.web.proxyProtocol.trustedIPs=127.0.0.1/32,10.0.0.0/8" + - "--entryPoints.websecure.proxyProtocol.trustedIPs=127.0.0.1/32,10.0.0.0/8" + - "--entryPoints.web.forwardedHeaders.trustedIPs=127.0.0.1/32,10.0.0.0/8" + - "--entryPoints.websecure.forwardedHeaders.trustedIPs=127.0.0.1/32,10.0.0.0/8" \ No newline at end of file diff --git a/templates/traefik_config.yaml.tpl b/templates/traefik_config.yaml.tpl new file mode 100644 index 0000000..787f5a9 --- /dev/null +++ b/templates/traefik_config.yaml.tpl @@ -0,0 +1,21 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: traefik + namespace: kube-system +spec: + valuesContent: |- + service: + enabled: true + type: LoadBalancer + annotations: + "load-balancer.hetzner.cloud/name": "traefik" + "load-balancer.hetzner.cloud/use-private-ip": "true" + "load-balancer.hetzner.cloud/location": "${location}" + "load-balancer.hetzner.cloud/type": "${lb_server_type}" + "load-balancer.hetzner.cloud/uses-proxyprotocol": "true" + additionalArguments: + - "--entryPoints.web.proxyProtocol.trustedIPs=127.0.0.1/32,10.0.0.0/8" + - "--entryPoints.websecure.proxyProtocol.trustedIPs=127.0.0.1/32,10.0.0.0/8" + - "--entryPoints.web.forwardedHeaders.trustedIPs=127.0.0.1/32,10.0.0.0/8" + - "--entryPoints.websecure.forwardedHeaders.trustedIPs=127.0.0.1/32,10.0.0.0/8" \ No newline at end of file diff --git a/variables.tf b/variables.tf index c561b9e..25be7cc 100644 --- a/variables.tf +++ b/variables.tf @@ -43,3 +43,8 @@ variable "agent_server_type" { description = "Default agent server type" default = "cx21" } + +variable "lb_server_type" { + description = "Default load balancer server type" + default = "lb11" +}