merged staging

This commit is contained in:
Karim Naufal 2022-05-04 03:24:45 +02:00
commit cb012699a6
No known key found for this signature in database
GPG Key ID: 9CB4A7C28C139CA5
13 changed files with 137 additions and 29 deletions

View File

@ -38,6 +38,8 @@ _Please note that we are not affiliates of Hetzner; this is just an open-source
- Possibility to have a single node cluster with a proper ingress controller. - Possibility to have a single node cluster with a proper ingress controller.
- Ability to add nodes and nodepools when the cluster is running. - Ability to add nodes and nodepools when the cluster is running.
- Traefik ingress controller attached to a Hetzner load balancer with proxy protocol turned on. - Traefik ingress controller attached to a Hetzner load balancer with proxy protocol turned on.
- Possibility to turn Longhorn on, and optionally also turn Hetzner CSI off.
- Ability to switch to Calico as CNI, and Cilium can also be easily added.
- Tons of flexible configuration options to suit all needs. - Tons of flexible configuration options to suit all needs.
_It uses Terraform to deploy as it's easy to use, and Hetzner provides a great [Hetzner Terraform Provider](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs)._ _It uses Terraform to deploy as it's easy to use, and Hetzner provides a great [Hetzner Terraform Provider](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs)._
@ -54,7 +56,7 @@ Follow those simple steps, and your world's cheapest Kube cluster will be up and
First and foremost, you need to have a Hetzner Cloud account. You can sign up for free [here](https://hetzner.com/cloud/). First and foremost, you need to have a Hetzner Cloud account. You can sign up for free [here](https://hetzner.com/cloud/).
Then you'll need to have [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli), [kubectl](https://kubernetes.io/docs/tasks/tools/) cli, and [hcloud](<https://github.com/hetznercloud/cli>) the Hetzner cli. The easiest way is to use the [homebrew](https://brew.sh/) package manager to install them (available on Linux, Mac, and Windows Linux Subsystem). Then you'll need to have [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli), [kubectl](https://kubernetes.io/docs/tasks/tools/) cli and [hcloud](<https://github.com/hetznercloud/cli>) the Hetzner cli. The easiest way is to use the [homebrew](https://brew.sh/) package manager to install them (available on Linux, Mac, and Windows Linux Subsystem).
```sh ```sh
brew install terraform brew install terraform

View File

@ -13,6 +13,7 @@ module "agents" {
location = each.value.location location = each.value.location
server_type = each.value.server_type server_type = each.value.server_type
ipv4_subnet_id = hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0]].id ipv4_subnet_id = hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0]].id
packages_to_install = concat(var.enable_longhorn ? ["open-iscsi"] : [], [])
private_ipv4 = cidrhost(hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0]].ip_range, each.value.index + 101) private_ipv4 = cidrhost(hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0]].ip_range, each.value.index + 101)

View File

@ -13,6 +13,7 @@ module "control_planes" {
location = each.value.location location = each.value.location
server_type = each.value.server_type server_type = each.value.server_type
ipv4_subnet_id = hcloud_network_subnet.control_plane[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0]].id ipv4_subnet_id = hcloud_network_subnet.control_plane[[for i, v in var.control_plane_nodepools : i if v.name == each.value.nodepool_name][0]].id
packages_to_install = concat(var.enable_longhorn ? ["open-iscsi"] : [], [])
# We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely # We leave some room so 100 eventual Hetzner LBs that can be created perfectly safely
# It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough. # It leaves the subnet with 254 x 254 - 100 = 64416 IPs to use, so probably enough.

45
init.tf
View File

@ -80,18 +80,26 @@ resource "null_resource" "kustomization" {
content = yamlencode({ content = yamlencode({
apiVersion = "kustomize.config.k8s.io/v1beta1" apiVersion = "kustomize.config.k8s.io/v1beta1"
kind = "Kustomization" kind = "Kustomization"
resources = concat([
"https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${local.ccm_version}/ccm-networks.yaml", resources = concat(
"https://raw.githubusercontent.com/hetznercloud/csi-driver/${local.csi_version}/deploy/kubernetes/hcloud-csi.yml", [
"https://github.com/weaveworks/kured/releases/download/${local.kured_version}/kured-${local.kured_version}-dockerhub.yaml", "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/${local.ccm_version}/ccm-networks.yaml",
"https://raw.githubusercontent.com/rancher/system-upgrade-controller/master/manifests/system-upgrade-controller.yaml", "https://github.com/weaveworks/kured/releases/download/${local.kured_version}/kured-${local.kured_version}-dockerhub.yaml",
], local.is_single_node_cluster ? [] : var.traefik_enabled ? ["traefik_config.yaml"] : [] "https://raw.githubusercontent.com/rancher/system-upgrade-controller/master/manifests/system-upgrade-controller.yaml",
, var.cni_plugin == "calico" ? ["https://projectcalico.docs.tigera.io/manifests/calico.yaml"] : []), ],
patchesStrategicMerge = concat([ var.disable_hetzner_csi ? [] : ["https://raw.githubusercontent.com/hetznercloud/csi-driver/${local.csi_version}/deploy/kubernetes/hcloud-csi.yml"],
file("${path.module}/kustomize/kured.yaml"), var.enable_longhorn ? ["longhorn.yaml"] : [],
file("${path.module}/kustomize/system-upgrade-controller.yaml"), local.is_single_node_cluster ? [] : var.traefik_enabled ? ["traefik_config.yaml"] : [],
"ccm.yaml" var.cni_plugin == "calico" ? ["https://projectcalico.docs.tigera.io/manifests/calico.yaml"] : []
], var.cni_plugin == "calico" ? ["calico.yaml"] : []) ),
patchesStrategicMerge = concat(
[
file("${path.module}/kustomize/kured.yaml"),
file("${path.module}/kustomize/ccm.yaml"),
file("${path.module}/kustomize/system-upgrade-controller.yaml")
],
var.cni_plugin == "calico" ? [file("${path.module}/kustomize/calico.yaml")] : []
)
}) })
destination = "/var/post_install/kustomization.yaml" destination = "/var/post_install/kustomization.yaml"
} }
@ -142,6 +150,16 @@ resource "null_resource" "kustomization" {
destination = "/var/post_install/plans.yaml" destination = "/var/post_install/plans.yaml"
} }
# Upload the Longhorn config
provisioner "file" {
content = templatefile(
"${path.module}/templates/longhorn.yaml.tpl",
{
disable_hetzner_csi = var.disable_hetzner_csi
})
destination = "/var/post_install/longhorn.yaml"
}
# Deploy secrets, logging is automatically disabled due to sensitive variables # Deploy secrets, logging is automatically disabled due to sensitive variables
provisioner "remote-exec" { provisioner "remote-exec" {
inline = [ inline = [
@ -181,6 +199,7 @@ resource "null_resource" "kustomization" {
} }
depends_on = [ depends_on = [
null_resource.first_control_plane null_resource.first_control_plane,
local_sensitive_file.kubeconfig
] ]
} }

View File

@ -13,7 +13,7 @@ locals {
ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag ccm_version = var.hetzner_ccm_version != null ? var.hetzner_ccm_version : data.github_release.hetzner_ccm.release_tag
csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag csi_version = var.hetzner_csi_version != null ? var.hetzner_csi_version : data.github_release.hetzner_csi.release_tag
kured_version = data.github_release.kured.release_tag kured_version = var.kured_version != null ? var.kured_version : data.github_release.kured.release_tag
common_commands_install_k3s = [ common_commands_install_k3s = [
"set -ex", "set -ex",

View File

@ -1,16 +1,23 @@
locals { locals {
# ssh public key
ssh_public_key = trimspace(file(var.public_key)) ssh_public_key = trimspace(file(var.public_key))
# ssh_private_key is either the contents of var.private_key or null to use a ssh agent. # ssh_private_key is either the contents of var.private_key or null to use a ssh agent.
ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key)) ssh_private_key = var.private_key == null ? null : trimspace(file(var.private_key))
# ssh_identity is not set if the private key is passed directly, but if ssh agent is used, the public key tells ssh agent which private key to use. # ssh_identity is not set if the private key is passed directly, but if ssh agent is used, the public key tells ssh agent which private key to use.
# For terraforms provisioner.connection.agent_identity, we need the public key as a string. # For terraforms provisioner.connection.agent_identity, we need the public key as a string.
ssh_identity = var.private_key == null ? local.ssh_public_key : null ssh_identity = var.private_key == null ? local.ssh_public_key : null
# ssh_identity_file is used for ssh "-i" flag, its the private key if that is set, or a public key file # ssh_identity_file is used for ssh "-i" flag, its the private key if that is set, or a public key file
# if an ssh agent is used. # if an ssh agent is used.
ssh_identity_file = var.private_key == null ? var.public_key : var.private_key ssh_identity_file = var.private_key == null ? var.public_key : var.private_key
# shared flags for ssh to ignore host keys, to use our ssh identity file for all connections during provisioning. # shared flags for ssh to ignore host keys, to use our ssh identity file for all connections during provisioning.
ssh_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${local.ssh_identity_file}" ssh_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${local.ssh_identity_file}"
# Final list of packages to install
needed_packages = join(" ", concat(["k3s-selinux"], var.packages_to_install))
# the hosts name with its unique suffix attached # the hosts name with its unique suffix attached
name = "${var.name}-${random_string.server.id}" name = "${var.name}-${random_string.server.id}"
} }

View File

@ -65,11 +65,12 @@ resource "hcloud_server" "server" {
EOT EOT
} }
# Install k3s-selinux (compatible version) # Install k3s-selinux (compatible version) and open-iscsi
provisioner "remote-exec" { provisioner "remote-exec" {
inline = [ inline = [<<-EOT
"set -ex", set -ex
"transactional-update shell <<< 'rpm --import https://rpm.rancher.io/public.key;zypper install -y https://github.com/k3s-io/k3s-selinux/releases/download/v0.5.stable.1/k3s-selinux-0.5-1.sle.noarch.rpm'" transactional-update shell <<< "zypper --gpg-auto-import-keys install -y ${local.needed_packages}"
EOT
] ]
} }
@ -84,6 +85,17 @@ resource "hcloud_server" "server" {
done done
EOT EOT
} }
# Enable open-iscsi
provisioner "remote-exec" {
inline = [<<-EOT
set -ex
if [[ $(systemctl list-units --all -t service --full --no-legend "iscsid.service" | sed 's/^\s*//g' | cut -f1 -d' ') == iscsid.service ]]; then
systemctl enable --now iscsid
fi
EOT
]
}
} }
resource "hcloud_server_network" "server" { resource "hcloud_server_network" "server" {

View File

@ -23,6 +23,17 @@ write_files:
REBOOT_METHOD=kured REBOOT_METHOD=kured
path: /etc/transactional-update.conf path: /etc/transactional-update.conf
# Create Rancher repo config
- content: |
[rancher-k3s-common-stable]
name=Rancher K3s Common (stable)
baseurl=https://rpm.rancher.io/k3s/stable/common/microos/noarch
enabled=1
gpgcheck=1
repo_gpgcheck=0
gpgkey=https://rpm.rancher.io/public.key
path: /etc/zypp/repos.d/rancher-k3s-common.repo
# Add ssh authorized keys # Add ssh authorized keys
ssh_authorized_keys: ssh_authorized_keys:
%{ for key in sshAuthorizedKeys ~} %{ for key in sshAuthorizedKeys ~}

View File

@ -62,3 +62,9 @@ variable "server_type" {
description = "The server type" description = "The server type"
type = string type = string
} }
variable "packages_to_install" {
description = "Packages to install"
type = list(string)
default = []
}

View File

@ -0,0 +1,22 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: longhorn
---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: longhorn
namespace: kube-system
spec:
chart: longhorn
repo: https://charts.longhorn.io
targetNamespace: longhorn
valuesContent: |-
defaultSettings:
defaultDataPath: /var/longhorn
persistence:
defaultFsType: ext4
defaultClassReplicaCount: 2
%{ if disable_hetzner_csi ~}defaultClass: true%{ else ~}defaultClass: false%{ endif ~}

View File

@ -103,10 +103,18 @@ load_balancer_location = "fsn1"
### The following values are entirely optional ### The following values are entirely optional
# If you want to use a specific Hetzner CCM and CSI version, set them below; otherwise, leave them as-is for the latest versions # To use local storage on the nodes, you can enable Longhorn, default is "false".
# enable_longhorn = true
# To disable Hetzner CSI storage, you can set the following to true, default is "false".
# disable_hetzner_csi = true
# If you want to use a specific Hetzner CCM and CSI version, set them below; otherwise, leave them as-is for the latest versions.
# hetzner_ccm_version = "" # hetzner_ccm_version = ""
# hetzner_csi_version = "" # hetzner_csi_version = ""
# If you want to specify the Kured version, set it below - otherwise it'll use the latest version available
# kured_version = ""
# We give you the possibility to use letsencrypt directly with Traefik because it's an easy setup, however it's not optimal, # We give you the possibility to use letsencrypt directly with Traefik because it's an easy setup, however it's not optimal,
# as the free version of Traefik causes a little bit of downtime when when the certificates get renewed. For proper SSL management, # as the free version of Traefik causes a little bit of downtime when when the certificates get renewed. For proper SSL management,
@ -127,7 +135,8 @@ load_balancer_location = "fsn1"
# If you want to disable the automatic upgrade of k3s, you can set this to false. The default is "true". # If you want to disable the automatic upgrade of k3s, you can set this to false. The default is "true".
# automatically_upgrade_k3s = false # automatically_upgrade_k3s = false
# Allows you to specify either stable, latest, or testing (defaults to stable), see https://rancher.com/docs/k3s/latest/en/upgrades/basic/ # Allows you to specify either stable, latest, testing or supported minor versions (defaults to stable)
# see https://rancher.com/docs/k3s/latest/en/upgrades/basic/ and https://update.k3s.io/v1-release/channels
# initial_k3s_channel = "latest" # initial_k3s_channel = "latest"
# The cluster name, by default "k3s" # The cluster name, by default "k3s"
@ -167,8 +176,8 @@ load_balancer_location = "fsn1"
# CAVEATS: Calico is not supported for single node setups, because of the following issue https://github.com/k3s-io/klipper-lb/issues/6. # CAVEATS: Calico is not supported for single node setups, because of the following issue https://github.com/k3s-io/klipper-lb/issues/6.
# cni_plugin = "calico" # cni_plugin = "calico"
# If you want to disable the k3s default network policy controller, use this flag # If you want to disable the k3s default network policy controller, use this flag!
# Calico overrides this value to true automatically # Calico overrides this value to true automatically, the default is "false".
# disable_network_policy = true # disable_network_policy = true
# If you want to disable the automatic use of placement group "spread". See https://docs.hetzner.com/cloud/placement-groups/overview/ # If you want to disable the automatic use of placement group "spread". See https://docs.hetzner.com/cloud/placement-groups/overview/

View File

@ -65,6 +65,12 @@ variable "hetzner_csi_version" {
description = "Version of Container Storage Interface driver for Hetzner Cloud" description = "Version of Container Storage Interface driver for Hetzner Cloud"
} }
variable "kured_version" {
type = string
default = null
description = "Version of Kured"
}
variable "traefik_enabled" { variable "traefik_enabled" {
type = bool type = bool
default = true default = true
@ -101,7 +107,7 @@ variable "initial_k3s_channel" {
description = "Allows you to specify an initial k3s channel" description = "Allows you to specify an initial k3s channel"
validation { validation {
condition = contains(["stable", "latest", "testing"], var.initial_k3s_channel) condition = contains(["stable", "latest", "testing", "v1.16", "v1.17", "v1.18", "v1.19", "v1.20", "v1.21", "v1.22", "v1.23"], var.initial_k3s_channel)
error_message = "The initial k3s channel must be one of stable, latest or testing." error_message = "The initial k3s channel must be one of stable, latest or testing."
} }
} }
@ -157,3 +163,15 @@ variable "cni_plugin" {
default = "flannel" default = "flannel"
description = "CNI plugin for k3s" description = "CNI plugin for k3s"
} }
variable "enable_longhorn" {
type = bool
default = false
description = "Enable Longhorn"
}
variable "disable_hetzner_csi" {
type = bool
default = false
description = "Disable hetzner csi driver"
}