Add homeserver to infra

infrastructure/create-resources/ansible/inventory/group_vars/all.yml

@@ -3,9 +3,9 @@ k3s_version: v1.22.3+k3s1
 ansible_user: root
 systemd_dir: /etc/systemd/system
 systemd_network_dir: /etc/systemd/network
-master_ip: "{{ hostvars[groups['serverctl_master_hosts'][0]]['ansible_host'] | default(groups['serverctl_master_hosts'][0]) }}"
-extra_server_args: ""
-extra_agent_args: ""
+master_ip: "{{ hostvars[groups['serverctl_master_hosts'][0]]['wireguard_ip'] | default(groups['serverctl_master_hosts'][0]) }}"
+extra_server_args: "--flannel-iface=serverctl-wg0"
+extra_agent_args: "--flannel-iface=serverctl-wg0"
 
 ansible_become_method: su
 

infrastructure/create-resources/ansible/inventory/hosts.cfg

@@ -1,14 +1,32 @@
 [serverctl_master_hosts]
+95.217.155.228 ansible_host=95.217.155.228 wireguard_ip=10.1.1.1
 
 [serverctl_node_hosts]
+65.21.50.146 ansible_host=65.21.50.146 wireguard_ip=10.1.1.10
+95.216.162.16 ansible_host=95.216.162.16 wireguard_ip=10.1.1.11
 
-[serverctl_mesh_nodes]
-
-[serverctl_mesh_nodes:vars]
-pipelining=true
-ansible_ssh_user=root
-ansible_ssh_port=22
+[serverctl_home_servers]
+192.168.1.150 ansible_host=192.168.1.150 wireguard_ip=10.1.1.8
+#192.168.1.233 ansible_host=192.168.1.233 wireguard_ip=10.1.1.9
 
 [serverctl_cluster:children]
 serverctl_master_hosts
 serverctl_node_hosts
+
+[serverctl_super_cluster:children]
+serverctl_cluster
+serverctl_home_servers
+
+[serverctl_home_servers:vars]
+client_server=True
+
+[serverctl_super_cluster:vars]
+pipelining=true
+ansible_ssh_user=root
+ansible_ssh_port=22
+
+[serverctl_cluster:vars]
+client_server=False
+pipelining=true
+ansible_ssh_user=root
+ansible_ssh_port=22

infrastructure/create-resources/ansible/kubeconfig.yml

@@ -0,0 +1,8 @@
+- hosts: serverctl_master_hosts[0]
+  become: yes
+  tasks:
+    - name: Fetch kubeconfig
+      ansible.builtin.fetch:
+        src: ~/.kube/config
+        dest: temp/.kube/config
+

infrastructure/create-resources/ansible/ping.yml

@@ -1,5 +1,5 @@
 ---
-- hosts: serverctl_cluster
+- hosts: serverctl_super_cluster
   gather_facts: yes
   tasks:
     - name: ping

infrastructure/create-resources/ansible/roles/firewall/tasks/main.yml

@@ -0,0 +1,67 @@
+---
+- name: update packages
+  apt:
+    update_cache: yes
+    cache_valid_time: 3600
+  become: yes
+
+- name: install ufw
+  apt:
+    name: ufw
+    state: present
+  become: yes
+  when: ufw_enabled
+
+- name: Allow SSH in UFW
+  ufw:
+    rule: allow
+    port: "{{ ansible_ssh_port }}"
+    proto: tcp
+  become: yes
+  when: ufw_enabled
+
+- name: Allow wireguard port in UFW
+  ufw:
+    rule: allow
+    port: "{{ wireguard_port }}"
+    proto: udp
+  become: yes
+  when: ufw_enabled
+
+- name: Set ufw logging
+  ufw:
+    logging: "on"
+  become: yes
+  when: ufw_enabled
+
+- name: inter-node Wireguard UFW connectivity
+  ufw:
+    rule: allow
+    src: "{{ hostvars[item].wireguard_ip }}"
+  with_items: "{{ groups['all'] }}"
+  become: yes
+  when: ufw_enabled and item != inventory_hostname
+
+- name: Reject everything and enable UFW
+  ufw:
+    state: enabled
+    policy: reject
+    log: yes
+  become: yes
+  when: ufw_enabled
+
+- name: Allow 6443 in UFW /tcp
+  ufw:
+    rule: allow
+    port: "6443"
+    proto: tcp
+  become: yes
+  when: ufw_enabled
+
+- name: Allow 6443 in UFW udp
+  ufw:
+    rule: allow
+    port: "6443"
+    proto: udp
+  become: yes
+  when: ufw_enabled

infrastructure/create-resources/ansible/roles/k3s/master/tasks/main.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: Copy K3s service file
   register: k3s_service
   template:

infrastructure/create-resources/ansible/roles/k3s/master/templates/k3s.service.j2

@@ -7,7 +7,7 @@ After=network-online.target
 Type=notify
 ExecStartPre=-/sbin/modprobe br_netfilter
 ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} {{ extra_server_args | default("") }}
+ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} {{ extra_server_args | default("") }} --advertise-address {{master_ip}}
 KillMode=process
 Delegate=yes
 # Having non-zero Limit*s causes performance problems due to accounting overhead

infrastructure/create-resources/ansible/roles/k3s/node/templates/k3s.service.j2

@@ -7,7 +7,7 @@ After=network-online.target
 Type=notify
 ExecStartPre=-/sbin/modprobe br_netfilter
 ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s agent --server https://{{ master_ip }}:6443 --token {{ hostvars[groups['serverctl_master_hosts'][0]]['token'] }} {{ extra_agent_args | default("") }}
+ExecStart=/usr/local/bin/k3s agent --server https://{{ master_ip }}:6443 --token {{ hostvars[groups['serverctl_master_hosts'][0]]['token'] }} {{ extra_agent_args | default("") }} --node-ip {{inventory_hostname}}
 KillMode=process
 Delegate=yes
 # Having non-zero Limit*s causes performance problems due to accounting overhead

infrastructure/create-resources/ansible/roles/wireguard/mesh/tasks/main.yml

@@ -1,93 +1,54 @@
 ---
-- name: update packages
-  apt:
-    update_cache: yes
-    cache_valid_time: 3600
-  become: yes
-
-- name: install ufw
-  apt:
-    name: ufw
-    state: present
-  become: yes
-  when: ufw_enabled
-
-- name: Allow SSH in UFW
-  ufw:
-    rule: allow
-    port: "{{ ansible_ssh_port }}"
-    proto: tcp
-  become: yes
-  when: ufw_enabled
-
-- name: Set ufw logging
-  ufw:
-    logging: "on"
-  become: yes
-  when: ufw_enabled
-
-- name: inter-node Wireguard UFW connectivity
-  ufw:
-    rule: allow
-    src: "{{ hostvars[item].wireguard_ip }}"
-  with_items: "{{ groups['all'] }}"
-  become: yes
-  when: ufw_enabled and item != inventory_hostname
-
-- name: Reject everything and enable UFW
-  ufw:
-    state: enabled
-    policy: reject
-    log: yes
-  become: yes
-  when: ufw_enabled
-
-- name: enable and persist ip forwarding
-  sysctl:
-    name: net.ipv4.ip_forward
-    value: "1"
-    state: present
-    sysctl_set: yes
-    reload: yes
+- name: Print distro
+  ansible.builtin.debug:
+    msg: Current distro {{ansible_distribution}}
 
 - name: install wireguard
   apt:
     name: wireguard
     state: present
   become: yes
+  when: ansible_distribution == 'Debian' or ansible_distribution == "Ubuntu"
+
+- name: install wireguard
+  pacman:
+    name: wireguard-tools
+    state: present
+  become: yes
+  when: ansible_distribution == "Archlinux"
 
 - name: generate wireguard keypair
-  shell: wg genkey | tee /etc/wireguard/privatekey | wg pubkey | tee /etc/wireguard/publickey
+  shell: wg genkey | tee /etc/wireguard/serverctl-privatekey | wg pubkey | tee /etc/wireguard/serverctl-publickey
   args:
-    creates: /etc/wireguard/privatekey
+    creates: /etc/wireguard/serverctl-privatekey
   become: yes
 
 - name: register private key
-  shell: cat /etc/wireguard/privatekey
+  shell: cat /etc/wireguard/serverctl-privatekey
   register: wireguard_private_key
   changed_when: false
   become: yes
 
 - name: register public key
-  shell: cat /etc/wireguard/publickey
+  shell: cat /etc/wireguard/serverctl-publickey
   register: wireguard_public_key
   changed_when: false
   become: yes
 
 - name: generate preshared keypair
-  shell: "wg genpsk > /etc/wireguard/psk-{{item}}"
+  shell: "wg genpsk > /etc/wireguard/serverctl-psk-{{item}}"
   args:
-    creates: "/etc/wireguard/psk-{{item}}"
+    creates: "/etc/wireguard/serverctl-psk-{{item}}"
   when: inventory_hostname < item
-  with_items: "{{groups['serverctl_cluster']}}"
+  with_items: "{{groups['serverctl_super_cluster']}}"
   become: yes
 
 - name: register preshared key
-  shell: "cat /etc/wireguard/psk-{{item}}"
+  shell: "cat /etc/wireguard/serverctl-psk-{{item}}"
   register: wireguard_preshared_key
   changed_when: false
   when: inventory_hostname < item
-  with_items: "{{groups['serverctl_cluster']}}"
+  with_items: "{{groups['serverctl_super_cluster']}}"
   become: yes
 
 - name: message preshared keys
@@ -96,10 +57,15 @@
   with_items: "{{wireguard_preshared_key.results}}"
   become: yes
 
+#- name: print hostvars
+#  ansible.builtin.debug:
+#    msg: "{{hostvars[item]}}"
+#  with_items: "{{groups['serverctl_super_cluster']}}"
+
 - name: Setup wg0 device
   template:
     src: 'systemd.netdev'
-    dest: '{{systemd_network_dir}}/99-wg0.netdev'
+    dest: '{{systemd_network_dir}}/99-serverctl-wg0.netdev'
     owner: root
     group: systemd-network
     mode: 0640
@@ -109,7 +75,7 @@
 - name: Setup wg0 network
   template:
     src: 'systemd.network'
-    dest: "{{systemd_network_dir}}/99-wg0.network"
+    dest: "{{systemd_network_dir}}/99-serverctl-wg0.network"
     owner: root
     group: systemd-network
     mode: 0640

infrastructure/create-resources/ansible/roles/wireguard/mesh/templates/systemd.netdev

@@ -1,20 +1,22 @@
 [NetDev]
-Name=wg0
+Name=serverctl-wg0
 Kind=wireguard
-Description=WireGuard tunnel wg0
+Description=WireGuard tunnel serverctl-wg0
 
 [WireGuard]
 ListenPort={{ wireguard_port }}
 PrivateKey={{ wireguard_private_key.stdout }}
 
-{% for peer in groups['serverctl_cluster'] %}
+{% for peer in groups['serverctl_super_cluster'] %}
 {% if peer != inventory_hostname %}
 
 [WireGuardPeer]
 PublicKey={{ hostvars[peer].wireguard_public_key.stdout }}
 PresharedKey={{ wireguard_preshared_keys[peer] if inventory_hostname < peer else hostvars[peer].wireguard_preshared_keys[inventory_hostname] }}
 AllowedIPs={{ hostvars[peer].wireguard_ip }}/32
+{% if not hostvars[peer].client_server %}
 Endpoint={{ hostvars[peer].ansible_host }}:{{ wireguard_port }}
 PersistentKeepalive=25
 {% endif %}
+{% endif %}
 {% endfor %}

infrastructure/create-resources/ansible/roles/wireguard/mesh/templates/systemd.network

@@ -1,5 +1,5 @@
 [Match]
-Name=wg0
+Name=serverctl-wg0
 
 [Network]
 Address={{ wireguard_ip }}/{{ wireguard_mask_bits }}

infrastructure/create-resources/ansible/site.yml

@@ -5,15 +5,21 @@
   roles:
     - role: prereq
     - role: download
-    - role: './wireguard/mesh'
+    - role: firewall
+
+- hosts: serverctl_super_cluster
+  gather_facts: yes
+  become: yes
+  roles:
+    - role: wireguard/mesh
 
 - hosts: serverctl_master_hosts
   become: yes
   roles:
     - role: "./k3s/master"
 
-#- hosts: serverctl_node_hosts
-#  become: yes
-#  roles:
-#    - role: "./k3s/node"
-#
+- hosts: serverctl_node_hosts
+  become: yes
+  roles:
+    - role: "./k3s/node"
+

infrastructure/create-resources/templates/hosts.tftpl

@@ -1,18 +1,35 @@
 [serverctl_master_hosts]
 %{ for ip in serverctl_masters ~}
-${ip} ansible_host=${ip} wireguard_ip=${cidrhost("192.168.0.0/24", index(serverctl_masters, ip) + 1)}
+${ip} ansible_host=${ip} wireguard_ip=${cidrhost("10.1.1.0/24", index(serverctl_masters, ip) + 1)}
 %{ endfor ~}
 
 [serverctl_node_hosts]
 %{ for ip in serverctl_nodes ~}
-${ip} ansible_host=${ip} wireguard_ip=${cidrhost("192.168.1.0/24", index(serverctl_nodes, ip) + 1)}
+${ip} ansible_host=${ip} wireguard_ip=${cidrhost("10.1.1.0/24", index(serverctl_nodes, ip) + 10)}
 %{ endfor ~}
 
+[serverctl_home_servers]
+192.168.1.150 ansible_host=192.168.1.150 wireguard_ip=10.1.1.8
+#192.168.1.233 ansible_host=192.168.1.233 wireguard_ip=10.1.1.9
+
 [serverctl_cluster:children]
 serverctl_master_hosts
 serverctl_node_hosts
 
-[serverctl_cluster:vars]
+[serverctl_super_cluster:children]
+serverctl_cluster
+serverctl_home_servers
+
+[serverctl_home_servers:vars]
+client_server=True
+
+[serverctl_super_cluster:vars]
+pipelining=true
+ansible_ssh_user=root
+ansible_ssh_port=22
+
+[serverctl_cluster:vars]
+client_server=False
 pipelining=true
 ansible_ssh_user=root
 ansible_ssh_port=22