diff --git a/infrastructure/create-resources/ansible/inventory/group_vars/all.yml b/infrastructure/create-resources/ansible/inventory/group_vars/all.yml index 9039b33..3f229de 100644 --- a/infrastructure/create-resources/ansible/inventory/group_vars/all.yml +++ b/infrastructure/create-resources/ansible/inventory/group_vars/all.yml @@ -3,9 +3,9 @@ k3s_version: v1.22.3+k3s1 ansible_user: root systemd_dir: /etc/systemd/system systemd_network_dir: /etc/systemd/network -master_ip: "{{ hostvars[groups['serverctl_master_hosts'][0]]['ansible_host'] | default(groups['serverctl_master_hosts'][0]) }}" -extra_server_args: "" -extra_agent_args: "" +master_ip: "{{ hostvars[groups['serverctl_master_hosts'][0]]['wireguard_ip'] | default(groups['serverctl_master_hosts'][0]) }}" +extra_server_args: "--flannel-iface=serverctl-wg0" +extra_agent_args: "--flannel-iface=serverctl-wg0" ansible_become_method: su diff --git a/infrastructure/create-resources/ansible/inventory/hosts.cfg b/infrastructure/create-resources/ansible/inventory/hosts.cfg index ec39c8d..e492eca 100755 --- a/infrastructure/create-resources/ansible/inventory/hosts.cfg +++ b/infrastructure/create-resources/ansible/inventory/hosts.cfg @@ -1,14 +1,32 @@ [serverctl_master_hosts] +95.217.155.228 ansible_host=95.217.155.228 wireguard_ip=10.1.1.1 [serverctl_node_hosts] +65.21.50.146 ansible_host=65.21.50.146 wireguard_ip=10.1.1.10 +95.216.162.16 ansible_host=95.216.162.16 wireguard_ip=10.1.1.11 -[serverctl_mesh_nodes] - -[serverctl_mesh_nodes:vars] -pipelining=true -ansible_ssh_user=root -ansible_ssh_port=22 +[serverctl_home_servers] +192.168.1.150 ansible_host=192.168.1.150 wireguard_ip=10.1.1.8 +#192.168.1.233 ansible_host=192.168.1.233 wireguard_ip=10.1.1.9 [serverctl_cluster:children] serverctl_master_hosts serverctl_node_hosts + +[serverctl_super_cluster:children] +serverctl_cluster +serverctl_home_servers + +[serverctl_home_servers:vars] +client_server=True + +[serverctl_super_cluster:vars] +pipelining=true +ansible_ssh_user=root +ansible_ssh_port=22 + +[serverctl_cluster:vars] +client_server=False +pipelining=true +ansible_ssh_user=root +ansible_ssh_port=22 diff --git a/infrastructure/create-resources/ansible/kubeconfig.yml b/infrastructure/create-resources/ansible/kubeconfig.yml new file mode 100644 index 0000000..4da6445 --- /dev/null +++ b/infrastructure/create-resources/ansible/kubeconfig.yml @@ -0,0 +1,8 @@ +- hosts: serverctl_master_hosts[0] + become: yes + tasks: + - name: Fetch kubeconfig + ansible.builtin.fetch: + src: ~/.kube/config + dest: temp/.kube/config + diff --git a/infrastructure/create-resources/ansible/ping.yml b/infrastructure/create-resources/ansible/ping.yml index f58cde5..a4979f5 100644 --- a/infrastructure/create-resources/ansible/ping.yml +++ b/infrastructure/create-resources/ansible/ping.yml @@ -1,5 +1,5 @@ --- -- hosts: serverctl_cluster +- hosts: serverctl_super_cluster gather_facts: yes tasks: - name: ping diff --git a/infrastructure/create-resources/ansible/roles/firewall/tasks/main.yml b/infrastructure/create-resources/ansible/roles/firewall/tasks/main.yml new file mode 100644 index 0000000..faf6563 --- /dev/null +++ b/infrastructure/create-resources/ansible/roles/firewall/tasks/main.yml @@ -0,0 +1,67 @@ +--- +- name: update packages + apt: + update_cache: yes + cache_valid_time: 3600 + become: yes + +- name: install ufw + apt: + name: ufw + state: present + become: yes + when: ufw_enabled + +- name: Allow SSH in UFW + ufw: + rule: allow + port: "{{ ansible_ssh_port }}" + proto: tcp + become: yes + when: ufw_enabled + +- name: Allow wireguard port in UFW + ufw: + rule: allow + port: "{{ wireguard_port }}" + proto: udp + become: yes + when: ufw_enabled + +- name: Set ufw logging + ufw: + logging: "on" + become: yes + when: ufw_enabled + +- name: inter-node Wireguard UFW connectivity + ufw: + rule: allow + src: "{{ hostvars[item].wireguard_ip }}" + with_items: "{{ groups['all'] }}" + become: yes + when: ufw_enabled and item != inventory_hostname + +- name: Reject everything and enable UFW + ufw: + state: enabled + policy: reject + log: yes + become: yes + when: ufw_enabled + +- name: Allow 6443 in UFW /tcp + ufw: + rule: allow + port: "6443" + proto: tcp + become: yes + when: ufw_enabled + +- name: Allow 6443 in UFW udp + ufw: + rule: allow + port: "6443" + proto: udp + become: yes + when: ufw_enabled diff --git a/infrastructure/create-resources/ansible/roles/k3s/master/tasks/main.yml b/infrastructure/create-resources/ansible/roles/k3s/master/tasks/main.yml index 6fd0cd0..77b58f6 100644 --- a/infrastructure/create-resources/ansible/roles/k3s/master/tasks/main.yml +++ b/infrastructure/create-resources/ansible/roles/k3s/master/tasks/main.yml @@ -1,4 +1,5 @@ --- + - name: Copy K3s service file register: k3s_service template: diff --git a/infrastructure/create-resources/ansible/roles/k3s/master/templates/k3s.service.j2 b/infrastructure/create-resources/ansible/roles/k3s/master/templates/k3s.service.j2 index ee560ae..fbeba45 100644 --- a/infrastructure/create-resources/ansible/roles/k3s/master/templates/k3s.service.j2 +++ b/infrastructure/create-resources/ansible/roles/k3s/master/templates/k3s.service.j2 @@ -7,7 +7,7 @@ After=network-online.target Type=notify ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} {{ extra_server_args | default("") }} +ExecStart=/usr/local/bin/k3s server --data-dir {{ k3s_server_location }} {{ extra_server_args | default("") }} --advertise-address {{master_ip}} KillMode=process Delegate=yes # Having non-zero Limit*s causes performance problems due to accounting overhead diff --git a/infrastructure/create-resources/ansible/roles/k3s/node/templates/k3s.service.j2 b/infrastructure/create-resources/ansible/roles/k3s/node/templates/k3s.service.j2 index 4c6f3f8..33b09f8 100644 --- a/infrastructure/create-resources/ansible/roles/k3s/node/templates/k3s.service.j2 +++ b/infrastructure/create-resources/ansible/roles/k3s/node/templates/k3s.service.j2 @@ -7,7 +7,7 @@ After=network-online.target Type=notify ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s agent --server https://{{ master_ip }}:6443 --token {{ hostvars[groups['serverctl_master_hosts'][0]]['token'] }} {{ extra_agent_args | default("") }} +ExecStart=/usr/local/bin/k3s agent --server https://{{ master_ip }}:6443 --token {{ hostvars[groups['serverctl_master_hosts'][0]]['token'] }} {{ extra_agent_args | default("") }} --node-ip {{inventory_hostname}} KillMode=process Delegate=yes # Having non-zero Limit*s causes performance problems due to accounting overhead diff --git a/infrastructure/create-resources/ansible/roles/wireguard/mesh/tasks/main.yml b/infrastructure/create-resources/ansible/roles/wireguard/mesh/tasks/main.yml index c32d524..b4216d0 100644 --- a/infrastructure/create-resources/ansible/roles/wireguard/mesh/tasks/main.yml +++ b/infrastructure/create-resources/ansible/roles/wireguard/mesh/tasks/main.yml @@ -1,93 +1,54 @@ --- -- name: update packages - apt: - update_cache: yes - cache_valid_time: 3600 - become: yes - -- name: install ufw - apt: - name: ufw - state: present - become: yes - when: ufw_enabled - -- name: Allow SSH in UFW - ufw: - rule: allow - port: "{{ ansible_ssh_port }}" - proto: tcp - become: yes - when: ufw_enabled - -- name: Set ufw logging - ufw: - logging: "on" - become: yes - when: ufw_enabled - -- name: inter-node Wireguard UFW connectivity - ufw: - rule: allow - src: "{{ hostvars[item].wireguard_ip }}" - with_items: "{{ groups['all'] }}" - become: yes - when: ufw_enabled and item != inventory_hostname - -- name: Reject everything and enable UFW - ufw: - state: enabled - policy: reject - log: yes - become: yes - when: ufw_enabled - -- name: enable and persist ip forwarding - sysctl: - name: net.ipv4.ip_forward - value: "1" - state: present - sysctl_set: yes - reload: yes +- name: Print distro + ansible.builtin.debug: + msg: Current distro {{ansible_distribution}} - name: install wireguard apt: name: wireguard state: present become: yes + when: ansible_distribution == 'Debian' or ansible_distribution == "Ubuntu" + +- name: install wireguard + pacman: + name: wireguard-tools + state: present + become: yes + when: ansible_distribution == "Archlinux" - name: generate wireguard keypair - shell: wg genkey | tee /etc/wireguard/privatekey | wg pubkey | tee /etc/wireguard/publickey + shell: wg genkey | tee /etc/wireguard/serverctl-privatekey | wg pubkey | tee /etc/wireguard/serverctl-publickey args: - creates: /etc/wireguard/privatekey + creates: /etc/wireguard/serverctl-privatekey become: yes - name: register private key - shell: cat /etc/wireguard/privatekey + shell: cat /etc/wireguard/serverctl-privatekey register: wireguard_private_key changed_when: false become: yes - name: register public key - shell: cat /etc/wireguard/publickey + shell: cat /etc/wireguard/serverctl-publickey register: wireguard_public_key changed_when: false become: yes - name: generate preshared keypair - shell: "wg genpsk > /etc/wireguard/psk-{{item}}" + shell: "wg genpsk > /etc/wireguard/serverctl-psk-{{item}}" args: - creates: "/etc/wireguard/psk-{{item}}" + creates: "/etc/wireguard/serverctl-psk-{{item}}" when: inventory_hostname < item - with_items: "{{groups['serverctl_cluster']}}" + with_items: "{{groups['serverctl_super_cluster']}}" become: yes - name: register preshared key - shell: "cat /etc/wireguard/psk-{{item}}" + shell: "cat /etc/wireguard/serverctl-psk-{{item}}" register: wireguard_preshared_key changed_when: false when: inventory_hostname < item - with_items: "{{groups['serverctl_cluster']}}" + with_items: "{{groups['serverctl_super_cluster']}}" become: yes - name: message preshared keys @@ -96,10 +57,15 @@ with_items: "{{wireguard_preshared_key.results}}" become: yes +#- name: print hostvars +# ansible.builtin.debug: +# msg: "{{hostvars[item]}}" +# with_items: "{{groups['serverctl_super_cluster']}}" + - name: Setup wg0 device template: src: 'systemd.netdev' - dest: '{{systemd_network_dir}}/99-wg0.netdev' + dest: '{{systemd_network_dir}}/99-serverctl-wg0.netdev' owner: root group: systemd-network mode: 0640 @@ -109,7 +75,7 @@ - name: Setup wg0 network template: src: 'systemd.network' - dest: "{{systemd_network_dir}}/99-wg0.network" + dest: "{{systemd_network_dir}}/99-serverctl-wg0.network" owner: root group: systemd-network mode: 0640 diff --git a/infrastructure/create-resources/ansible/roles/wireguard/mesh/templates/systemd.netdev b/infrastructure/create-resources/ansible/roles/wireguard/mesh/templates/systemd.netdev index 6b44b83..5113d0d 100644 --- a/infrastructure/create-resources/ansible/roles/wireguard/mesh/templates/systemd.netdev +++ b/infrastructure/create-resources/ansible/roles/wireguard/mesh/templates/systemd.netdev @@ -1,20 +1,22 @@ [NetDev] -Name=wg0 +Name=serverctl-wg0 Kind=wireguard -Description=WireGuard tunnel wg0 +Description=WireGuard tunnel serverctl-wg0 [WireGuard] ListenPort={{ wireguard_port }} PrivateKey={{ wireguard_private_key.stdout }} -{% for peer in groups['serverctl_cluster'] %} +{% for peer in groups['serverctl_super_cluster'] %} {% if peer != inventory_hostname %} [WireGuardPeer] PublicKey={{ hostvars[peer].wireguard_public_key.stdout }} PresharedKey={{ wireguard_preshared_keys[peer] if inventory_hostname < peer else hostvars[peer].wireguard_preshared_keys[inventory_hostname] }} AllowedIPs={{ hostvars[peer].wireguard_ip }}/32 +{% if not hostvars[peer].client_server %} Endpoint={{ hostvars[peer].ansible_host }}:{{ wireguard_port }} PersistentKeepalive=25 {% endif %} +{% endif %} {% endfor %} \ No newline at end of file diff --git a/infrastructure/create-resources/ansible/roles/wireguard/mesh/templates/systemd.network b/infrastructure/create-resources/ansible/roles/wireguard/mesh/templates/systemd.network index c282bf9..cef1219 100644 --- a/infrastructure/create-resources/ansible/roles/wireguard/mesh/templates/systemd.network +++ b/infrastructure/create-resources/ansible/roles/wireguard/mesh/templates/systemd.network @@ -1,5 +1,5 @@ [Match] -Name=wg0 +Name=serverctl-wg0 [Network] Address={{ wireguard_ip }}/{{ wireguard_mask_bits }} \ No newline at end of file diff --git a/infrastructure/create-resources/ansible/site.yml b/infrastructure/create-resources/ansible/site.yml index 1281058..a2a95d8 100644 --- a/infrastructure/create-resources/ansible/site.yml +++ b/infrastructure/create-resources/ansible/site.yml @@ -5,15 +5,21 @@ roles: - role: prereq - role: download - - role: './wireguard/mesh' + - role: firewall + +- hosts: serverctl_super_cluster + gather_facts: yes + become: yes + roles: + - role: wireguard/mesh - hosts: serverctl_master_hosts become: yes roles: - role: "./k3s/master" -#- hosts: serverctl_node_hosts -# become: yes -# roles: -# - role: "./k3s/node" -# +- hosts: serverctl_node_hosts + become: yes + roles: + - role: "./k3s/node" + diff --git a/infrastructure/create-resources/templates/hosts.tftpl b/infrastructure/create-resources/templates/hosts.tftpl index ddbe71f..52a2859 100644 --- a/infrastructure/create-resources/templates/hosts.tftpl +++ b/infrastructure/create-resources/templates/hosts.tftpl @@ -1,18 +1,35 @@ [serverctl_master_hosts] %{ for ip in serverctl_masters ~} -${ip} ansible_host=${ip} wireguard_ip=${cidrhost("192.168.0.0/24", index(serverctl_masters, ip) + 1)} +${ip} ansible_host=${ip} wireguard_ip=${cidrhost("10.1.1.0/24", index(serverctl_masters, ip) + 1)} %{ endfor ~} [serverctl_node_hosts] %{ for ip in serverctl_nodes ~} -${ip} ansible_host=${ip} wireguard_ip=${cidrhost("192.168.1.0/24", index(serverctl_nodes, ip) + 1)} +${ip} ansible_host=${ip} wireguard_ip=${cidrhost("10.1.1.0/24", index(serverctl_nodes, ip) + 10)} %{ endfor ~} +[serverctl_home_servers] +192.168.1.150 ansible_host=192.168.1.150 wireguard_ip=10.1.1.8 +#192.168.1.233 ansible_host=192.168.1.233 wireguard_ip=10.1.1.9 + [serverctl_cluster:children] serverctl_master_hosts serverctl_node_hosts -[serverctl_cluster:vars] +[serverctl_super_cluster:children] +serverctl_cluster +serverctl_home_servers + +[serverctl_home_servers:vars] +client_server=True + +[serverctl_super_cluster:vars] +pipelining=true +ansible_ssh_user=root +ansible_ssh_port=22 + +[serverctl_cluster:vars] +client_server=False pipelining=true ansible_ssh_user=root ansible_ssh_port=22