[feature/operators]: Improve Ansible UX, Nginx indempotency and error handling (#6310)

* make wireguard enabled flag bulletproof

* correct firewall setting

* add nginx handler

* make systemd template case sensitive

* twek nginx and ssl template

* finalize nginx and certbot configs

* add nginx purge command

* fix typo

* add removing vm guide

ansible/nym-node/playbooks/group_vars/all.yml

@@ -2,7 +2,7 @@
 ansible_ssh_private_key_file: ~/.ssh/<SSH_KEY>
 
 # nym_version: "v2025.21-mozzarella"
-# 
+#
 # NOTE:
 # if you want to pin Nym to a specific version instead of using the
 # latest release from GitHub in /tasks/main.yml then
@@ -13,17 +13,17 @@ tunnel_manager_url: "https://github.com/nymtech/nym/raw/refs/heads/develop/scrip
 quic_bridge_deployment_url: "https://raw.githubusercontent.com/nymtech/nym/refs/heads/develop/scripts/nym-node-setup/quic_bridge_deployment.sh"
 
 # NOTE: These values will be used globally unless overwritten per node in inventory/all
-ansible_user: root                       # used for ssh, like `ssh root@nym-exit.ch-1.mynodes.net`
-email: "<EMAIL>"                         # used in certbot, description.toml and landing page
-website: "<WEBSITE>"                     # it is used in the description.toml
-description: "<NODE_PUBLIC_DESCRIPTION>" # or define per node in inventory/all
+ansible_user: root                         # used for ssh, like `ssh root@nym-exit.ch-1.mynodes.net`
+email: "<EMAIL>"                           # used in certbot, description.toml and landing page
+website: "<WEBSITE>"                       # it is used in the description.toml
+description: "<NODE_PUBLIC_DESCRIPTION>"   # or define per node in inventory/all
 
 # NOTE: Set these vars if you want them globally for all nodes
 # Per node changes in inventory/all will overwrite these global ones:
-hostname: ""            # this is a fallback, keep it and setup hostname per node in inventory/all
-# moniker: "<MONIKER>"                    # if not setup here not in inventory/all it get's derived from the hostname
-# mode: <MODE>                            # entry-gateway/exit-gateway/mixnode 
-# wireguard_enabled: <WIREGUARD_ENABLED>  # true/false
+hostname: ""                                # this is a fallback, keep it and setup hostname per node in inventory/all
+# moniker: "<MONIKER>"                      # if not setup here not in inventory/all it get's derived from the hostname
+# mode: <MODE>                              # entry-gateway/exit-gateway/mixnode
+# wireguard_enabled: <WIREGUARD_ENABLED>    # true/false
 
 # NOTE: Possible vars to incule on landing page, etc.
 # operator_name: "<OPERATOR_NAME>"
@@ -41,4 +41,4 @@ packages:
   - ca-certificates
   - jq
   - wget
-  - ufw
+  - ufw

ansible/nym-node/roles/base/tasks/main.yml

@@ -1,9 +1,10 @@
+---
 - name: Set hostname
   hostname:
     name: "{{ hostname }}"
   when: hostname is defined and hostname | length > 0
 
-- name: Install aptitude 
+- name: Install aptitude
   apt:
     name: aptitude
     update_cache: yes
@@ -14,9 +15,9 @@
   apt:
     update_cache: yes
     upgrade: yes
-    
+
 - name: Install essential packages
   package:
     name: "{{ packages }}"
     state: latest
-    update_cache: yes
+    update_cache: yes

ansible/nym-node/roles/nginx/handlers/main.yml

@@ -0,0 +1,10 @@
+---
+- name: Reload nginx
+  service:
+    name: nginx
+    state: reloaded
+
+- name: Restart nginx
+  service:
+    name: nginx
+    state: restarted

ansible/nym-node/roles/nginx/tasks/main.yml

@@ -1,3 +1,4 @@
+---
 - name: Install nginx and certbot
   apt:
     name:
@@ -5,57 +6,168 @@
       - certbot
       - python3-certbot-nginx
     state: present
+    update_cache: yes
 
-- name: Create web root directory
+- name: Ensure nginx snippets directory exists
+  file:
+    path: /etc/nginx/snippets
+    state: directory
+    mode: "0755"
+
+# own SSL defaults - don't rely on certbot files
+- name: Install Nym SSL options snippet
+  copy:
+    dest: /etc/nginx/snippets/nym-ssl-options.conf
+    mode: "0644"
+    content: |
+      ssl_session_cache shared:NYMSSL:10m;
+      ssl_session_timeout 1d;
+      ssl_session_tickets off;
+
+      ssl_protocols TLSv1.2 TLSv1.3;
+      ssl_prefer_server_ciphers off;
+
+      # Reasonable modern cipher set (works across Ubuntu nginx builds)
+      ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305";
+
+      # OCSP stapling is nice but can break if resolver isn't set; keep minimal here.
+  notify: Restart nginx
+
+- name: Ensure web root directory exists
   file:
     path: "/var/www/{{ hostname }}"
     state: directory
     mode: "0755"
 
-- name: Create landing page template
-  tags: landing
+- name: Deploy landing page
   template:
     src: landing.html.j2
     dest: "/var/www/{{ hostname }}/index.html"
+    mode: "0644"
+  notify: Restart nginx
 
-- name: Remove default nginx site
+# remove default site - safe on fresh + redeploy
+- name: Disable default nginx site symlink
   file:
     path: /etc/nginx/sites-enabled/default
     state: absent
+  notify: Restart nginx
 
-- name: Add bare-bones nginx template
+- name: Remove default nginx site definition if present
+  file:
+    path: /etc/nginx/sites-available/default
+    state: absent
+  notify: Restart nginx
+
+# always deploy/enable HTTP vhost
+- name: Deploy HTTP vhost
   template:
     src: nginx-site.conf.j2
     dest: "/etc/nginx/sites-available/{{ hostname }}"
+    mode: "0644"
+  notify: Restart nginx
 
-- name: Enable nginx config
+- name: Enable HTTP vhost (force correct symlink)
   file:
     src: "/etc/nginx/sites-available/{{ hostname }}"
     dest: "/etc/nginx/sites-enabled/{{ hostname }}"
     state: link
+    force: true
+  notify: Restart nginx
 
-- name: Validate nginx configuration
+# detect if cert exists already
+- name: Check whether certificate exists
+  stat:
+    path: "/etc/letsencrypt/live/{{ hostname }}/fullchain.pem"
+  register: le_cert
+
+# if cert does NOT exist yet, ensure SSL/WSS are NOT enabled
+- name: Ensure SSL and WSS vhosts are disabled until cert exists
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - "/etc/nginx/sites-enabled/{{ hostname }}-ssl"
+    - "/etc/nginx/sites-enabled/nym-wss-config"
+  when: not le_cert.stat.exists
+  notify: Restart nginx
+
+- name: Ensure nginx is enabled and running (needed for ACME http-01)
+  service:
+    name: nginx
+    state: started
+    enabled: yes
+
+- name: Validate nginx configuration (HTTP stage)
   command: nginx -t
   changed_when: false
 
-- name: Obtain SSL certificate
-  command:
-    cmd: "certbot --nginx --non-interactive --agree-tos --redirect -m {{ email }} -d {{ hostname }}"
+- name: Flush handlers (ensure HTTP is active before certbot)
+  meta: flush_handlers
 
-- name: Add wss config from nginx template
+# certbot strategy:
+# - if cert exists: webroot - doesn't touch nginx
+# - else:  --nginx works first-time; may touch nginx
+- name: Obtain/renew certificate
+  command:
+    cmd: >-
+      {% if le_cert.stat.exists %}
+      certbot certonly --webroot
+      -w /var/www/{{ hostname }}
+      --non-interactive --agree-tos --keep-until-expiring
+      -m {{ email }} -d {{ hostname }}
+      {% else %}
+      certbot --nginx
+      --non-interactive --agree-tos --redirect
+      -m {{ email }} -d {{ hostname }}
+      {% endif %}
+  register: certbot_result
+  failed_when: false
+
+# re-check cert after certbot attempt
+- name: Re-check whether certificate exists after certbot
+  stat:
+    path: "/etc/letsencrypt/live/{{ hostname }}/fullchain.pem"
+  register: le_cert_after
+
+# only deploy/enable SSL & WSS if cert exists
+- name: Deploy HTTPS vhost for {{ hostname }}
+  template:
+    src: nginx-site-ssl.conf.j2
+    dest: "/etc/nginx/sites-available/{{ hostname }}-ssl"
+    mode: "0644"
+  when: le_cert_after.stat.exists
+  notify: Restart nginx
+
+- name: Enable HTTPS vhost (force correct symlink)
+  file:
+    src: "/etc/nginx/sites-available/{{ hostname }}-ssl"
+    dest: "/etc/nginx/sites-enabled/{{ hostname }}-ssl"
+    state: link
+    force: true
+  when: le_cert_after.stat.exists
+  notify: Restart nginx
+
+- name: Deploy WSS vhost
   template:
     src: wss-config.conf.j2
     dest: "/etc/nginx/sites-available/nym-wss-config"
+    mode: "0644"
+  when: le_cert_after.stat.exists
+  notify: Restart nginx
 
-- name: Enable WSS config
+- name: Enable WSS vhost (force correct symlink)
   file:
     src: "/etc/nginx/sites-available/nym-wss-config"
     dest: "/etc/nginx/sites-enabled/nym-wss-config"
     state: link
+    force: true
+  when: le_cert_after.stat.exists
+  notify: Restart nginx
 
-- name: Validate nginx config after wss
+- name: Validate nginx configuration (final)
   command: nginx -t
   changed_when: false
 
-- name: Restart nginx to apply changes
-  service: name=nginx state=restarted enabled=yes
+- name: Flush handlers (apply restart after successful tests)
+  meta: flush_handlers

ansible/nym-node/roles/nginx/tasks/templates/nginx-site-ssl.conf.j2

@@ -0,0 +1,17 @@
+server {
+    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
+
+    server_name {{ hostname }};
+
+    ssl_certificate     /etc/letsencrypt/live/{{ hostname }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ hostname }}/privkey.pem;
+    include /etc/nginx/snippets/nym-ssl-options.conf;
+
+    location / {
+        proxy_pass http://127.0.0.1:8080;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+}

ansible/nym-node/roles/nginx/tasks/templates/nginx-site.conf.j2

@@ -4,10 +4,15 @@ server {
 
     server_name {{ hostname }};
 
-    location / {
-        proxy_pass http://127.0.0.1:8080;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header Host $host;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    root /var/www/{{ hostname }};
+    index index.html;
+
+    location ^~ /.well-known/acme-challenge/ {
+        default_type "text/plain";
+        try_files $uri =404;
     }
-}
+
+    location / {
+        return 301 https://$host$request_uri;
+    }
+}

ansible/nym-node/roles/nginx/tasks/templates/wss-config.conf.j2

@@ -4,10 +4,9 @@ server {
 
     server_name {{ hostname }};
 
-    ssl_certificate /etc/letsencrypt/live/{{ hostname }}/fullchain.pem;
+    ssl_certificate     /etc/letsencrypt/live/{{ hostname }}/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/{{ hostname }}/privkey.pem;
-    include /etc/letsencrypt/options-ssl-nginx.conf;
-    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
+    include /etc/nginx/snippets/nym-ssl-options.conf;
 
     access_log /var/log/nginx/access.log;
     error_log  /var/log/nginx/error.log;

ansible/nym-node/roles/nym/defaults/main.yml

@@ -6,10 +6,6 @@ nym_install_dir: /root/nym-binaries
 http_bind_address: "0.0.0.0:8080"        # maps to --http-bind-address
 mixnet_bind_address: "0.0.0.0:1789"      # maps to --mixnet-bind-address
 
-
-# WireGuard boolean
-wireguard_enabled: "{{ wireguard_enabled | default(false) | bool }}"
-
 # Landing page base dir, hostname is appended in the task
 landing_page_assets_base_dir: "/var/www"
 
@@ -37,4 +33,4 @@ nym_ufw_rules:
   - { port: 8080,  proto: tcp }
   - { port: 9000,  proto: tcp }
   - { port: 9001,  proto: tcp }
-  - { port: 51822, proto: udp }
+  - { port: 51822, proto: udp }

ansible/nym-node/roles/nym/handlers/main.yml

@@ -1,3 +1,4 @@
+---
 - name: Reload systemd
   systemd:
     daemon_reload: yes

ansible/nym-node/roles/nym/tasks/config.yml

@@ -1,5 +1,5 @@
 ---
-# Useful when the host is behind a NAT
+# useful when the host is behind a NAT
 - name: Fetch the public IP address
   command: "curl -4 canhazip.com"
   register: ipv4
@@ -11,7 +11,7 @@
     public_ip: "{{ ipv4.stdout | default(ansible_default_ipv4.address) }}"
 
 - name: Initialize nym node
-  # Delete the part from --hostname onward if you run  mode=mixnode only
+  # delete the part from --hostname onward if you run  mode=mixnode only
   command:
     cmd: >
       {{ nym_install_dir }}/nym-node run
@@ -25,7 +25,7 @@
       {{ nym_extra_flags }}
 
       --hostname {{ hostname }}
-      --wireguard-enabled {{ wireguard_enabled }}
+      --wireguard-enabled {{ (wireguard_enabled | default('false') | bool) | ternary('true','false') }}
       --landing-page-assets-path {{ landing_page_assets_base_dir }}/{{ hostname }}/
       {% if nym_write_flag %}-w{% endif %}
       {% if nym_init_only_flag %}--init-only{% endif %}

ansible/nym-node/roles/nym/tasks/firewall.yml

@@ -1,3 +1,12 @@
+---
+- name: Ensure UFW is installed
+  apt:
+    name: ufw
+    state: present
+    update_cache: yes
+  when: nym_ufw_enable
+
+
 - name: Configure UFW rules
   ufw:
     rule: allow
@@ -14,9 +23,10 @@
 - name: Allow bandwidth/topup rule inside WG tunnel
   command: >
     ufw allow in on nymwg to any port 51830 proto tcp comment 'bandwidth queries/topup'
+  changed_when: false
   when:
     - nym_ufw_enable
-    - (wireguard_enabled | bool)
+    - (wireguard_enabled | default(false) | bool)
 
 - name: Enable UFW
   ufw:

ansible/nym-node/roles/nym/templates/nym-node.service.j2

@@ -6,10 +6,10 @@ StartLimitBurst=10
 [Service]
 User={{ ansible_user }}
 LimitNOFILE=65536
-ExecStart=/root/nym-binaries/nym-node run --mode {{ mode }} --accept-operator-terms-and-conditions --wireguard-enabled {{ wireguard_enabled }}
+ExecStart=/root/nym-binaries/nym-node run --mode {{ mode }} --accept-operator-terms-and-conditions --wireguard-enabled {{ (wireguard_enabled | default(false) | bool) | ternary('true','false') }}
 KillSignal=SIGINT
 Restart=on-failure
 RestartSec=30
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target

ansible/nym-node/roles/tunnel/tasks/main.yml

@@ -1,14 +1,11 @@
-- name: Download network-tunnel-manager.sh
-  tags: network tunnel manager
-  get_url:
-    url: "{{ tunnel_manager_url }}"
-    dest: "/root/nym-binaries/network-tunnel-manager.sh"
-    mode: "0755"
-
+---
 - name: Configure tunnel manager
-  tags: network tunnel manager
+  tags:
+    - network_tunnel_manager
   become: true
   command:
     cmd: "/root/nym-binaries/network-tunnel-manager.sh {{ item }}"
   loop:
-    - complete_networking_configuration
+    - complete_networking_configuration
+  register: tunnel_mgr
+  failed_when: false

ansible/nym-node/roles/upgrade/tasks/finalize.yml

@@ -116,7 +116,7 @@
   when: not ansible_check_mode and (upgrade_ok | default(false)) == false
 
 # optional: hard-fail the play for CI environments
-#- name: Fail the play to signal upgrade failure
+#- name: fail the play to signal upgrade failure
 #  fail:
 #    msg: "nym-node upgrade failed; rolled back to previous binary."
 #  when: not ansible_check_mode and (upgrade_ok | default(false)) == false

documentation/docs/pages/operators/nodes/nym-node/configuration/proxy-configuration.mdx

@@ -332,3 +332,13 @@ wscat -c wss://<HOSTNAME>:<WSS_PORT>
 ```
 
 - Check Swagger API of your node using the hostname: `https://<HOSTNAME>/api/v1/swagger/#/`
+
+## Troubleshooting
+
+In some cases Nginx may cache expired certificates, old configurations and other snippets creating confussion in a proper routing of your server. To purge this cache you can run:
+```sh
+apt purge nginx nginx-common
+apt install nginx
+service nginx reload && service nginx restart
+```
+This will pickup only the current configuration.

documentation/docs/pages/operators/nodes/preliminary-steps/vps-setup/advanced.mdx

@@ -803,3 +803,44 @@ ssh root@<IPv4> -i ~/.ssh/your_ssh_key
 </Steps>
 
 Now your VM is almost ready for `nym-node` [setup](../../nym-node/setup). Before you proceed, ssh in and [configure all prerequisities](../vps-setup#vps-configuration) needed for `nym-node` installation and operation.
+
+## Removing Virtual Machines
+
+If you setup your VM in a wrong way, or you simply don't use it anymore, you can remove it.
+
+<Callout type="warning" emoji="⚠️">
+**These commands will erase everything on the VM, make sure to backup everything you may need in the future bewfore executing this!**
+</Callout>
+
+<Steps>
+###### 1. SSH to the host server
+
+###### 2. List all VMs
+```sh
+virsh list --all
+```
+
+###### 3. Shut down and remove VM
+- To remove a VM run this sequence
+```sh
+# shutdown
+virsh shutdown <VM_NAME>
+sleep 10
+
+# destroy
+virsh destroy <VM_NAME>
+
+# undefine and purge storage
+virsh undefine <VM_NAME> --remove-all-storage
+
+# ensure the storage is deleted
+rm /var/lib/libvirt/images/<VM_NAME>.img 
+```
+
+###### 4. List all VMs again
+
+- The list should not contain the VM that you just deleted:
+```sh
+virsh list --all
+```
+</ Steps>