From d32b680351f32a16fb1f9b658378fe67758cc1de Mon Sep 17 00:00:00 2001 From: import this <97586125+serinko@users.noreply.github.com> Date: Tue, 10 Mar 2026 13:28:39 +0000 Subject: [PATCH] Server Ansible maintenance & documentation (#6514) * Create ansible playbook for trimming and rotationg logs * add docs for triming and log rotation * update ansible docs * add info on logic * cleanup the cleanup guide * update scraped stats * ready for review * address review --- ansible/nym-node/playbooks/group_vars/all.yml | 102 ++++++- .../nym-node/playbooks/system-maintenance.yml | 38 +++ .../nym-node/roles/apt_cleanup/tasks/main.yml | 21 ++ .../roles/classic_log_cleanup/tasks/main.yml | 20 ++ .../roles/fstrim_15min/defaults/main.yml | 3 + .../roles/fstrim_15min/tasks/main.yml | 31 ++ .../roles/journal_vacuum/defaults/main.yml | 3 + .../roles/journal_vacuum/tasks/main.yml | 6 + .../roles/journald_limits/defaults/main.yml | 0 .../roles/journald_limits/handlers/main.yml | 5 + .../roles/journald_limits/tasks/main.yml | 20 ++ .../roles/nymnode_logging/defaults/main.yml | 7 + .../roles/nymnode_logging/tasks/main.yml | 26 ++ ansible/nym-node/roles/report/tasks/main.yml | 8 + .../roles/rsyslog_disable/tasks/main.yml | 13 + .../roles/snap_cleanup/tasks/main.yml | 10 + .../operators/snippets/prune-logs-vm.mdx | 283 ++++++++++++++++++ .../operators/snippets/prune-logs-vps.mdx | 142 +++++++++ .../circulating-supply.json | 4 +- .../nyx-outputs/circulating-supply.md | 2 +- .../nyx-outputs/epoch-reward-budget.md | 2 +- .../nyx-outputs/stake-saturation.md | 2 +- .../nyx-outputs/staking-target.md | 2 +- .../nyx-outputs/staking_supply.md | 2 +- .../nyx-outputs/token-table.md | 8 +- .../api-scraping-outputs/reward-params.json | 8 +- .../outputs/api-scraping-outputs/time-now.md | 2 +- .../command-outputs/node-api-check-help.md | 2 +- .../node-api-check-query-help.md | 4 +- .../nym-node-cli-install-help.md | 3 +- .../outputs/nyx-outputs/token-table.md | 2 +- .../pages/operators/orchestration/ansible.mdx | 21 ++ .../operators/troubleshooting/vps-isp.mdx | 83 ++--- .../scripts/api-scraping/api_targets.py | 2 +- 34 files changed, 801 insertions(+), 86 deletions(-) create mode 100644 ansible/nym-node/playbooks/system-maintenance.yml create mode 100644 ansible/nym-node/roles/apt_cleanup/tasks/main.yml create mode 100644 ansible/nym-node/roles/classic_log_cleanup/tasks/main.yml create mode 100644 ansible/nym-node/roles/fstrim_15min/defaults/main.yml create mode 100644 ansible/nym-node/roles/fstrim_15min/tasks/main.yml create mode 100644 ansible/nym-node/roles/journal_vacuum/defaults/main.yml create mode 100644 ansible/nym-node/roles/journal_vacuum/tasks/main.yml create mode 100644 ansible/nym-node/roles/journald_limits/defaults/main.yml create mode 100644 ansible/nym-node/roles/journald_limits/handlers/main.yml create mode 100644 ansible/nym-node/roles/journald_limits/tasks/main.yml create mode 100644 ansible/nym-node/roles/nymnode_logging/defaults/main.yml create mode 100644 ansible/nym-node/roles/nymnode_logging/tasks/main.yml create mode 100644 ansible/nym-node/roles/report/tasks/main.yml create mode 100644 ansible/nym-node/roles/rsyslog_disable/tasks/main.yml create mode 100644 ansible/nym-node/roles/snap_cleanup/tasks/main.yml create mode 100644 documentation/docs/components/operators/snippets/prune-logs-vm.mdx create mode 100644 documentation/docs/components/operators/snippets/prune-logs-vps.mdx diff --git a/ansible/nym-node/playbooks/group_vars/all.yml b/ansible/nym-node/playbooks/group_vars/all.yml index d48861c05a..e62136b2b9 100644 --- a/ansible/nym-node/playbooks/group_vars/all.yml +++ b/ansible/nym-node/playbooks/group_vars/all.yml @@ -1,32 +1,38 @@ --- ansible_ssh_private_key_file: ~/.ssh/ -# nym_version: "v2025.21-mozzarella" -# -# NOTE: -# if you want to pin Nym to a specific version instead of using the -# latest release from GitHub in /tasks/main.yml then -# uncomment the line above and set the tag - cli_url: "https://github.com/nymtech/nym/releases/download/nym-binaries-{{ nym_version }}/nym-cli" tunnel_manager_url: "https://github.com/nymtech/nym/raw/refs/heads/develop/scripts/nym-node-setup/network-tunnel-manager.sh" quic_bridge_deployment_url: "https://raw.githubusercontent.com/nymtech/nym/refs/heads/develop/scripts/nym-node-setup/quic_bridge_deployment.sh" -# NOTE: These values will be used globally unless overwritten per node in inventory/all +############################################################################### +## GLOBAL VARS +## These values will be used globally unless overwritten per node in inventory/all +############################################################################### + ansible_user: root # used for ssh, like `ssh root@nym-exit.ch-1.mynodes.net` email: "" # used in certbot, description.toml and landing page website: "" # it is used in the description.toml description: "" # or define per node in inventory/all +# operator_name: "" # used in landing page if provided + +############################################################################### +## GLOBAL VARS +## These values will be used globally unless overwritten per node in inventory/all +## Set these vars only if you want them globally for all nodes +## Per node changes in inventory/all will overwrite these global vars +############################################################################### -# NOTE: Set these vars if you want them globally for all nodes -# Per node changes in inventory/all will overwrite these global ones: -hostname: "" # this is a fallback, keep it and setup hostname per node in inventory/all # moniker: "" # if not setup here not in inventory/all it get's derived from the hostname # mode: # entry-gateway/exit-gateway/mixnode # wireguard_enabled: # true/false +hostname: "" # this is a fallback, keep it and setup hostname per node in inventory/all + +############################################################################### +## GLOBAL PACKAGES +## These will be installed during deployment +############################################################################### -# NOTE: Possible vars to incule on landing page, etc. -# operator_name: "" packages: - tmux @@ -42,3 +48,73 @@ packages: - jq - wget - ufw + + +############################################################################### +## OPTIONAL OVERRIDES +## All values below already have defaults in the playbook/roles +## Uncomment only if you want to override them +############################################################################### + +############################################################################### +## SYSTEM MAINTENANCE PLAYBOOK KNOBS +############################################################################### + +# nym_version: "v2025.21-mozzarella" + +## NOTE: +## if you want to pin Nym to a specific version instead of using the +## latest release from GitHub in /tasks/main.yml then +## uncomment the line above and set the tag + +############################################################################### +## SYSTEM MAINTENANCE PLAYBOOK KNOBS +############################################################################### + +## JOURNALD LIMITS + +# journald_system_max_use: "100M" # max persistent journal size +# journald_runtime_max_use: "50M" # max runtime journal size +# journald_system_max_file_size: "25M" # max single journal file +# journald_runtime_max_file_size: "10M" # max runtime journal file +# journald_max_retention_sec: "3day" # retention time + +# journald_rate_limit_interval: "30s" # rate limit window +# journald_rate_limit_burst: "1000" # rate limit burst + + +## NYM-NODE LOG CONTROL + +# nymnode_log_level_max: "warning" # drop INFO logs +# nymnode_rate_limit_interval: "30s" # per nym-node rate limit window +# nymnode_rate_limit_burst: "200" # per nym-node rate limit burst + + +## JOURNAL VACUUM TARGETS + +# journal_vacuum_size: "100M" +# journal_vacuum_time: "3days" + + +## RSYSLOG + +# disable_rsyslog: true + + +## FSTRIM SCHEDULE + +# fstrim_every_calendar: "*:0/15" # Aggressive +# fstrim_every_calendar: "hourly" # Less aggressive + + +## OPTIONAL CLEANUPS + +# enable_apt_cleanup: true +# enable_snap_cleanup: true + + +## WRITEBACK TUNING + +# enable_writeback_tuning: true +# writeback_dirty_writeback_centisecs: 1500 +# writeback_dirty_expire_centisecs: 6000 \ No newline at end of file diff --git a/ansible/nym-node/playbooks/system-maintenance.yml b/ansible/nym-node/playbooks/system-maintenance.yml new file mode 100644 index 0000000000..6fd924d2ef --- /dev/null +++ b/ansible/nym-node/playbooks/system-maintenance.yml @@ -0,0 +1,38 @@ +--- +- name: Restrict logging, vacuum journals, and enable periodic trim + hosts: all + become: true + gather_facts: false + + # global knobs - override in inventory/group_vars/host_vars as needed + vars: + journald_system_max_use: "100M" + journald_runtime_max_use: "50M" + journald_system_max_file_size: "25M" + journald_runtime_max_file_size: "10M" + journald_max_retention_sec: "3day" + journald_rate_limit_interval: "30s" + journald_rate_limit_burst: "1000" + + # per nym-node rate limit + level cap + nymnode_log_level_max: "warning" + nymnode_rate_limit_interval: "30s" + nymnode_rate_limit_burst: "200" + + # journal vacuum targets + journal_vacuum_size: "100M" + journal_vacuum_time: "3days" + + # fstrim cadence (note: the systemd override uses cron-like calendar) + fstrim_every_calendar: "*:0/15" + + roles: + - role: journald_limits + - role: nymnode_logging + - role: rsyslog_disable + - role: journal_vacuum + - role: classic_log_cleanup + - role: apt_cleanup + - role: snap_cleanup + - role: fstrim_15min + - role: report \ No newline at end of file diff --git a/ansible/nym-node/roles/apt_cleanup/tasks/main.yml b/ansible/nym-node/roles/apt_cleanup/tasks/main.yml new file mode 100644 index 0000000000..c8d69cba0d --- /dev/null +++ b/ansible/nym-node/roles/apt_cleanup/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Clean apt cache + command: apt-get clean + ignore_errors: true + +- name: Autoremove unused packages + command: apt-get -y autoremove + ignore_errors: true + +- name: Remove apt lists to reclaim space (they will be re-fetched on update) + file: + path: /var/lib/apt/lists + state: absent + ignore_errors: true + +- name: Recreate apt lists directory + file: + path: /var/lib/apt/lists + state: directory + mode: "0755" + ignore_errors: true \ No newline at end of file diff --git a/ansible/nym-node/roles/classic_log_cleanup/tasks/main.yml b/ansible/nym-node/roles/classic_log_cleanup/tasks/main.yml new file mode 100644 index 0000000000..991a9a6f4a --- /dev/null +++ b/ansible/nym-node/roles/classic_log_cleanup/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Remove classic /var/log files if present (optional) + file: + path: "{{ item }}" + state: absent + loop: + - /var/log/syslog + - /var/log/syslog.1 + - /var/log/kern.log + - /var/log/kern.log.1 + - /var/log/auth.log + - /var/log/auth.log.1 + - /var/log/ufw.log + - /var/log/ufw.log.1 + ignore_errors: true + +# This is best-effort and may still fail if other packages' postrotate scripts assume services exist. +- name: Force logrotate (best-effort) + command: "logrotate --force /etc/logrotate.conf" + ignore_errors: true \ No newline at end of file diff --git a/ansible/nym-node/roles/fstrim_15min/defaults/main.yml b/ansible/nym-node/roles/fstrim_15min/defaults/main.yml new file mode 100644 index 0000000000..954c2eea96 --- /dev/null +++ b/ansible/nym-node/roles/fstrim_15min/defaults/main.yml @@ -0,0 +1,3 @@ +--- +fstrim_timer_dropin_dir: "/etc/systemd/system/fstrim.timer.d" +fstrim_every_calendar: "*:0/15" \ No newline at end of file diff --git a/ansible/nym-node/roles/fstrim_15min/tasks/main.yml b/ansible/nym-node/roles/fstrim_15min/tasks/main.yml new file mode 100644 index 0000000000..9741af68e5 --- /dev/null +++ b/ansible/nym-node/roles/fstrim_15min/tasks/main.yml @@ -0,0 +1,31 @@ +--- +- name: Ensure systemd drop-in dir for fstrim.timer exists + file: + path: "{{ fstrim_timer_dropin_dir }}" + state: directory + mode: "0755" + +- name: Override fstrim.timer schedule + copy: + dest: "{{ fstrim_timer_dropin_dir }}/override.conf" + mode: "0644" + content: | + [Timer] + OnCalendar= + OnCalendar={{ fstrim_every_calendar }} + Persistent=true + RandomizedDelaySec=0 + +- name: Reload systemd after fstrim override + systemd: + daemon_reload: true + +- name: Enable and start fstrim timer + systemd: + name: fstrim.timer + enabled: true + state: started + +- name: Run fstrim now (best-effort) + command: fstrim -av + ignore_errors: true \ No newline at end of file diff --git a/ansible/nym-node/roles/journal_vacuum/defaults/main.yml b/ansible/nym-node/roles/journal_vacuum/defaults/main.yml new file mode 100644 index 0000000000..1272192a2d --- /dev/null +++ b/ansible/nym-node/roles/journal_vacuum/defaults/main.yml @@ -0,0 +1,3 @@ +--- +journal_vacuum_size: "100M" +journal_vacuum_time: "3days" \ No newline at end of file diff --git a/ansible/nym-node/roles/journal_vacuum/tasks/main.yml b/ansible/nym-node/roles/journal_vacuum/tasks/main.yml new file mode 100644 index 0000000000..4919c30ae0 --- /dev/null +++ b/ansible/nym-node/roles/journal_vacuum/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: Vacuum journal to size cap (hard) + command: "journalctl --vacuum-size={{ journal_vacuum_size }}" + +- name: Vacuum journal older than retention window (time) + command: "journalctl --vacuum-time={{ journal_vacuum_time }}" \ No newline at end of file diff --git a/ansible/nym-node/roles/journald_limits/defaults/main.yml b/ansible/nym-node/roles/journald_limits/defaults/main.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ansible/nym-node/roles/journald_limits/handlers/main.yml b/ansible/nym-node/roles/journald_limits/handlers/main.yml new file mode 100644 index 0000000000..640fa27ac7 --- /dev/null +++ b/ansible/nym-node/roles/journald_limits/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart journald + systemd: + name: systemd-journald + state: restarted \ No newline at end of file diff --git a/ansible/nym-node/roles/journald_limits/tasks/main.yml b/ansible/nym-node/roles/journald_limits/tasks/main.yml new file mode 100644 index 0000000000..f7bb46717b --- /dev/null +++ b/ansible/nym-node/roles/journald_limits/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Configure journald limits (persistent, capped, rate-limited) + copy: + dest: /etc/systemd/journald.conf + mode: "0644" + content: | + [Journal] + Storage=persistent + Compress=yes + Seal=yes + + SystemMaxUse={{ journald_system_max_use }} + RuntimeMaxUse={{ journald_runtime_max_use }} + SystemMaxFileSize={{ journald_system_max_file_size }} + RuntimeMaxFileSize={{ journald_runtime_max_file_size }} + MaxRetentionSec={{ journald_max_retention_sec }} + + RateLimitIntervalSec={{ journald_rate_limit_interval }} + RateLimitBurst={{ journald_rate_limit_burst }} + notify: Restart journald \ No newline at end of file diff --git a/ansible/nym-node/roles/nymnode_logging/defaults/main.yml b/ansible/nym-node/roles/nymnode_logging/defaults/main.yml new file mode 100644 index 0000000000..19ae7e1fe2 --- /dev/null +++ b/ansible/nym-node/roles/nymnode_logging/defaults/main.yml @@ -0,0 +1,7 @@ +--- +nymnode_log_level_max: "warning" +nymnode_rate_limit_interval: "30s" +nymnode_rate_limit_burst: "200" +nymnode_unit_name: "nym-node" # set to "nym-node.service" if your distro expects it +nymnode_dropin_dir: "/etc/systemd/system/nym-node.service.d" +nymnode_dropin_file: "10-logging.conf" \ No newline at end of file diff --git a/ansible/nym-node/roles/nymnode_logging/tasks/main.yml b/ansible/nym-node/roles/nymnode_logging/tasks/main.yml new file mode 100644 index 0000000000..6c890d7302 --- /dev/null +++ b/ansible/nym-node/roles/nymnode_logging/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: Ensure systemd drop-in dir for nym-node exists + file: + path: "{{ nymnode_dropin_dir }}" + state: directory + mode: "0755" + +- name: Cap nym-node logs + apply per-unit rate limiting + copy: + dest: "{{ nymnode_dropin_dir }}/{{ nymnode_dropin_file }}" + mode: "0644" + content: | + [Service] + LogLevelMax={{ nymnode_log_level_max }} + LogRateLimitIntervalSec={{ nymnode_rate_limit_interval }} + LogRateLimitBurst={{ nymnode_rate_limit_burst }} + +- name: Reload systemd after nym-node drop-in + systemd: + daemon_reload: true + +- name: Restart nym-node to apply new logging limits (best-effort) + systemd: + name: "{{ nymnode_unit_name }}" + state: restarted + ignore_errors: true \ No newline at end of file diff --git a/ansible/nym-node/roles/report/tasks/main.yml b/ansible/nym-node/roles/report/tasks/main.yml new file mode 100644 index 0000000000..041c19329e --- /dev/null +++ b/ansible/nym-node/roles/report/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- name: Show journal disk usage + command: journalctl --disk-usage + register: journal_usage + changed_when: false + +- debug: + var: journal_usage.stdout \ No newline at end of file diff --git a/ansible/nym-node/roles/rsyslog_disable/tasks/main.yml b/ansible/nym-node/roles/rsyslog_disable/tasks/main.yml new file mode 100644 index 0000000000..d9783c59e4 --- /dev/null +++ b/ansible/nym-node/roles/rsyslog_disable/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: Stop/disable rsyslog if installed (best-effort) + systemd: + name: rsyslog + state: stopped + enabled: false + ignore_errors: true + +- name: Remove rsyslog logrotate stanza if present (prevents logrotate failures) + file: + path: /etc/logrotate.d/rsyslog + state: absent + ignore_errors: true \ No newline at end of file diff --git a/ansible/nym-node/roles/snap_cleanup/tasks/main.yml b/ansible/nym-node/roles/snap_cleanup/tasks/main.yml new file mode 100644 index 0000000000..a444e340c6 --- /dev/null +++ b/ansible/nym-node/roles/snap_cleanup/tasks/main.yml @@ -0,0 +1,10 @@ +--- +- name: Remove disabled snap revisions (best-effort) + shell: | + set -euo pipefail + snap list --all | awk '/disabled/{print $1, $3}' | while read -r name rev; do + snap remove "$name" --revision="$rev" || true + done + args: + executable: /bin/bash + ignore_errors: true \ No newline at end of file diff --git a/documentation/docs/components/operators/snippets/prune-logs-vm.mdx b/documentation/docs/components/operators/snippets/prune-logs-vm.mdx new file mode 100644 index 0000000000..b0d2e7812d --- /dev/null +++ b/documentation/docs/components/operators/snippets/prune-logs-vm.mdx @@ -0,0 +1,283 @@ +import { Steps } from 'nextra/components'; +import { Callout } from 'nextra/components'; + +### Self-Managed VMs, Hypervisor (Host) Cleanup & Ansible Automated Maintenance + +This chapter covers both: +- Guest VM configuration +- Host libvirt qcow2 behavior + +If you run [KVM/libvirt](/operators/nodes/preliminary-steps/vps-setup/advanced), then this chapter is an essential part of your server hygiene procedure. + +**Verify disk usage** + +You can always identify what is consuming disk space + +```sh +df -h +du -xhd1 /var | sort -h +du -xhd1 /var/log | sort -h +find /var -type f -exec ls -lh {} + 2>/dev/null | sort -k 5 -h | tail -n 30 +journalctl --disk-usage +``` + +#### Inside each VM + + +In case your VMs are paused on the hypervisor (the host server), with no disk space left, start with [freeing some space on the hypervisor](/operators/troubleshooting/vps-isp#on-the-hypervisor-host-machine) and then return to this chapter, when the VMs are running again. + + + +**This part can be automated via [Ansible](/operators/orchestration/ansible), using the playbook [`system-maintenance.yml`](/operators/orchestration/ansible#4-system-maintenance). If this is your way, then proceed with Ansible and skip directly to the part called [*On the Hypervisor (host machine)*](#on-the-hypervisor-host-machine).** + + + +###### 1. Ensure root FS is *not* mounted with continuous discard + +- This step is to enable `fstrim` to work in the next step and prevent two same logic processes fighting each other + +```sh +mount | grep " / " +``` + +- If you see: +```sh +type ext4 (... discard ...) +``` + +- Remove discard from `/etc/fstab` and `remount`: + +```sh +mount -o remount,nodiscard / +mount | grep " / " +``` + +Continuous discard can cause problematic allocation behavior. + + +###### 2. Enable frequent `fstrim` + +- This service will trim free space every 15 minutes + +```sh +mkdir -p /etc/systemd/system/fstrim.timer.d +``` + +```sh +cat >/etc/systemd/system/fstrim.timer.d/override.conf <<'EOF' +[Timer] +OnCalendar= +OnCalendar=*:0/15 +Persistent=true +RandomizedDelaySec=0 +EOF +``` + +- Reload and restart the daemon: +```sh +systemctl daemon-reload +systemctl enable --now fstrim.timer +fstrim -av +``` + +###### 3. Prune `journald` +- Trim logs by running these commands + +```sh +journalctl --vacuum-size=100M +journalctl --vacuum-time=3days +journalctl --disk-usage +``` +- Compare the outcome of the last one with the previous check + +###### 4. Make `journald` limits persistent + +- Create a new file `/etc/systemd/journald.conf.d/limits.conf` with these hard caps: +```sh +mkdir -p /etc/systemd/journald.conf.d + +cat >/etc/systemd/journald.conf.d/limits.conf <<'EOF' +[Journal] +Storage=persistent +Compress=yes +Seal=yes + +SystemMaxUse=100M +RuntimeMaxUse=50M +SystemMaxFileSize=25M +RuntimeMaxFileSize=10M +MaxRetentionSec=3day + +RateLimitIntervalSec=30s +RateLimitBurst=1000 +EOF + +systemctl restart systemd-journald +``` + +###### 5. Disable classic `rsyslog` + +Prevent duplicate logging as most Ubuntu VPS images run both `journald` and `rsyslog` by default. +That duplicates logs into `/var/log/syslog`. + +- Disable rsyslog by these commands: + +```sh +systemctl disable --now rsyslog.service || true +systemctl mask rsyslog.service || true +pgrep -x rsyslogd >/dev/null 2>&1 && pkill -9 -x rsyslogd || true +``` + +- Optional cleanup of `syslog`: + +```sh +rm -f /var/log/syslog /var/log/syslog.1 +rm -f /var/log/kern.log /var/log/kern.log.1 +rm -f /var/log/auth.log /var/log/auth.log.1 +rm -f /var/log/ufw.log /var/log/ufw.log.1 +``` +> Note: Running `logrotate --force /etc/logrotate.conf` may fail after disabling `rsyslog`. That is expected and safe. + +###### 6. Reduce `nym-node` log growth + +- Create a new dir for a service file controlling `nym-node` logs: + +```sh +mkdir -p /etc/systemd/system/nym-node.service.d +``` + +- Create the service file: +```sh +cat >/etc/systemd/system/nym-node.service.d/10-logging.conf <<'EOF' +[Service] +LogLevelMax=warning +LogRateLimitIntervalSec=30s +LogRateLimitBurst=300 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=nym-node +EOF + +systemctl daemon-reload +systemctl restart nym-node +``` + + + + +#### On the Hypervisor (host machine) + + +**Do *NOT* shrink the disk size with a `nym-node` on it - this would irreversibly corrupt your data! Instead follow the steps in this sub-chapter.** + + +**All the steps below are done from the host machine.** + + +###### 1. Check your VMs state + +- These two commands give you a quick overview of your VMs +```sh +virsh list --all +ls -lh /var/lib/libvirt/images +``` + +###### 2. Verify `qcow2` actual allocation vs virtual size + +- The allocation defined by creating VMs is not equal to the actual usage + +```sh +qemu-img info --force-share /var/lib/libvirt/images/.img + +# for example: +# qemu-img info --force-share /var/lib/libvirt/images/ubuntu1.img +``` + +- Look at: +``` +virtual size +file length +disk size +``` + +###### 3. Monitor actual host allocation growth + +- Watch how disk allocation changes in 5 minutes +- This provides you an easy calculation how much disk space is consumed in a day, week or month + +```sh +watch -n 300 'du -sh /var/lib/libvirt/images' +df -h +``` + +###### 4. Sparsify + +- If images grew previously, sparsify once +- Shutdown a VM first: +```sh +virsh shutdown + +# for example: +# virsh shutdown ubuntu1 +``` +- Then `sparsify`: + +```sh +virt-sparsify --in-place /var/lib/libvirt/images/.img + +# for example: +# virt-sparsify --in-place /var/lib/libvirt/images/ubuntu1.img +``` +- Finally re-start the machine +```sh +virsh start + +# for example: +# virsh start ubuntu1 +``` + +- Repeat per VM if necessary + +- If you want to sparsify all of them, you can run it in a loop +```sh +for vm in $(virsh list --all --name); do + virsh shutdown "$vm" + + while [ "$(virsh domstate "$vm")" != "shut off" ]; do + sleep 5 + done + + virt-sparsify --in-place /var/lib/libvirt/images/${vm}.img +done +``` + +- Then use same logic to start them all + +```sh +for vm in $(virsh list --all --name); do + virsh start "$vm" +done +``` +- Finally make sure that they are all running: +```sh +virsh list --all +``` + +###### 5. Ensure host filesystem supports hole punching + +```sh +df -T /var/lib/libvirt/images +stat -f -c %T /var/lib/libvirt/images +``` + +- The filesystem must support hole punching (fallocate) such as `ext4` or `xfs` + + +###### 6. Final validation + +- Finally check that how much space is on your host disk: +```sh +df -h +du -sh /var/lib/libvirt/images +``` + \ No newline at end of file diff --git a/documentation/docs/components/operators/snippets/prune-logs-vps.mdx b/documentation/docs/components/operators/snippets/prune-logs-vps.mdx new file mode 100644 index 0000000000..eb937275e1 --- /dev/null +++ b/documentation/docs/components/operators/snippets/prune-logs-vps.mdx @@ -0,0 +1,142 @@ +import { Steps } from 'nextra/components'; + +### VPS Disk Usage & Log Control Guide + +The steps below are for manual cleanup and space reclaim on a VPS hosting a `nym-node`. + + + +###### 1. Identify what is consuming disk space + +```sh +df -h +du -xhd1 /var | sort -h +du -xhd1 /var/log | sort -h +find /var -type f -exec ls -lh {} + 2>/dev/null | sort -k 5 -h | tail -n 30 +journalctl --disk-usage +``` + +###### 2. Prune `journald` +- Trim logs by running these commands + +```sh +journalctl --vacuum-size=100M +journalctl --vacuum-time=3days +journalctl --disk-usage +``` +- Compare the outcome of the last one with the previous check + +###### 3. Make `journald` limits persistent + +- Create a new file `/etc/systemd/journald.conf` with these hard caps: +```sh +mkdir -p /etc/systemd/journald.conf.d + +cat >/etc/systemd/journald.conf.d/limits.conf <<'EOF' +[Journal] +Storage=persistent +Compress=yes +Seal=yes + +SystemMaxUse=100M +RuntimeMaxUse=50M +SystemMaxFileSize=25M +RuntimeMaxFileSize=10M +MaxRetentionSec=3day + +RateLimitIntervalSec=30s +RateLimitBurst=1000 +EOF + +systemctl restart systemd-journald +``` + + +###### 4. Disable classic `rsyslog` + +Prevent duplicate logging as most Ubuntu VPS images run both `journald` and `rsyslog` by default. +That duplicates logs into `/var/log/syslog`. + +- Disable rsyslog by these commands: + +```sh +systemctl disable --now rsyslog.service || true +systemctl mask rsyslog.service || true +pgrep -x rsyslogd >/dev/null 2>&1 && pkill -9 -x rsyslogd || true +``` + +- Optional cleanup of `syslog`: + +```sh +rm -f /var/log/syslog /var/log/syslog.1 +rm -f /var/log/kern.log /var/log/kern.log.1 +rm -f /var/log/auth.log /var/log/auth.log.1 +rm -f /var/log/ufw.log /var/log/ufw.log.1 +``` +> Note: Running `logrotate --force /etc/logrotate.conf` may fail after disabling `rsyslog`. That is expected and safe. + +###### 5. Reduce `nym-node` log growth + +- Create a new dir for a service file controlling `nym-node` logs: + +```sh +mkdir -p /etc/systemd/system/nym-node.service.d +``` + +- Create the service file: +```sh +cat >/etc/systemd/system/nym-node.service.d/10-logging.conf <<'EOF' +[Service] +LogLevelMax=warning +LogRateLimitIntervalSec=30s +LogRateLimitBurst=300 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=nym-node +EOF + +systemctl daemon-reload +systemctl restart nym-node +``` + +###### 6. Optional: Tune kernel writeback + +- This config reduces burst disk pressure + +```sh +cat >/etc/sysctl.d/99-writeback-tuning.conf <<'EOF' +vm.dirty_writeback_centisecs=1500 +vm.dirty_expire_centisecs=6000 +EOF + +sysctl --system +``` + +###### 7. Enable periodic Trim + +- This reclaims the free space + +```sh +systemctl enable --now fstrim.timer +fstrim -av +``` + +###### 8. Reclaim apt cache space + +- APT cache can become heavy too, reclaim that space by these commands: + +```sh +apt-get clean +apt-get -y autoremove +rm -rf /var/lib/apt/lists/* +mkdir -p /var/lib/apt/lists +``` + +###### 9. Verify final state + +- Check disk space now after the cleanup +```sh +df -h +journalctl --disk-usage +``` + diff --git a/documentation/docs/components/outputs/api-scraping-outputs/circulating-supply.json b/documentation/docs/components/outputs/api-scraping-outputs/circulating-supply.json index 1d29b76edf..8d9dd66d11 100644 --- a/documentation/docs/components/outputs/api-scraping-outputs/circulating-supply.json +++ b/documentation/docs/components/outputs/api-scraping-outputs/circulating-supply.json @@ -5,7 +5,7 @@ }, "mixmining_reserve": { "denom": "unym", - "amount": "172542733168986" + "amount": "170550581010206" }, "vesting_tokens": { "denom": "unym", @@ -13,6 +13,6 @@ }, "circulating_supply": { "denom": "unym", - "amount": "827457266831014" + "amount": "829449418989794" } } diff --git a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/circulating-supply.md b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/circulating-supply.md index 3747ac3017..0aa2957362 100644 --- a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/circulating-supply.md +++ b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/circulating-supply.md @@ -1 +1 @@ -827_457_266 +829_449_418 diff --git a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/epoch-reward-budget.md b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/epoch-reward-budget.md index d38a8f0ca0..404ff7db78 100644 --- a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/epoch-reward-budget.md +++ b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/epoch-reward-budget.md @@ -1 +1 @@ -4_792 +4_737 diff --git a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/stake-saturation.md b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/stake-saturation.md index ed8f9dc9fe..aaff511975 100644 --- a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/stake-saturation.md +++ b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/stake-saturation.md @@ -1 +1 @@ -253_163 +253_773 diff --git a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/staking-target.md b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/staking-target.md index dc4c855ff0..ecf68907cd 100644 --- a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/staking-target.md +++ b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/staking-target.md @@ -1 +1 @@ -60_759_293 +60_905_575 diff --git a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/staking_supply.md b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/staking_supply.md index 762d44b4ee..711f7fff53 100644 --- a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/staking_supply.md +++ b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/staking_supply.md @@ -1 +1 @@ -60_759_292 +60_905_574 diff --git a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/token-table.md b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/token-table.md index deab11591f..61cdaa8e98 100644 --- a/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/token-table.md +++ b/documentation/docs/components/outputs/api-scraping-outputs/nyx-outputs/token-table.md @@ -1,7 +1,7 @@ | **Item** | **Description** | **Amount in NYM** | |:-------------------|:------------------------------------------------------|--------------------:| | Total Supply | Maximum amount of NYM token in existence | 1_000_000_000 | -| Mixmining Reserve | Tokens releasing for operators rewards | 172_542_733 | -| Vesting Tokens | Tokens locked outside of cicrulation for future claim | 0 | -| Circulating Supply | Amount of unlocked tokens | 827_457_266 | -| Stake Saturation | Optimal size of node self-bond + delegation | 253_163 | +| Mixmining Reserve | Tokens releasing for operators rewards | 170_550_581 | +| Vesting Tokens | Tokens locked outside of circulation for future claim | 0 | +| Circulating Supply | Amount of unlocked tokens | 829_449_418 | +| Stake Saturation | Optimal size of node self-bond + delegation | 253_773 | diff --git a/documentation/docs/components/outputs/api-scraping-outputs/reward-params.json b/documentation/docs/components/outputs/api-scraping-outputs/reward-params.json index 1092b27856..d6e66d1485 100644 --- a/documentation/docs/components/outputs/api-scraping-outputs/reward-params.json +++ b/documentation/docs/components/outputs/api-scraping-outputs/reward-params.json @@ -1,10 +1,10 @@ { "interval": { - "reward_pool": "172542733168986.886192478589145223", - "staking_supply": "60759292488059.481869771104188242", + "reward_pool": "170550581010206.362081368363611499", + "staking_supply": "60905574069554.404272283888050139", "staking_supply_scale_factor": "0.07342892", - "epoch_reward_budget": "4792853699.138524616457738587", - "stake_saturation_point": "253163718700.247841124046267451", + "epoch_reward_budget": "4737516139.172398946704676766", + "stake_saturation_point": "253773225289.810017801182866875", "sybil_resistance": "0.3", "active_set_work_factor": "10", "interval_pool_emission": "0.02" diff --git a/documentation/docs/components/outputs/api-scraping-outputs/time-now.md b/documentation/docs/components/outputs/api-scraping-outputs/time-now.md index 32d8432bb0..69c1baa4a2 100644 --- a/documentation/docs/components/outputs/api-scraping-outputs/time-now.md +++ b/documentation/docs/components/outputs/api-scraping-outputs/time-now.md @@ -1 +1 @@ -Tuesday, February 24th 2026, 10:08:37 UTC +Tuesday, March 10th 2026, 13:20:42 UTC diff --git a/documentation/docs/components/outputs/command-outputs/node-api-check-help.md b/documentation/docs/components/outputs/command-outputs/node-api-check-help.md index d41e42995c..875b2921ae 100644 --- a/documentation/docs/components/outputs/command-outputs/node-api-check-help.md +++ b/documentation/docs/components/outputs/command-outputs/node-api-check-help.md @@ -11,7 +11,7 @@ positional arguments: version_count (v, version) Sum of nodes in given version(s) -optional arguments: +options: -h, --help show this help message and exit -V, --version show program's version number and exit ``` diff --git a/documentation/docs/components/outputs/command-outputs/node-api-check-query-help.md b/documentation/docs/components/outputs/command-outputs/node-api-check-query-help.md index b96adaebe5..bfa068ddb0 100644 --- a/documentation/docs/components/outputs/command-outputs/node-api-check-query-help.md +++ b/documentation/docs/components/outputs/command-outputs/node-api-check-query-help.md @@ -6,12 +6,12 @@ usage: Nym-node API check query_stats [-h] [--no_routing_history] positional arguments: id supply nym-node identity key -optional arguments: +options: -h, --help show this help message and exit --no_routing_history Display node stats without routing history --no_verloc_metrics Display node stats without verloc metrics -m, --markdown Display results in markdown format - -o, --output [OUTPUT] + -o [OUTPUT], --output [OUTPUT] Save results to file (in current dir or supply with path without filename) ``` diff --git a/documentation/docs/components/outputs/command-outputs/nym-node-cli-install-help.md b/documentation/docs/components/outputs/command-outputs/nym-node-cli-install-help.md index ddc7c867b3..70baf6f736 100644 --- a/documentation/docs/components/outputs/command-outputs/nym-node-cli-install-help.md +++ b/documentation/docs/components/outputs/command-outputs/nym-node-cli-install-help.md @@ -12,7 +12,8 @@ usage: nym-node-cli install [-h] [-V] [-d BRANCH] [-v] options: -h, --help show this help message and exit -V, --version show program's version number and exit - -d, --dev BRANCH Define github branch (default: develop) + -d BRANCH, --dev BRANCH + Define github branch (default: develop) -v, --verbose Show full error tracebacks --mode {mixnode,entry-gateway,exit-gateway} Node mode: 'mixnode', 'entry-gateway', or 'exit- diff --git a/documentation/docs/components/outputs/nyx-outputs/token-table.md b/documentation/docs/components/outputs/nyx-outputs/token-table.md index b1f5c4dea3..c7429c3a02 100644 --- a/documentation/docs/components/outputs/nyx-outputs/token-table.md +++ b/documentation/docs/components/outputs/nyx-outputs/token-table.md @@ -2,6 +2,6 @@ |:-------------------|:------------------------------------------------------|--------------------:| | Total Supply | Maximum amount of NYM token in existence | 1_000_000_000 | | Mixmining Reserve | Tokens releasing for operators rewards | 196_896_265 | -| Vesting Tokens | Tokens locked outside of cicrulation for future claim | 500 | +| Vesting Tokens | Tokens locked outside of circulation for future claim | 500 | | Circulating Supply | Amount of unlocked tokens | 803_103_234 | | Stake Saturation | Optimal size of node self-bond + delegation | 1_016_987 | diff --git a/documentation/docs/pages/operators/orchestration/ansible.mdx b/documentation/docs/pages/operators/orchestration/ansible.mdx index 8a261c03f8..8a2540797a 100644 --- a/documentation/docs/pages/operators/orchestration/ansible.mdx +++ b/documentation/docs/pages/operators/orchestration/ansible.mdx @@ -216,6 +216,27 @@ cd playbooks ansible-playbook upgrade.yml ``` +###### 4. System Maintenance + +A playbook to de-clogg and cleanup the servers (VMs) hosting `nym-node`. This playbook should be used for operators having issues with their servers filling up disk space too fast. For operators hosting many VMs on a hypervisor dedicated / bare metal server - run this playbook alongside [the guide to clean up the hypervisor](/operators/troubleshooting/vps-isp#on-the-hypervisor-host-machine). + +Maintaining the VMs using this playbook and reclaiming space on hypervisor will prevent your servers from: + +- Sudden *disk full* crashes +- `journald` runaway growth +- Duplicate `rsyslog` logs +- `nym-node` log growth +- `qcow2` like allocation growth inside VPS storage + +The playbook is located in `roles/upgrade/defaults/system-maintenance.yml`, but the most important part is in `playbooks/group_vars/all.yml`, starting with a header `## SYSTEM MAINTENANCE PLAYBOOK KNOBS`. This commented list of adjustable knobs serves as a dashboard to tweak values of the playbook. Any overwrite will superseed the default ones in the playbook itself. + +This playbook will run roles on all the inventory entries in parallel by default. + +```sh +cd playbooks +ansible-playbook system-maintenance.yml +``` + diff --git a/documentation/docs/pages/operators/troubleshooting/vps-isp.mdx b/documentation/docs/pages/operators/troubleshooting/vps-isp.mdx index 40c108c6dd..2a0ed2cedc 100644 --- a/documentation/docs/pages/operators/troubleshooting/vps-isp.mdx +++ b/documentation/docs/pages/operators/troubleshooting/vps-isp.mdx @@ -3,11 +3,43 @@ import { Callout } from 'nextra/components'; import { VarInfo } from 'components/variable-info.tsx'; import { Steps } from 'nextra/components'; import { MyTab } from 'components/generic-tabs.tsx'; +import PruneLogsVM from 'components/operators/snippets/prune-logs-vm.mdx'; +import PruneLogsVPS from 'components/operators/snippets/prune-logs-vps.mdx'; # Troubleshooting VPS Setup + +## System Hygiene + +Maintaining a Nym node requires keeping a good care of the hosting server. Running a `nym-node` as a standalone process or wrapped in a service can produce gigabytes of logs. Eventually your operation can malfunction due to the logs chewing up too much disk space or memory. + +This chapter guides operators to address these points: + +- Sudden *disk full* crashes +- `journald` runaway growth +- Duplicate `rsyslog` logs +- `nym-node` log growth +- `qcow2` like allocation growth inside VPS storage + +Choose if you administer VPS based node manually, or self-administered VMs on a hypervizor, using [Ansible](/operators/orchestration/ansible). + + +`rm` is a powerful tool, without an easy way of revoking. If you need to extract or backup anything, do it now. Make sure you understand what you removing before you execute these commands. + + + +
+ Prune logs: VPS, + Prune logs: VM, hypervisor & Ansible, + ]} defaultIndex="0"> + + + +
+ ## IPv6 troubleshooting @@ -105,54 +137,3 @@ To find the right IP configuration, contact your VPS provider for support to fin On self-hosted machine it's a bit more tricky. In that case as an operator you must be sure that your ISP allows for public IPv4 and IPv6 and then it may be a bit of playing around to find the right configuration. One way may be to bind your binary with the `--host` flag to local address `127.0.0.1` and run `echo "$(curl -4 https://ifconfig.me)"` to get a public address which you use to bond your Mix Node to `nym-api` via Nym wallet. It's up to you as a node operator to ensure that your public and private IPs match up properly. - -### Pruning Logs - -Running a `nym-node` as a standalone process or wrapped in a service can produce gigabytes of logs. Eventually your operation can malfunction due to the logs chewing up too much disk space or memory. Below are two scripts that can help you clean this up. - - -`rm` is a powerful tool, without an easy way of revoking. If you need to extract or backup anything, do it now. Make sure you understand what you removing before you execute these commands. - - - - - -###### 1. See what's eating all your diskspace - -```sh -sudo find /var -type f -printf "%s\t%p\n" | sort -n -r | head -n 20 | ls -lh -sudo find /var -type f -exec ls -lh {} + 2>/dev/null | sort -k 5 -n -r | head -n 20 - -sudo du -h --max-depth=1 /var -sudo du -h /var/log -``` -###### 2. Prune those logs -```sh -sudo rm -f /var/log/syslog.1 -sudo rm -f /var/log/syslog -journalctl --disk-usage -sudo journalctl --vacuum-time=3d -sudo journalctl --vacuum-size=50M -``` - -###### 3.Enforce log rotation -```sh -sudo logrotate --force /etc/logrotate.conf -sudo service rsyslog restart -``` - -###### 4. Remove all old packages -```sh -sudo apt-get clean -sudo apt-get autoremove - -sudo apt-get clean -sudo rm -rf /var/lib/apt/lists/* -sudo apt-get update - -for snap in $(sudo snap list --all | awk '/disabled/{print $1, $3}'); do - sudo snap remove $snap -done -``` - - diff --git a/documentation/scripts/api-scraping/api_targets.py b/documentation/scripts/api-scraping/api_targets.py index 6bf8015047..9532b97138 100644 --- a/documentation/scripts/api-scraping/api_targets.py +++ b/documentation/scripts/api-scraping/api_targets.py @@ -100,7 +100,7 @@ def _get_stake_saturation(): def _get_desc_column(): supply = "Maximum amount of NYM token in existence" reserve = "Tokens releasing for operators rewards" - vesting = "Tokens locked outside of cicrulation for future claim" + vesting = "Tokens locked outside of circulation for future claim" circulating = "Amount of unlocked tokens" desc_column = [supply, reserve, vesting, circulating] return desc_column