Server Ansible maintenance & documentation (#6514)
* Create ansible playbook for trimming and rotationg logs * add docs for triming and log rotation * update ansible docs * add info on logic * cleanup the cleanup guide * update scraped stats * ready for review * address review
This commit is contained in:
import this
@@ -1,32 +1,38 @@
|
||||
---
|
||||
ansible_ssh_private_key_file: ~/.ssh/<SSH_KEY>
|
||||
|
||||
# nym_version: "v2025.21-mozzarella"
|
||||
#
|
||||
# NOTE:
|
||||
# if you want to pin Nym to a specific version instead of using the
|
||||
# latest release from GitHub in /tasks/main.yml then
|
||||
# uncomment the line above and set the tag
|
||||
|
||||
cli_url: "https://github.com/nymtech/nym/releases/download/nym-binaries-{{ nym_version }}/nym-cli"
|
||||
tunnel_manager_url: "https://github.com/nymtech/nym/raw/refs/heads/develop/scripts/nym-node-setup/network-tunnel-manager.sh"
|
||||
quic_bridge_deployment_url: "https://raw.githubusercontent.com/nymtech/nym/refs/heads/develop/scripts/nym-node-setup/quic_bridge_deployment.sh"
|
||||
|
||||
# NOTE: These values will be used globally unless overwritten per node in inventory/all
|
||||
###############################################################################
|
||||
## GLOBAL VARS
|
||||
## These values will be used globally unless overwritten per node in inventory/all
|
||||
###############################################################################
|
||||
|
||||
ansible_user: root # used for ssh, like `ssh root@nym-exit.ch-1.mynodes.net`
|
||||
email: "<EMAIL>" # used in certbot, description.toml and landing page
|
||||
website: "<WEBSITE>" # it is used in the description.toml
|
||||
description: "<NODE_PUBLIC_DESCRIPTION>" # or define per node in inventory/all
|
||||
# operator_name: "<OPERATOR_NAME>" # used in landing page if provided
|
||||
|
||||
###############################################################################
|
||||
## GLOBAL VARS
|
||||
## These values will be used globally unless overwritten per node in inventory/all
|
||||
## Set these vars only if you want them globally for all nodes
|
||||
## Per node changes in inventory/all will overwrite these global vars
|
||||
###############################################################################
|
||||
|
||||
# NOTE: Set these vars if you want them globally for all nodes
|
||||
# Per node changes in inventory/all will overwrite these global ones:
|
||||
hostname: "" # this is a fallback, keep it and setup hostname per node in inventory/all
|
||||
# moniker: "<MONIKER>" # if not setup here not in inventory/all it get's derived from the hostname
|
||||
# mode: <MODE> # entry-gateway/exit-gateway/mixnode
|
||||
# wireguard_enabled: <WIREGUARD_ENABLED> # true/false
|
||||
hostname: "" # this is a fallback, keep it and setup hostname per node in inventory/all
|
||||
|
||||
###############################################################################
|
||||
## GLOBAL PACKAGES
|
||||
## These will be installed during deployment
|
||||
###############################################################################
|
||||
|
||||
# NOTE: Possible vars to incule on landing page, etc.
|
||||
# operator_name: "<OPERATOR_NAME>"
|
||||
|
||||
packages:
|
||||
- tmux
|
||||
@@ -42,3 +48,73 @@ packages:
|
||||
- jq
|
||||
- wget
|
||||
- ufw
|
||||
|
||||
|
||||
###############################################################################
|
||||
## OPTIONAL OVERRIDES
|
||||
## All values below already have defaults in the playbook/roles
|
||||
## Uncomment only if you want to override them
|
||||
###############################################################################
|
||||
|
||||
###############################################################################
|
||||
## SYSTEM MAINTENANCE PLAYBOOK KNOBS
|
||||
###############################################################################
|
||||
|
||||
# nym_version: "v2025.21-mozzarella"
|
||||
|
||||
## NOTE:
|
||||
## if you want to pin Nym to a specific version instead of using the
|
||||
## latest release from GitHub in /tasks/main.yml then
|
||||
## uncomment the line above and set the tag
|
||||
|
||||
###############################################################################
|
||||
## SYSTEM MAINTENANCE PLAYBOOK KNOBS
|
||||
###############################################################################
|
||||
|
||||
## JOURNALD LIMITS
|
||||
|
||||
# journald_system_max_use: "100M" # max persistent journal size
|
||||
# journald_runtime_max_use: "50M" # max runtime journal size
|
||||
# journald_system_max_file_size: "25M" # max single journal file
|
||||
# journald_runtime_max_file_size: "10M" # max runtime journal file
|
||||
# journald_max_retention_sec: "3day" # retention time
|
||||
|
||||
# journald_rate_limit_interval: "30s" # rate limit window
|
||||
# journald_rate_limit_burst: "1000" # rate limit burst
|
||||
|
||||
|
||||
## NYM-NODE LOG CONTROL
|
||||
|
||||
# nymnode_log_level_max: "warning" # drop INFO logs
|
||||
# nymnode_rate_limit_interval: "30s" # per nym-node rate limit window
|
||||
# nymnode_rate_limit_burst: "200" # per nym-node rate limit burst
|
||||
|
||||
|
||||
## JOURNAL VACUUM TARGETS
|
||||
|
||||
# journal_vacuum_size: "100M"
|
||||
# journal_vacuum_time: "3days"
|
||||
|
||||
|
||||
## RSYSLOG
|
||||
|
||||
# disable_rsyslog: true
|
||||
|
||||
|
||||
## FSTRIM SCHEDULE
|
||||
|
||||
# fstrim_every_calendar: "*:0/15" # Aggressive
|
||||
# fstrim_every_calendar: "hourly" # Less aggressive
|
||||
|
||||
|
||||
## OPTIONAL CLEANUPS
|
||||
|
||||
# enable_apt_cleanup: true
|
||||
# enable_snap_cleanup: true
|
||||
|
||||
|
||||
## WRITEBACK TUNING
|
||||
|
||||
# enable_writeback_tuning: true
|
||||
# writeback_dirty_writeback_centisecs: 1500
|
||||
# writeback_dirty_expire_centisecs: 6000
|
||||
@@ -0,0 +1,38 @@
|
||||
---
|
||||
- name: Restrict logging, vacuum journals, and enable periodic trim
|
||||
hosts: all
|
||||
become: true
|
||||
gather_facts: false
|
||||
|
||||
# global knobs - override in inventory/group_vars/host_vars as needed
|
||||
vars:
|
||||
journald_system_max_use: "100M"
|
||||
journald_runtime_max_use: "50M"
|
||||
journald_system_max_file_size: "25M"
|
||||
journald_runtime_max_file_size: "10M"
|
||||
journald_max_retention_sec: "3day"
|
||||
journald_rate_limit_interval: "30s"
|
||||
journald_rate_limit_burst: "1000"
|
||||
|
||||
# per nym-node rate limit + level cap
|
||||
nymnode_log_level_max: "warning"
|
||||
nymnode_rate_limit_interval: "30s"
|
||||
nymnode_rate_limit_burst: "200"
|
||||
|
||||
# journal vacuum targets
|
||||
journal_vacuum_size: "100M"
|
||||
journal_vacuum_time: "3days"
|
||||
|
||||
# fstrim cadence (note: the systemd override uses cron-like calendar)
|
||||
fstrim_every_calendar: "*:0/15"
|
||||
|
||||
roles:
|
||||
- role: journald_limits
|
||||
- role: nymnode_logging
|
||||
- role: rsyslog_disable
|
||||
- role: journal_vacuum
|
||||
- role: classic_log_cleanup
|
||||
- role: apt_cleanup
|
||||
- role: snap_cleanup
|
||||
- role: fstrim_15min
|
||||
- role: report
|
||||
@@ -0,0 +1,21 @@
|
||||
---
|
||||
- name: Clean apt cache
|
||||
command: apt-get clean
|
||||
ignore_errors: true
|
||||
|
||||
- name: Autoremove unused packages
|
||||
command: apt-get -y autoremove
|
||||
ignore_errors: true
|
||||
|
||||
- name: Remove apt lists to reclaim space (they will be re-fetched on update)
|
||||
file:
|
||||
path: /var/lib/apt/lists
|
||||
state: absent
|
||||
ignore_errors: true
|
||||
|
||||
- name: Recreate apt lists directory
|
||||
file:
|
||||
path: /var/lib/apt/lists
|
||||
state: directory
|
||||
mode: "0755"
|
||||
ignore_errors: true
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
- name: Remove classic /var/log files if present (optional)
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: absent
|
||||
loop:
|
||||
- /var/log/syslog
|
||||
- /var/log/syslog.1
|
||||
- /var/log/kern.log
|
||||
- /var/log/kern.log.1
|
||||
- /var/log/auth.log
|
||||
- /var/log/auth.log.1
|
||||
- /var/log/ufw.log
|
||||
- /var/log/ufw.log.1
|
||||
ignore_errors: true
|
||||
|
||||
# This is best-effort and may still fail if other packages' postrotate scripts assume services exist.
|
||||
- name: Force logrotate (best-effort)
|
||||
command: "logrotate --force /etc/logrotate.conf"
|
||||
ignore_errors: true
|
||||
@@ -0,0 +1,3 @@
|
||||
---
|
||||
fstrim_timer_dropin_dir: "/etc/systemd/system/fstrim.timer.d"
|
||||
fstrim_every_calendar: "*:0/15"
|
||||
@@ -0,0 +1,31 @@
|
||||
---
|
||||
- name: Ensure systemd drop-in dir for fstrim.timer exists
|
||||
file:
|
||||
path: "{{ fstrim_timer_dropin_dir }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Override fstrim.timer schedule
|
||||
copy:
|
||||
dest: "{{ fstrim_timer_dropin_dir }}/override.conf"
|
||||
mode: "0644"
|
||||
content: |
|
||||
[Timer]
|
||||
OnCalendar=
|
||||
OnCalendar={{ fstrim_every_calendar }}
|
||||
Persistent=true
|
||||
RandomizedDelaySec=0
|
||||
|
||||
- name: Reload systemd after fstrim override
|
||||
systemd:
|
||||
daemon_reload: true
|
||||
|
||||
- name: Enable and start fstrim timer
|
||||
systemd:
|
||||
name: fstrim.timer
|
||||
enabled: true
|
||||
state: started
|
||||
|
||||
- name: Run fstrim now (best-effort)
|
||||
command: fstrim -av
|
||||
ignore_errors: true
|
||||
@@ -0,0 +1,3 @@
|
||||
---
|
||||
journal_vacuum_size: "100M"
|
||||
journal_vacuum_time: "3days"
|
||||
@@ -0,0 +1,6 @@
|
||||
---
|
||||
- name: Vacuum journal to size cap (hard)
|
||||
command: "journalctl --vacuum-size={{ journal_vacuum_size }}"
|
||||
|
||||
- name: Vacuum journal older than retention window (time)
|
||||
command: "journalctl --vacuum-time={{ journal_vacuum_time }}"
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
- name: Restart journald
|
||||
systemd:
|
||||
name: systemd-journald
|
||||
state: restarted
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
- name: Configure journald limits (persistent, capped, rate-limited)
|
||||
copy:
|
||||
dest: /etc/systemd/journald.conf
|
||||
mode: "0644"
|
||||
content: |
|
||||
[Journal]
|
||||
Storage=persistent
|
||||
Compress=yes
|
||||
Seal=yes
|
||||
|
||||
SystemMaxUse={{ journald_system_max_use }}
|
||||
RuntimeMaxUse={{ journald_runtime_max_use }}
|
||||
SystemMaxFileSize={{ journald_system_max_file_size }}
|
||||
RuntimeMaxFileSize={{ journald_runtime_max_file_size }}
|
||||
MaxRetentionSec={{ journald_max_retention_sec }}
|
||||
|
||||
RateLimitIntervalSec={{ journald_rate_limit_interval }}
|
||||
RateLimitBurst={{ journald_rate_limit_burst }}
|
||||
notify: Restart journald
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
nymnode_log_level_max: "warning"
|
||||
nymnode_rate_limit_interval: "30s"
|
||||
nymnode_rate_limit_burst: "200"
|
||||
nymnode_unit_name: "nym-node" # set to "nym-node.service" if your distro expects it
|
||||
nymnode_dropin_dir: "/etc/systemd/system/nym-node.service.d"
|
||||
nymnode_dropin_file: "10-logging.conf"
|
||||
@@ -0,0 +1,26 @@
|
||||
---
|
||||
- name: Ensure systemd drop-in dir for nym-node exists
|
||||
file:
|
||||
path: "{{ nymnode_dropin_dir }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
|
||||
- name: Cap nym-node logs + apply per-unit rate limiting
|
||||
copy:
|
||||
dest: "{{ nymnode_dropin_dir }}/{{ nymnode_dropin_file }}"
|
||||
mode: "0644"
|
||||
content: |
|
||||
[Service]
|
||||
LogLevelMax={{ nymnode_log_level_max }}
|
||||
LogRateLimitIntervalSec={{ nymnode_rate_limit_interval }}
|
||||
LogRateLimitBurst={{ nymnode_rate_limit_burst }}
|
||||
|
||||
- name: Reload systemd after nym-node drop-in
|
||||
systemd:
|
||||
daemon_reload: true
|
||||
|
||||
- name: Restart nym-node to apply new logging limits (best-effort)
|
||||
systemd:
|
||||
name: "{{ nymnode_unit_name }}"
|
||||
state: restarted
|
||||
ignore_errors: true
|
||||
@@ -0,0 +1,8 @@
|
||||
---
|
||||
- name: Show journal disk usage
|
||||
command: journalctl --disk-usage
|
||||
register: journal_usage
|
||||
changed_when: false
|
||||
|
||||
- debug:
|
||||
var: journal_usage.stdout
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
- name: Stop/disable rsyslog if installed (best-effort)
|
||||
systemd:
|
||||
name: rsyslog
|
||||
state: stopped
|
||||
enabled: false
|
||||
ignore_errors: true
|
||||
|
||||
- name: Remove rsyslog logrotate stanza if present (prevents logrotate failures)
|
||||
file:
|
||||
path: /etc/logrotate.d/rsyslog
|
||||
state: absent
|
||||
ignore_errors: true
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
- name: Remove disabled snap revisions (best-effort)
|
||||
shell: |
|
||||
set -euo pipefail
|
||||
snap list --all | awk '/disabled/{print $1, $3}' | while read -r name rev; do
|
||||
snap remove "$name" --revision="$rev" || true
|
||||
done
|
||||
args:
|
||||
executable: /bin/bash
|
||||
ignore_errors: true
|
||||
@@ -0,0 +1,283 @@
|
||||
import { Steps } from 'nextra/components';
|
||||
import { Callout } from 'nextra/components';
|
||||
|
||||
### Self-Managed VMs, Hypervisor (Host) Cleanup & Ansible Automated Maintenance
|
||||
|
||||
This chapter covers both:
|
||||
- Guest VM configuration
|
||||
- Host libvirt qcow2 behavior
|
||||
|
||||
If you run [KVM/libvirt](/operators/nodes/preliminary-steps/vps-setup/advanced), then this chapter is an essential part of your server hygiene procedure.
|
||||
|
||||
**Verify disk usage**
|
||||
|
||||
You can always identify what is consuming disk space
|
||||
|
||||
```sh
|
||||
df -h
|
||||
du -xhd1 /var | sort -h
|
||||
du -xhd1 /var/log | sort -h
|
||||
find /var -type f -exec ls -lh {} + 2>/dev/null | sort -k 5 -h | tail -n 30
|
||||
journalctl --disk-usage
|
||||
```
|
||||
|
||||
#### Inside each VM
|
||||
|
||||
<Callout type="info" emoji="ℹ️">
|
||||
In case your VMs are paused on the hypervisor (the host server), with no disk space left, start with [freeing some space on the hypervisor](/operators/troubleshooting/vps-isp#on-the-hypervisor-host-machine) and then return to this chapter, when the VMs are running again.
|
||||
</ Callout>
|
||||
|
||||
<Callout>
|
||||
**This part can be automated via [Ansible](/operators/orchestration/ansible), using the playbook [`system-maintenance.yml`](/operators/orchestration/ansible#4-system-maintenance). If this is your way, then proceed with Ansible and skip directly to the part called [*On the Hypervisor (host machine)*](#on-the-hypervisor-host-machine).**
|
||||
</ Callout>
|
||||
|
||||
<Steps >
|
||||
###### 1. Ensure root FS is *not* mounted with continuous discard
|
||||
|
||||
- This step is to enable `fstrim` to work in the next step and prevent two same logic processes fighting each other
|
||||
|
||||
```sh
|
||||
mount | grep " / "
|
||||
```
|
||||
|
||||
- If you see:
|
||||
```sh
|
||||
type ext4 (... discard ...)
|
||||
```
|
||||
|
||||
- Remove discard from `/etc/fstab` and `remount`:
|
||||
|
||||
```sh
|
||||
mount -o remount,nodiscard /
|
||||
mount | grep " / "
|
||||
```
|
||||
|
||||
Continuous discard can cause problematic allocation behavior.
|
||||
|
||||
|
||||
###### 2. Enable frequent `fstrim`
|
||||
|
||||
- This service will trim free space every 15 minutes
|
||||
|
||||
```sh
|
||||
mkdir -p /etc/systemd/system/fstrim.timer.d
|
||||
```
|
||||
|
||||
```sh
|
||||
cat >/etc/systemd/system/fstrim.timer.d/override.conf <<'EOF'
|
||||
[Timer]
|
||||
OnCalendar=
|
||||
OnCalendar=*:0/15
|
||||
Persistent=true
|
||||
RandomizedDelaySec=0
|
||||
EOF
|
||||
```
|
||||
|
||||
- Reload and restart the daemon:
|
||||
```sh
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now fstrim.timer
|
||||
fstrim -av
|
||||
```
|
||||
|
||||
###### 3. Prune `journald`
|
||||
- Trim logs by running these commands
|
||||
|
||||
```sh
|
||||
journalctl --vacuum-size=100M
|
||||
journalctl --vacuum-time=3days
|
||||
journalctl --disk-usage
|
||||
```
|
||||
- Compare the outcome of the last one with the previous check
|
||||
|
||||
###### 4. Make `journald` limits persistent
|
||||
|
||||
- Create a new file `/etc/systemd/journald.conf.d/limits.conf` with these hard caps:
|
||||
```sh
|
||||
mkdir -p /etc/systemd/journald.conf.d
|
||||
|
||||
cat >/etc/systemd/journald.conf.d/limits.conf <<'EOF'
|
||||
[Journal]
|
||||
Storage=persistent
|
||||
Compress=yes
|
||||
Seal=yes
|
||||
|
||||
SystemMaxUse=100M
|
||||
RuntimeMaxUse=50M
|
||||
SystemMaxFileSize=25M
|
||||
RuntimeMaxFileSize=10M
|
||||
MaxRetentionSec=3day
|
||||
|
||||
RateLimitIntervalSec=30s
|
||||
RateLimitBurst=1000
|
||||
EOF
|
||||
|
||||
systemctl restart systemd-journald
|
||||
```
|
||||
|
||||
###### 5. Disable classic `rsyslog`
|
||||
|
||||
Prevent duplicate logging as most Ubuntu VPS images run both `journald` and `rsyslog` by default.
|
||||
That duplicates logs into `/var/log/syslog`.
|
||||
|
||||
- Disable rsyslog by these commands:
|
||||
|
||||
```sh
|
||||
systemctl disable --now rsyslog.service || true
|
||||
systemctl mask rsyslog.service || true
|
||||
pgrep -x rsyslogd >/dev/null 2>&1 && pkill -9 -x rsyslogd || true
|
||||
```
|
||||
|
||||
- Optional cleanup of `syslog`:
|
||||
|
||||
```sh
|
||||
rm -f /var/log/syslog /var/log/syslog.1
|
||||
rm -f /var/log/kern.log /var/log/kern.log.1
|
||||
rm -f /var/log/auth.log /var/log/auth.log.1
|
||||
rm -f /var/log/ufw.log /var/log/ufw.log.1
|
||||
```
|
||||
> Note: Running `logrotate --force /etc/logrotate.conf` may fail after disabling `rsyslog`. That is expected and safe.
|
||||
|
||||
###### 6. Reduce `nym-node` log growth
|
||||
|
||||
- Create a new dir for a service file controlling `nym-node` logs:
|
||||
|
||||
```sh
|
||||
mkdir -p /etc/systemd/system/nym-node.service.d
|
||||
```
|
||||
|
||||
- Create the service file:
|
||||
```sh
|
||||
cat >/etc/systemd/system/nym-node.service.d/10-logging.conf <<'EOF'
|
||||
[Service]
|
||||
LogLevelMax=warning
|
||||
LogRateLimitIntervalSec=30s
|
||||
LogRateLimitBurst=300
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=nym-node
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl restart nym-node
|
||||
```
|
||||
|
||||
</ Steps>
|
||||
|
||||
|
||||
#### On the Hypervisor (host machine)
|
||||
|
||||
<Callout type="warning" emoji="⚠️">
|
||||
**Do *NOT* shrink the disk size with a `nym-node` on it - this would irreversibly corrupt your data! Instead follow the steps in this sub-chapter.**
|
||||
</ Callout>
|
||||
|
||||
**All the steps below are done from the host machine.**
|
||||
|
||||
<Steps>
|
||||
###### 1. Check your VMs state
|
||||
|
||||
- These two commands give you a quick overview of your VMs
|
||||
```sh
|
||||
virsh list --all
|
||||
ls -lh /var/lib/libvirt/images
|
||||
```
|
||||
|
||||
###### 2. Verify `qcow2` actual allocation vs virtual size
|
||||
|
||||
- The allocation defined by creating VMs is not equal to the actual usage
|
||||
|
||||
```sh
|
||||
qemu-img info --force-share /var/lib/libvirt/images/<IMAGE_NAME>.img
|
||||
|
||||
# for example:
|
||||
# qemu-img info --force-share /var/lib/libvirt/images/ubuntu1.img
|
||||
```
|
||||
|
||||
- Look at:
|
||||
```
|
||||
virtual size
|
||||
file length
|
||||
disk size
|
||||
```
|
||||
|
||||
###### 3. Monitor actual host allocation growth
|
||||
|
||||
- Watch how disk allocation changes in 5 minutes
|
||||
- This provides you an easy calculation how much disk space is consumed in a day, week or month
|
||||
|
||||
```sh
|
||||
watch -n 300 'du -sh /var/lib/libvirt/images'
|
||||
df -h
|
||||
```
|
||||
|
||||
###### 4. Sparsify
|
||||
|
||||
- If images grew previously, sparsify once
|
||||
- Shutdown a VM first:
|
||||
```sh
|
||||
virsh shutdown <IMAGE_NAME>
|
||||
|
||||
# for example:
|
||||
# virsh shutdown ubuntu1
|
||||
```
|
||||
- Then `sparsify`:
|
||||
|
||||
```sh
|
||||
virt-sparsify --in-place /var/lib/libvirt/images/<IMAGE_NAME>.img
|
||||
|
||||
# for example:
|
||||
# virt-sparsify --in-place /var/lib/libvirt/images/ubuntu1.img
|
||||
```
|
||||
- Finally re-start the machine
|
||||
```sh
|
||||
virsh start <IMAGE_NAME>
|
||||
|
||||
# for example:
|
||||
# virsh start ubuntu1
|
||||
```
|
||||
|
||||
- Repeat per VM if necessary
|
||||
|
||||
- If you want to sparsify all of them, you can run it in a loop
|
||||
```sh
|
||||
for vm in $(virsh list --all --name); do
|
||||
virsh shutdown "$vm"
|
||||
|
||||
while [ "$(virsh domstate "$vm")" != "shut off" ]; do
|
||||
sleep 5
|
||||
done
|
||||
|
||||
virt-sparsify --in-place /var/lib/libvirt/images/${vm}.img
|
||||
done
|
||||
```
|
||||
|
||||
- Then use same logic to start them all
|
||||
|
||||
```sh
|
||||
for vm in $(virsh list --all --name); do
|
||||
virsh start "$vm"
|
||||
done
|
||||
```
|
||||
- Finally make sure that they are all running:
|
||||
```sh
|
||||
virsh list --all
|
||||
```
|
||||
|
||||
###### 5. Ensure host filesystem supports hole punching
|
||||
|
||||
```sh
|
||||
df -T /var/lib/libvirt/images
|
||||
stat -f -c %T /var/lib/libvirt/images
|
||||
```
|
||||
|
||||
- The filesystem must support hole punching (fallocate) such as `ext4` or `xfs`
|
||||
|
||||
|
||||
###### 6. Final validation
|
||||
|
||||
- Finally check that how much space is on your host disk:
|
||||
```sh
|
||||
df -h
|
||||
du -sh /var/lib/libvirt/images
|
||||
```
|
||||
</ Steps>
|
||||
@@ -0,0 +1,142 @@
|
||||
import { Steps } from 'nextra/components';
|
||||
|
||||
### VPS Disk Usage & Log Control Guide
|
||||
|
||||
The steps below are for manual cleanup and space reclaim on a VPS hosting a `nym-node`.
|
||||
|
||||
<Steps>
|
||||
|
||||
###### 1. Identify what is consuming disk space
|
||||
|
||||
```sh
|
||||
df -h
|
||||
du -xhd1 /var | sort -h
|
||||
du -xhd1 /var/log | sort -h
|
||||
find /var -type f -exec ls -lh {} + 2>/dev/null | sort -k 5 -h | tail -n 30
|
||||
journalctl --disk-usage
|
||||
```
|
||||
|
||||
###### 2. Prune `journald`
|
||||
- Trim logs by running these commands
|
||||
|
||||
```sh
|
||||
journalctl --vacuum-size=100M
|
||||
journalctl --vacuum-time=3days
|
||||
journalctl --disk-usage
|
||||
```
|
||||
- Compare the outcome of the last one with the previous check
|
||||
|
||||
###### 3. Make `journald` limits persistent
|
||||
|
||||
- Create a new file `/etc/systemd/journald.conf` with these hard caps:
|
||||
```sh
|
||||
mkdir -p /etc/systemd/journald.conf.d
|
||||
|
||||
cat >/etc/systemd/journald.conf.d/limits.conf <<'EOF'
|
||||
[Journal]
|
||||
Storage=persistent
|
||||
Compress=yes
|
||||
Seal=yes
|
||||
|
||||
SystemMaxUse=100M
|
||||
RuntimeMaxUse=50M
|
||||
SystemMaxFileSize=25M
|
||||
RuntimeMaxFileSize=10M
|
||||
MaxRetentionSec=3day
|
||||
|
||||
RateLimitIntervalSec=30s
|
||||
RateLimitBurst=1000
|
||||
EOF
|
||||
|
||||
systemctl restart systemd-journald
|
||||
```
|
||||
|
||||
|
||||
###### 4. Disable classic `rsyslog`
|
||||
|
||||
Prevent duplicate logging as most Ubuntu VPS images run both `journald` and `rsyslog` by default.
|
||||
That duplicates logs into `/var/log/syslog`.
|
||||
|
||||
- Disable rsyslog by these commands:
|
||||
|
||||
```sh
|
||||
systemctl disable --now rsyslog.service || true
|
||||
systemctl mask rsyslog.service || true
|
||||
pgrep -x rsyslogd >/dev/null 2>&1 && pkill -9 -x rsyslogd || true
|
||||
```
|
||||
|
||||
- Optional cleanup of `syslog`:
|
||||
|
||||
```sh
|
||||
rm -f /var/log/syslog /var/log/syslog.1
|
||||
rm -f /var/log/kern.log /var/log/kern.log.1
|
||||
rm -f /var/log/auth.log /var/log/auth.log.1
|
||||
rm -f /var/log/ufw.log /var/log/ufw.log.1
|
||||
```
|
||||
> Note: Running `logrotate --force /etc/logrotate.conf` may fail after disabling `rsyslog`. That is expected and safe.
|
||||
|
||||
###### 5. Reduce `nym-node` log growth
|
||||
|
||||
- Create a new dir for a service file controlling `nym-node` logs:
|
||||
|
||||
```sh
|
||||
mkdir -p /etc/systemd/system/nym-node.service.d
|
||||
```
|
||||
|
||||
- Create the service file:
|
||||
```sh
|
||||
cat >/etc/systemd/system/nym-node.service.d/10-logging.conf <<'EOF'
|
||||
[Service]
|
||||
LogLevelMax=warning
|
||||
LogRateLimitIntervalSec=30s
|
||||
LogRateLimitBurst=300
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=nym-node
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl restart nym-node
|
||||
```
|
||||
|
||||
###### 6. Optional: Tune kernel writeback
|
||||
|
||||
- This config reduces burst disk pressure
|
||||
|
||||
```sh
|
||||
cat >/etc/sysctl.d/99-writeback-tuning.conf <<'EOF'
|
||||
vm.dirty_writeback_centisecs=1500
|
||||
vm.dirty_expire_centisecs=6000
|
||||
EOF
|
||||
|
||||
sysctl --system
|
||||
```
|
||||
|
||||
###### 7. Enable periodic Trim
|
||||
|
||||
- This reclaims the free space
|
||||
|
||||
```sh
|
||||
systemctl enable --now fstrim.timer
|
||||
fstrim -av
|
||||
```
|
||||
|
||||
###### 8. Reclaim apt cache space
|
||||
|
||||
- APT cache can become heavy too, reclaim that space by these commands:
|
||||
|
||||
```sh
|
||||
apt-get clean
|
||||
apt-get -y autoremove
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
mkdir -p /var/lib/apt/lists
|
||||
```
|
||||
|
||||
###### 9. Verify final state
|
||||
|
||||
- Check disk space now after the cleanup
|
||||
```sh
|
||||
df -h
|
||||
journalctl --disk-usage
|
||||
```
|
||||
</ Steps>
|
||||
@@ -5,7 +5,7 @@
|
||||
},
|
||||
"mixmining_reserve": {
|
||||
"denom": "unym",
|
||||
"amount": "172542733168986"
|
||||
"amount": "170550581010206"
|
||||
},
|
||||
"vesting_tokens": {
|
||||
"denom": "unym",
|
||||
@@ -13,6 +13,6 @@
|
||||
},
|
||||
"circulating_supply": {
|
||||
"denom": "unym",
|
||||
"amount": "827457266831014"
|
||||
"amount": "829449418989794"
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
827_457_266
|
||||
829_449_418
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
4_792
|
||||
4_737
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
253_163
|
||||
253_773
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
60_759_293
|
||||
60_905_575
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
60_759_292
|
||||
60_905_574
|
||||
|
||||
+4
-4
@@ -1,7 +1,7 @@
|
||||
| **Item** | **Description** | **Amount in NYM** |
|
||||
|:-------------------|:------------------------------------------------------|--------------------:|
|
||||
| Total Supply | Maximum amount of NYM token in existence | 1_000_000_000 |
|
||||
| Mixmining Reserve | Tokens releasing for operators rewards | 172_542_733 |
|
||||
| Vesting Tokens | Tokens locked outside of cicrulation for future claim | 0 |
|
||||
| Circulating Supply | Amount of unlocked tokens | 827_457_266 |
|
||||
| Stake Saturation | Optimal size of node self-bond + delegation | 253_163 |
|
||||
| Mixmining Reserve | Tokens releasing for operators rewards | 170_550_581 |
|
||||
| Vesting Tokens | Tokens locked outside of circulation for future claim | 0 |
|
||||
| Circulating Supply | Amount of unlocked tokens | 829_449_418 |
|
||||
| Stake Saturation | Optimal size of node self-bond + delegation | 253_773 |
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
{
|
||||
"interval": {
|
||||
"reward_pool": "172542733168986.886192478589145223",
|
||||
"staking_supply": "60759292488059.481869771104188242",
|
||||
"reward_pool": "170550581010206.362081368363611499",
|
||||
"staking_supply": "60905574069554.404272283888050139",
|
||||
"staking_supply_scale_factor": "0.07342892",
|
||||
"epoch_reward_budget": "4792853699.138524616457738587",
|
||||
"stake_saturation_point": "253163718700.247841124046267451",
|
||||
"epoch_reward_budget": "4737516139.172398946704676766",
|
||||
"stake_saturation_point": "253773225289.810017801182866875",
|
||||
"sybil_resistance": "0.3",
|
||||
"active_set_work_factor": "10",
|
||||
"interval_pool_emission": "0.02"
|
||||
|
||||
@@ -1 +1 @@
|
||||
Tuesday, February 24th 2026, 10:08:37 UTC
|
||||
Tuesday, March 10th 2026, 13:20:42 UTC
|
||||
|
||||
@@ -11,7 +11,7 @@ positional arguments:
|
||||
version_count (v, version)
|
||||
Sum of nodes in given version(s)
|
||||
|
||||
optional arguments:
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
-V, --version show program's version number and exit
|
||||
```
|
||||
|
||||
@@ -6,12 +6,12 @@ usage: Nym-node API check query_stats [-h] [--no_routing_history]
|
||||
positional arguments:
|
||||
id supply nym-node identity key
|
||||
|
||||
optional arguments:
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
--no_routing_history Display node stats without routing history
|
||||
--no_verloc_metrics Display node stats without verloc metrics
|
||||
-m, --markdown Display results in markdown format
|
||||
-o, --output [OUTPUT]
|
||||
-o [OUTPUT], --output [OUTPUT]
|
||||
Save results to file (in current dir or supply with
|
||||
path without filename)
|
||||
```
|
||||
|
||||
@@ -12,7 +12,8 @@ usage: nym-node-cli install [-h] [-V] [-d BRANCH] [-v]
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
-V, --version show program's version number and exit
|
||||
-d, --dev BRANCH Define github branch (default: develop)
|
||||
-d BRANCH, --dev BRANCH
|
||||
Define github branch (default: develop)
|
||||
-v, --verbose Show full error tracebacks
|
||||
--mode {mixnode,entry-gateway,exit-gateway}
|
||||
Node mode: 'mixnode', 'entry-gateway', or 'exit-
|
||||
|
||||
@@ -2,6 +2,6 @@
|
||||
|:-------------------|:------------------------------------------------------|--------------------:|
|
||||
| Total Supply | Maximum amount of NYM token in existence | 1_000_000_000 |
|
||||
| Mixmining Reserve | Tokens releasing for operators rewards | 196_896_265 |
|
||||
| Vesting Tokens | Tokens locked outside of cicrulation for future claim | 500 |
|
||||
| Vesting Tokens | Tokens locked outside of circulation for future claim | 500 |
|
||||
| Circulating Supply | Amount of unlocked tokens | 803_103_234 |
|
||||
| Stake Saturation | Optimal size of node self-bond + delegation | 1_016_987 |
|
||||
|
||||
@@ -216,6 +216,27 @@ cd playbooks
|
||||
ansible-playbook upgrade.yml
|
||||
```
|
||||
|
||||
###### 4. System Maintenance
|
||||
|
||||
A playbook to de-clogg and cleanup the servers (VMs) hosting `nym-node`. This playbook should be used for operators having issues with their servers filling up disk space too fast. For operators hosting many VMs on a hypervisor dedicated / bare metal server - run this playbook alongside [the guide to clean up the hypervisor](/operators/troubleshooting/vps-isp#on-the-hypervisor-host-machine).
|
||||
|
||||
Maintaining the VMs using this playbook and reclaiming space on hypervisor will prevent your servers from:
|
||||
|
||||
- Sudden *disk full* crashes
|
||||
- `journald` runaway growth
|
||||
- Duplicate `rsyslog` logs
|
||||
- `nym-node` log growth
|
||||
- `qcow2` like allocation growth inside VPS storage
|
||||
|
||||
The playbook is located in `roles/upgrade/defaults/system-maintenance.yml`, but the most important part is in `playbooks/group_vars/all.yml`, starting with a header `## SYSTEM MAINTENANCE PLAYBOOK KNOBS`. This commented list of adjustable knobs serves as a dashboard to tweak values of the playbook. Any overwrite will superseed the default ones in the playbook itself.
|
||||
|
||||
This playbook will run roles on all the inventory entries in parallel by default.
|
||||
|
||||
```sh
|
||||
cd playbooks
|
||||
ansible-playbook system-maintenance.yml
|
||||
```
|
||||
|
||||
</ Steps>
|
||||
|
||||
|
||||
|
||||
@@ -3,11 +3,43 @@ import { Callout } from 'nextra/components';
|
||||
import { VarInfo } from 'components/variable-info.tsx';
|
||||
import { Steps } from 'nextra/components';
|
||||
import { MyTab } from 'components/generic-tabs.tsx';
|
||||
import PruneLogsVM from 'components/operators/snippets/prune-logs-vm.mdx';
|
||||
import PruneLogsVPS from 'components/operators/snippets/prune-logs-vps.mdx';
|
||||
|
||||
# Troubleshooting VPS Setup
|
||||
|
||||
<VarInfo/>
|
||||
|
||||
|
||||
## System Hygiene
|
||||
|
||||
Maintaining a Nym node requires keeping a good care of the hosting server. Running a `nym-node` as a standalone process or wrapped in a service can produce gigabytes of logs. Eventually your operation can malfunction due to the logs chewing up too much disk space or memory.
|
||||
|
||||
This chapter guides operators to address these points:
|
||||
|
||||
- Sudden *disk full* crashes
|
||||
- `journald` runaway growth
|
||||
- Duplicate `rsyslog` logs
|
||||
- `nym-node` log growth
|
||||
- `qcow2` like allocation growth inside VPS storage
|
||||
|
||||
Choose if you administer VPS based node manually, or self-administered VMs on a hypervizor, using [Ansible](/operators/orchestration/ansible).
|
||||
|
||||
<Callout type="warning" emoji="⚠️">
|
||||
`rm` is a powerful tool, without an easy way of revoking. If you need to extract or backup anything, do it now. Make sure you understand what you removing before you execute these commands.
|
||||
</Callout>
|
||||
|
||||
|
||||
<div>
|
||||
<Tabs items={[
|
||||
<strong>Prune logs: VPS</strong>,
|
||||
<strong>Prune logs: VM, hypervisor & Ansible</strong>,
|
||||
]} defaultIndex="0">
|
||||
<MyTab><PruneLogsVPS /></MyTab>
|
||||
<MyTab><PruneLogsVM /></MyTab>
|
||||
</Tabs>
|
||||
</div>
|
||||
|
||||
## IPv6 troubleshooting
|
||||
|
||||
<Callout type="info" emoji="ℹ️">
|
||||
@@ -105,54 +137,3 @@ To find the right IP configuration, contact your VPS provider for support to fin
|
||||
On self-hosted machine it's a bit more tricky. In that case as an operator you must be sure that your ISP allows for public IPv4 and IPv6 and then it may be a bit of playing around to find the right configuration. One way may be to bind your binary with the `--host` flag to local address `127.0.0.1` and run `echo "$(curl -4 https://ifconfig.me)"` to get a public address which you use to bond your Mix Node to `nym-api` via Nym wallet.
|
||||
|
||||
It's up to you as a node operator to ensure that your public and private IPs match up properly.
|
||||
|
||||
### Pruning Logs
|
||||
|
||||
Running a `nym-node` as a standalone process or wrapped in a service can produce gigabytes of logs. Eventually your operation can malfunction due to the logs chewing up too much disk space or memory. Below are two scripts that can help you clean this up.
|
||||
|
||||
<Callout type="warning" emoji="⚠️">
|
||||
`rm` is a powerful tool, without an easy way of revoking. If you need to extract or backup anything, do it now. Make sure you understand what you removing before you execute these commands.
|
||||
</Callout>
|
||||
|
||||
|
||||
<Steps>
|
||||
|
||||
###### 1. See what's eating all your diskspace
|
||||
|
||||
```sh
|
||||
sudo find /var -type f -printf "%s\t%p\n" | sort -n -r | head -n 20 | ls -lh
|
||||
sudo find /var -type f -exec ls -lh {} + 2>/dev/null | sort -k 5 -n -r | head -n 20
|
||||
|
||||
sudo du -h --max-depth=1 /var
|
||||
sudo du -h /var/log
|
||||
```
|
||||
###### 2. Prune those logs
|
||||
```sh
|
||||
sudo rm -f /var/log/syslog.1
|
||||
sudo rm -f /var/log/syslog
|
||||
journalctl --disk-usage
|
||||
sudo journalctl --vacuum-time=3d
|
||||
sudo journalctl --vacuum-size=50M
|
||||
```
|
||||
|
||||
###### 3.Enforce log rotation
|
||||
```sh
|
||||
sudo logrotate --force /etc/logrotate.conf
|
||||
sudo service rsyslog restart
|
||||
```
|
||||
|
||||
###### 4. Remove all old packages
|
||||
```sh
|
||||
sudo apt-get clean
|
||||
sudo apt-get autoremove
|
||||
|
||||
sudo apt-get clean
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
sudo apt-get update
|
||||
|
||||
for snap in $(sudo snap list --all | awk '/disabled/{print $1, $3}'); do
|
||||
sudo snap remove $snap
|
||||
done
|
||||
```
|
||||
|
||||
</Steps>
|
||||
|
||||
@@ -100,7 +100,7 @@ def _get_stake_saturation():
|
||||
def _get_desc_column():
|
||||
supply = "Maximum amount of NYM token in existence"
|
||||
reserve = "Tokens releasing for operators rewards"
|
||||
vesting = "Tokens locked outside of cicrulation for future claim"
|
||||
vesting = "Tokens locked outside of circulation for future claim"
|
||||
circulating = "Amount of unlocked tokens"
|
||||
desc_column = [supply, reserve, vesting, circulating]
|
||||
return desc_column
|
||||
|
||||
Reference in New Issue
Block a user