localnet: add loadtest script and signoz docs
This commit is contained in:
+112
-28
@@ -1,37 +1,73 @@
|
||||
# Nym Localnet for Kata Container Runtimes
|
||||
# Nym Localnet
|
||||
|
||||
A complete Nym mixnet test environment running on Apple's container runtime for macOS (for now).
|
||||
A complete Nym mixnet test environment with OpenTelemetry instrumentation.
|
||||
Supports both Docker Desktop and Apple Container Runtime on macOS.
|
||||
|
||||
## Overview
|
||||
|
||||
This localnet setup provides a fully functional Nym mixnet for local development and testing:
|
||||
- **3 mixnodes** (layer 1, 2, 3)
|
||||
- **1 gateway** (entry + exit mode)
|
||||
- **2 gateways** (entry + exit mode)
|
||||
- **1 network-requester** (service provider)
|
||||
- **1 SOCKS5 client**
|
||||
- **OpenTelemetry tracing** via OTLP/gRPC to SigNoz (or any OTLP collector)
|
||||
|
||||
All components run in isolated containers with proper networking and dynamic IP resolution.
|
||||
When the `otel` feature is enabled (default), every nym-node exports traces covering
|
||||
the full packet lifecycle: ingress, Sphinx processing, forwarding, and final-hop delivery.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### Required
|
||||
- **macOS** (tested on macOS Sequoia 15.0+)
|
||||
- **Apple Container Runtime** - Built into macOS
|
||||
- **Docker Desktop** (for building images only)
|
||||
- **Docker Desktop** (recommended) or **Apple Container Runtime**
|
||||
- **Python 3** with `base58` library
|
||||
|
||||
### SigNoz (for trace viewing)
|
||||
|
||||
SigNoz is an open-source APM that receives and visualises OpenTelemetry data.
|
||||
Install it locally with Docker Compose -- this takes about 2 minutes:
|
||||
|
||||
```bash
|
||||
# Clone the SigNoz repository
|
||||
git clone -b main https://github.com/SigNoz/signoz.git ~/signoz
|
||||
cd ~/signoz/deploy
|
||||
|
||||
# Start SigNoz (runs ClickHouse, otel-collector, query-service, frontend)
|
||||
docker compose up -d
|
||||
|
||||
# Verify it is running
|
||||
docker ps --filter "name=signoz" --format "table {{.Names}}\t{{.Status}}"
|
||||
```
|
||||
|
||||
Once running:
|
||||
- **SigNoz UI**: http://localhost:8080
|
||||
- **OTLP gRPC collector**: localhost:4317 (used by nym-nodes)
|
||||
- **OTLP HTTP collector**: localhost:4318
|
||||
|
||||
The localnet script auto-detects the SigNoz Docker network (`signoz-net`) and
|
||||
routes OTel traffic directly to the collector container -- no manual endpoint
|
||||
configuration needed.
|
||||
|
||||
To stop SigNoz later:
|
||||
```bash
|
||||
cd ~/signoz/deploy && docker compose down
|
||||
```
|
||||
|
||||
### Installation
|
||||
```bash
|
||||
# Install Python dependencies
|
||||
pip3 install --break-system-packages base58
|
||||
|
||||
# Verify container runtime is available
|
||||
container --version
|
||||
|
||||
# Verify Docker is installed (for building)
|
||||
# Verify Docker is installed
|
||||
docker --version
|
||||
```
|
||||
|
||||
If using Apple Container Runtime instead of Docker:
|
||||
```bash
|
||||
container --version
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
@@ -198,54 +234,99 @@ container logs nym-gateway --follow
|
||||
### Status
|
||||
```bash
|
||||
# List all containers
|
||||
container list
|
||||
docker ps --filter "name=nym-" --format "table {{.Names}}\t{{.Status}}"
|
||||
|
||||
# Check specific container
|
||||
container logs nym-gateway
|
||||
docker logs nym-gateway
|
||||
|
||||
# Inspect network
|
||||
container network inspect nym-localnet-network
|
||||
docker network inspect nym-localnet-network
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
### Basic SOCKS5 Test
|
||||
```bash
|
||||
# Simple HTTP request with redirect following
|
||||
curl -L --socks5 localhost:1080 http://example.com
|
||||
# Simple HTTP request through the mixnet
|
||||
curl -x socks5h://127.0.0.1:1080 https://httpbin.org/get
|
||||
|
||||
# HTTPS request
|
||||
curl -L --socks5 localhost:1080 https://nymtech.net
|
||||
curl -x socks5h://127.0.0.1:1080 https://nymtech.net
|
||||
|
||||
# Download a file
|
||||
curl -L --socks5 localhost:1080 \
|
||||
curl -x socks5h://127.0.0.1:1080 \
|
||||
https://test-download-files-nym.s3.amazonaws.com/download-files/1MB.zip \
|
||||
--output /tmp/test.zip
|
||||
```
|
||||
|
||||
### Load Testing
|
||||
|
||||
A load test script is included to generate sustained traffic and populate SigNoz
|
||||
with meaningful trace data:
|
||||
|
||||
```bash
|
||||
# Default: 10 concurrent workers, 60 seconds
|
||||
./loadtest.sh
|
||||
|
||||
# Heavier load: 20 workers for 2 minutes
|
||||
./loadtest.sh -c 20 -d 120
|
||||
|
||||
# Light single-threaded test
|
||||
./loadtest.sh -c 1 -d 10
|
||||
|
||||
# Target a specific URL
|
||||
./loadtest.sh -c 5 -d 30 -u https://httpbin.org/bytes/4096
|
||||
```
|
||||
|
||||
The script reports live progress, then prints a summary with request counts,
|
||||
throughput, and latency percentiles (p50/p95/p99).
|
||||
|
||||
### Verify Network Topology
|
||||
```bash
|
||||
# View the generated topology
|
||||
container exec nym-gateway cat /localnet/network.json | jq .
|
||||
docker exec nym-gateway cat /localnet/network.json | jq .
|
||||
|
||||
# Check container IPs
|
||||
container list | grep nym-
|
||||
# Check container status
|
||||
docker ps --filter "name=nym-" --format "table {{.Names}}\t{{.Status}}"
|
||||
|
||||
# Verify all bonding files exist
|
||||
container exec nym-gateway ls -la /localnet/
|
||||
docker exec nym-gateway ls -la /localnet/
|
||||
```
|
||||
|
||||
### Test Mixnet Routing
|
||||
```bash
|
||||
# All traffic flows through: client → mix1 → mix2 → mix3 → gateway → internet
|
||||
# All traffic flows through: client -> gateway -> mix1 -> mix2 -> mix3 -> gateway -> internet
|
||||
# Watch logs to verify routing:
|
||||
container logs nym-mixnode1 --follow &
|
||||
container logs nym-mixnode2 --follow &
|
||||
container logs nym-mixnode3 --follow &
|
||||
container logs nym-gateway --follow &
|
||||
docker logs nym-mixnode1 --follow &
|
||||
docker logs nym-mixnode2 --follow &
|
||||
docker logs nym-mixnode3 --follow &
|
||||
docker logs nym-gateway --follow &
|
||||
|
||||
# Make a request
|
||||
curl -L --socks5 localhost:1080 https://nymtech.com
|
||||
curl -x socks5h://127.0.0.1:1080 https://nymtech.net
|
||||
```
|
||||
|
||||
## OpenTelemetry
|
||||
|
||||
OTel is enabled by default. Each nym-node exports traces via OTLP/gRPC covering
|
||||
packet ingress, Sphinx processing, forwarding, and final-hop delivery.
|
||||
|
||||
### Viewing Traces
|
||||
|
||||
- **SigNoz UI**: http://localhost:8080 -- filter by `serviceName = nym-node`
|
||||
- **Terminal report** (queries ClickHouse directly, no login needed):
|
||||
|
||||
```bash
|
||||
./otel-report.sh # last 15 minutes
|
||||
./otel-report.sh 60 # last 60 minutes
|
||||
./otel-report.sh live # auto-refresh every 10s
|
||||
```
|
||||
|
||||
### Disabling OTel
|
||||
|
||||
```bash
|
||||
OTEL_ENABLE=0 ./localnet.sh start # disable
|
||||
OTEL_ENDPOINT=http://my-collector:4317 ./localnet.sh start # custom collector
|
||||
```
|
||||
|
||||
### LP (Lewes Protocol) Testing
|
||||
@@ -289,8 +370,11 @@ This makes localnet perfect for rapid LP protocol development and testing.
|
||||
docker/localnet/
|
||||
├── README.md # This file
|
||||
├── localnet.sh # Main orchestration script
|
||||
├── Dockerfile.localnet # Docker image definition
|
||||
└── build_topology.py # Topology generator
|
||||
├── loadtest.sh # Load test / traffic generator
|
||||
├── otel-report.sh # Terminal-based OTel metrics report
|
||||
├── Dockerfile.localnet # Multi-stage Docker image (builder + slim runtime)
|
||||
├── build_topology.py # Topology generator
|
||||
└── localnet-logs.sh # Tmux-based multi-container log viewer
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
Executable
+297
@@ -0,0 +1,297 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Nym Localnet Load Test
|
||||
# Generates sustained traffic through the mixnet SOCKS5 proxy to produce
|
||||
# OTel traces and exercise the packet pipeline end-to-end.
|
||||
#
|
||||
# Usage:
|
||||
# ./loadtest.sh # defaults: 10 concurrent, 60s, mixed sizes
|
||||
# ./loadtest.sh -c 20 -d 120 # 20 concurrent, 120s
|
||||
# ./loadtest.sh -s 64k # fixed 64KB responses (many Sphinx fragments)
|
||||
# ./loadtest.sh -s 1k -c 5 -d 30 # small payloads, 5 workers
|
||||
#
|
||||
# Payload sizes (-s flag) map to Sphinx packet fragmentation:
|
||||
# 1k = ~1 Sphinx packet (sub-MTU, minimal fragmentation)
|
||||
# 4k = ~2-3 packets (small payload)
|
||||
# 16k = ~8-10 packets (medium payload)
|
||||
# 64k = ~32-35 packets (large payload, stresses forwarding)
|
||||
# 256k = ~128-130 packets (heavy payload, stresses queues)
|
||||
# 1m = ~512 packets (very heavy, potential backpressure)
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Localnet running (./localnet.sh start)
|
||||
# - SOCKS5 proxy available on localhost:1080
|
||||
|
||||
set -e
|
||||
|
||||
CONCURRENCY=10
|
||||
DURATION=60
|
||||
PROXY="socks5h://127.0.0.1:1080"
|
||||
PAYLOAD_SIZE=""
|
||||
CUSTOM_URL=""
|
||||
STATS_INTERVAL=5
|
||||
|
||||
# Default targets: mixed sizes for general testing
|
||||
TARGETS=(
|
||||
"https://httpbin.org/get"
|
||||
"https://httpbin.org/bytes/1024"
|
||||
"https://httpbin.org/delay/1"
|
||||
"https://example.com"
|
||||
"https://nym.com"
|
||||
)
|
||||
|
||||
# Convert human-readable size to bytes for httpbin
|
||||
parse_size() {
|
||||
local s
|
||||
s=$(echo "$1" | tr '[:upper:]' '[:lower:]')
|
||||
local num
|
||||
num=$(echo "$s" | sed 's/[a-z]*$//')
|
||||
case "$s" in
|
||||
*m|*mb) echo $(( num * 1024 * 1024 )) ;;
|
||||
*k|*kb) echo $(( num * 1024 )) ;;
|
||||
*) echo "$num" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [-c concurrency] [-d duration_secs] [-s payload_size] [-u url] [-p proxy]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -c Number of concurrent workers (default: $CONCURRENCY)"
|
||||
echo " -d Test duration in seconds (default: $DURATION)"
|
||||
echo " -s Response payload size: 1k, 4k, 16k, 64k, 256k, 1m (default: mixed)"
|
||||
echo " -u Custom target URL (overrides -s and default targets)"
|
||||
echo " -p SOCKS5 proxy address (default: $PROXY)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # 10 workers, 60s, mixed targets/sizes"
|
||||
echo " $0 -s 1k # small payloads (~1 Sphinx packet each)"
|
||||
echo " $0 -s 64k -c 5 # large payloads, 5 workers"
|
||||
echo " $0 -s 256k -c 2 -d 30 # very large payloads, observe queue pressure"
|
||||
echo " $0 -c 20 -d 120 # heavier concurrency, 2 minutes"
|
||||
exit 0
|
||||
}
|
||||
|
||||
while getopts "c:d:s:u:p:h" opt; do
|
||||
case $opt in
|
||||
c) CONCURRENCY=$OPTARG ;;
|
||||
d) DURATION=$OPTARG ;;
|
||||
s) PAYLOAD_SIZE=$OPTARG ;;
|
||||
u) CUSTOM_URL=$OPTARG ;;
|
||||
p) PROXY=$OPTARG ;;
|
||||
h) usage ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
log_ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
log_err() { echo -e "${RED}[ERROR]${NC} $*"; }
|
||||
|
||||
# Build sized URL if -s was specified
|
||||
SIZED_URL=""
|
||||
SIZE_LABEL="mixed"
|
||||
if [ -n "$PAYLOAD_SIZE" ]; then
|
||||
PAYLOAD_BYTES=$(parse_size "$PAYLOAD_SIZE")
|
||||
SIZED_URL="https://httpbin.org/bytes/${PAYLOAD_BYTES}"
|
||||
SIZE_LABEL="${PAYLOAD_SIZE} (~${PAYLOAD_BYTES} bytes)"
|
||||
fi
|
||||
|
||||
# Preflight checks
|
||||
if ! nc -z 127.0.0.1 1080 2>/dev/null; then
|
||||
log_err "SOCKS5 proxy not reachable on localhost:1080. Is the localnet running?"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Counters (written to temp files for cross-process aggregation)
|
||||
STATS_DIR=$(mktemp -d)
|
||||
cleanup() {
|
||||
kill $(jobs -p) 2>/dev/null || true
|
||||
rm -rf "$STATS_DIR"
|
||||
}
|
||||
trap cleanup INT TERM EXIT
|
||||
|
||||
pick_url() {
|
||||
if [ -n "$CUSTOM_URL" ]; then
|
||||
echo "$CUSTOM_URL"
|
||||
elif [ -n "$PAYLOAD_SIZE" ]; then
|
||||
echo "$SIZED_URL"
|
||||
else
|
||||
local idx=$((RANDOM % ${#TARGETS[@]}))
|
||||
echo "${TARGETS[$idx]}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Millisecond timestamp (works on both GNU and BSD/macOS date)
|
||||
now_ms() {
|
||||
python3 -c 'import time; print(int(time.time()*1000))'
|
||||
}
|
||||
|
||||
# Worker function: runs requests in a loop until duration expires
|
||||
worker() {
|
||||
local id=$1
|
||||
local end_time=$2
|
||||
local ok=0
|
||||
local fail=0
|
||||
|
||||
while [ "$(date +%s)" -lt "$end_time" ]; do
|
||||
local url
|
||||
url=$(pick_url)
|
||||
local start_ms
|
||||
start_ms=$(now_ms)
|
||||
|
||||
if curl -x "$PROXY" -m 15 -sf -o /dev/null -w "" "$url" 2>/dev/null; then
|
||||
ok=$((ok + 1))
|
||||
else
|
||||
fail=$((fail + 1))
|
||||
fi
|
||||
|
||||
local end_ms
|
||||
end_ms=$(now_ms)
|
||||
local latency=$((end_ms - start_ms))
|
||||
|
||||
echo "$latency" >> "$STATS_DIR/latencies_${id}.txt"
|
||||
done
|
||||
|
||||
echo "$ok" > "$STATS_DIR/ok_${id}.txt"
|
||||
echo "$fail" > "$STATS_DIR/fail_${id}.txt"
|
||||
}
|
||||
|
||||
echo ""
|
||||
log_info "=== Nym Localnet Load Test ==="
|
||||
log_info "Concurrency: $CONCURRENCY workers"
|
||||
log_info "Duration: ${DURATION}s"
|
||||
log_info "Payload: $SIZE_LABEL"
|
||||
if [ -n "$CUSTOM_URL" ]; then
|
||||
log_info "Target: $CUSTOM_URL"
|
||||
elif [ -n "$PAYLOAD_SIZE" ]; then
|
||||
log_info "Target: $SIZED_URL"
|
||||
else
|
||||
log_info "Targets: ${#TARGETS[@]} rotating URLs"
|
||||
fi
|
||||
log_info "Proxy: $PROXY"
|
||||
echo ""
|
||||
|
||||
# Quick connectivity check
|
||||
log_info "Preflight: testing SOCKS5 proxy..."
|
||||
if curl -x "$PROXY" -m 15 -sf -o /dev/null "https://httpbin.org/get"; then
|
||||
log_ok "SOCKS5 proxy is working"
|
||||
else
|
||||
log_err "SOCKS5 proxy test failed. Check localnet status."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
END_TIME=$(( $(date +%s) + DURATION ))
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
log_info "Starting $CONCURRENCY workers for ${DURATION}s..."
|
||||
echo ""
|
||||
|
||||
for i in $(seq 1 "$CONCURRENCY"); do
|
||||
worker "$i" "$END_TIME" &
|
||||
done
|
||||
|
||||
# Progress reporter (counts completed latency entries as a proxy for request count)
|
||||
while [ "$(date +%s)" -lt "$END_TIME" ]; do
|
||||
sleep "$STATS_INTERVAL"
|
||||
elapsed=$(( $(date +%s) - START_TIME ))
|
||||
remaining=$(( END_TIME - $(date +%s) ))
|
||||
if [ "$remaining" -lt 0 ]; then remaining=0; fi
|
||||
|
||||
total=0
|
||||
for f in "$STATS_DIR"/latencies_*.txt; do
|
||||
if [ -f "$f" ]; then
|
||||
count=$(wc -l < "$f" 2>/dev/null || echo 0)
|
||||
total=$((total + count))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$elapsed" -gt 0 ]; then
|
||||
rps=$(echo "scale=1; $total / $elapsed" | bc 2>/dev/null || echo "?")
|
||||
else
|
||||
rps="?"
|
||||
fi
|
||||
|
||||
printf "\r [%3ds / %3ds] requests: %d | ~%s req/s | remaining: %ds " \
|
||||
"$elapsed" "$DURATION" "$total" "$rps" "$remaining"
|
||||
done
|
||||
|
||||
echo ""
|
||||
log_info "Waiting for workers to finish..."
|
||||
wait 2>/dev/null || true
|
||||
|
||||
# Final stats
|
||||
echo ""
|
||||
log_info "=== Results ==="
|
||||
total_ok=0
|
||||
total_fail=0
|
||||
all_latencies=""
|
||||
|
||||
for f in "$STATS_DIR"/ok_*.txt; do
|
||||
[ -f "$f" ] && total_ok=$((total_ok + $(cat "$f" 2>/dev/null || echo 0)))
|
||||
done
|
||||
for f in "$STATS_DIR"/fail_*.txt; do
|
||||
[ -f "$f" ] && total_fail=$((total_fail + $(cat "$f" 2>/dev/null || echo 0)))
|
||||
done
|
||||
for f in "$STATS_DIR"/latencies_*.txt; do
|
||||
[ -f "$f" ] && all_latencies="$all_latencies $(cat "$f" 2>/dev/null | tr '\n' ' ')"
|
||||
done
|
||||
|
||||
total=$((total_ok + total_fail))
|
||||
actual_duration=$(( $(date +%s) - START_TIME ))
|
||||
|
||||
echo ""
|
||||
echo " Total requests: $total"
|
||||
echo " Successful: $total_ok"
|
||||
echo " Failed: $total_fail"
|
||||
if [ "$actual_duration" -gt 0 ]; then
|
||||
rps=$(echo "scale=2; $total / $actual_duration" | bc 2>/dev/null || echo "?")
|
||||
echo " Duration: ${actual_duration}s"
|
||||
echo " Throughput: ~${rps} req/s"
|
||||
fi
|
||||
|
||||
if [ -n "$all_latencies" ]; then
|
||||
sorted=$(echo "$all_latencies" | tr ' ' '\n' | sort -n | grep -v '^$')
|
||||
count=$(echo "$sorted" | wc -l | tr -d ' ')
|
||||
if [ "$count" -gt 0 ]; then
|
||||
p50_idx=$(( count * 50 / 100 ))
|
||||
p95_idx=$(( count * 95 / 100 ))
|
||||
p99_idx=$(( count * 99 / 100 ))
|
||||
[ "$p50_idx" -lt 1 ] && p50_idx=1
|
||||
[ "$p95_idx" -lt 1 ] && p95_idx=1
|
||||
[ "$p99_idx" -lt 1 ] && p99_idx=1
|
||||
|
||||
min_lat=$(echo "$sorted" | head -1)
|
||||
max_lat=$(echo "$sorted" | tail -1)
|
||||
p50=$(echo "$sorted" | sed -n "${p50_idx}p")
|
||||
p95=$(echo "$sorted" | sed -n "${p95_idx}p")
|
||||
p99=$(echo "$sorted" | sed -n "${p99_idx}p")
|
||||
|
||||
echo ""
|
||||
echo " Latency (ms):"
|
||||
echo " min: ${min_lat}ms"
|
||||
echo " p50: ${p50}ms"
|
||||
echo " p95: ${p95}ms"
|
||||
echo " p99: ${p99}ms"
|
||||
echo " max: ${max_lat}ms"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [ "$total_fail" -gt 0 ] && [ "$total" -gt 0 ]; then
|
||||
fail_pct=$(echo "scale=1; $total_fail * 100 / $total" | bc 2>/dev/null || echo "?")
|
||||
log_warn "Failure rate: ${fail_pct}% -- ${total_fail} of ${total} failed"
|
||||
else
|
||||
log_ok "All requests succeeded"
|
||||
fi
|
||||
echo ""
|
||||
log_info "View traces in SigNoz: http://localhost:8080/traces"
|
||||
log_info "Filter by service: nym-node"
|
||||
echo ""
|
||||
Executable
+222
@@ -0,0 +1,222 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Nym Localnet OTel Report
|
||||
# Queries ClickHouse directly to produce a terminal-based summary of
|
||||
# the core metrics captured by the OTel-instrumented nym-nodes.
|
||||
#
|
||||
# Usage:
|
||||
# ./otel-report.sh # last 15 minutes
|
||||
# ./otel-report.sh 60 # last 60 minutes
|
||||
# ./otel-report.sh live # live mode: refresh every 10s
|
||||
#
|
||||
# Prerequisites: localnet + SigNoz running
|
||||
|
||||
set -e
|
||||
|
||||
CH_CONTAINER="signoz-clickhouse"
|
||||
TRACES_TABLE="signoz_traces.distributed_signoz_index_v3"
|
||||
LOOKBACK_MIN=${1:-15}
|
||||
LIVE=false
|
||||
|
||||
if [ "$1" = "live" ]; then
|
||||
LIVE=true
|
||||
LOOKBACK_MIN=5
|
||||
fi
|
||||
|
||||
BLUE='\033[0;34m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BOLD='\033[1m'
|
||||
DIM='\033[2m'
|
||||
NC='\033[0m'
|
||||
|
||||
ch() {
|
||||
docker exec "$CH_CONTAINER" clickhouse-client --query "$1" 2>/dev/null
|
||||
}
|
||||
|
||||
divider() {
|
||||
echo -e "${DIM}$(printf '%.0s-' {1..78})${NC}"
|
||||
}
|
||||
|
||||
print_report() {
|
||||
local window="$1"
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD} Nym Localnet -- OTel Packet Pipeline Report${NC}"
|
||||
echo -e " ${DIM}Window: last ${window} minutes | $(date '+%Y-%m-%d %H:%M:%S')${NC}"
|
||||
divider
|
||||
|
||||
# 1. Throughput per operation
|
||||
echo -e "\n${BOLD} [1] Packet Throughput (packets/sec by operation)${NC}\n"
|
||||
ch "
|
||||
SELECT
|
||||
name AS operation,
|
||||
count(*) AS total,
|
||||
round(count(*) / (${window} * 60), 1) AS per_sec
|
||||
FROM ${TRACES_TABLE}
|
||||
WHERE timestamp >= now() - INTERVAL ${window} MINUTE
|
||||
AND serviceName = 'nym-node'
|
||||
AND name IN (
|
||||
'handle_received_nym_packet',
|
||||
'mixnode.sphinx_full_unwrap',
|
||||
'mixnode.forward_packet',
|
||||
'mixnode.final_hop'
|
||||
)
|
||||
GROUP BY name
|
||||
ORDER BY total DESC
|
||||
FORMAT PrettyCompactNoEscapes
|
||||
"
|
||||
|
||||
divider
|
||||
|
||||
# 2. Latency per operation
|
||||
echo -e "\n${BOLD} [2] Processing Latency (milliseconds)${NC}\n"
|
||||
ch "
|
||||
SELECT
|
||||
name AS operation,
|
||||
round(quantile(0.50)(duration_nano / 1e6), 3) AS p50_ms,
|
||||
round(quantile(0.95)(duration_nano / 1e6), 3) AS p95_ms,
|
||||
round(quantile(0.99)(duration_nano / 1e6), 3) AS p99_ms,
|
||||
round(quantile(0.999)(duration_nano / 1e6), 3) AS p999_ms,
|
||||
round(avg(duration_nano / 1e6), 3) AS avg_ms
|
||||
FROM ${TRACES_TABLE}
|
||||
WHERE timestamp >= now() - INTERVAL ${window} MINUTE
|
||||
AND serviceName = 'nym-node'
|
||||
AND name IN (
|
||||
'handle_received_nym_packet',
|
||||
'mixnode.sphinx_full_unwrap',
|
||||
'mixnode.forward_packet',
|
||||
'mixnode.final_hop'
|
||||
)
|
||||
AND duration_nano < 60000000000
|
||||
GROUP BY name
|
||||
ORDER BY p50_ms DESC
|
||||
FORMAT PrettyCompactNoEscapes
|
||||
"
|
||||
|
||||
divider
|
||||
|
||||
# 3. Error rate
|
||||
echo -e "\n${BOLD} [3] Error Rate${NC}\n"
|
||||
local errors
|
||||
errors=$(ch "
|
||||
SELECT
|
||||
name,
|
||||
countIf(has_error = true) AS errors,
|
||||
count(*) AS total,
|
||||
round(100.0 * countIf(has_error = true) / count(*), 3) AS error_pct
|
||||
FROM ${TRACES_TABLE}
|
||||
WHERE timestamp >= now() - INTERVAL ${window} MINUTE
|
||||
AND serviceName = 'nym-node'
|
||||
AND name IN (
|
||||
'handle_received_nym_packet',
|
||||
'mixnode.sphinx_full_unwrap',
|
||||
'mixnode.forward_packet',
|
||||
'mixnode.final_hop'
|
||||
)
|
||||
GROUP BY name
|
||||
HAVING errors > 0
|
||||
ORDER BY errors DESC
|
||||
FORMAT PrettyCompactNoEscapes
|
||||
")
|
||||
|
||||
if [ -z "$errors" ]; then
|
||||
echo -e " ${GREEN}No errors detected across all operations${NC}"
|
||||
else
|
||||
echo "$errors"
|
||||
fi
|
||||
|
||||
divider
|
||||
|
||||
# 4. Forwarding ratio (are packets being dropped between stages?)
|
||||
echo -e "\n${BOLD} [4] Pipeline Funnel (packet drop detection)${NC}\n"
|
||||
ch "
|
||||
SELECT
|
||||
name AS stage,
|
||||
count(*) AS packets,
|
||||
round(100.0 * count(*) / max(total_ingress), 1) AS pct_of_ingress
|
||||
FROM ${TRACES_TABLE}
|
||||
CROSS JOIN (
|
||||
SELECT count(*) AS total_ingress
|
||||
FROM ${TRACES_TABLE}
|
||||
WHERE timestamp >= now() - INTERVAL ${window} MINUTE
|
||||
AND serviceName = 'nym-node'
|
||||
AND name = 'handle_received_nym_packet'
|
||||
) AS t
|
||||
WHERE timestamp >= now() - INTERVAL ${window} MINUTE
|
||||
AND serviceName = 'nym-node'
|
||||
AND name IN (
|
||||
'handle_received_nym_packet',
|
||||
'mixnode.sphinx_full_unwrap',
|
||||
'mixnode.forward_packet',
|
||||
'mixnode.final_hop'
|
||||
)
|
||||
GROUP BY name
|
||||
ORDER BY packets DESC
|
||||
FORMAT PrettyCompactNoEscapes
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo -e " ${DIM}Expected ratios: sphinx_unwrap ~ 100%, forward ~ 75% (3 of 4 hops forward),${NC}"
|
||||
echo -e " ${DIM}final_hop ~ 25% (1 of 4 hops is the last one). Significantly lower = drops.${NC}"
|
||||
|
||||
divider
|
||||
|
||||
# 5. Throughput over time (1-minute buckets)
|
||||
echo -e "\n${BOLD} [5] Throughput Timeline (1-min buckets, ingress packets)${NC}\n"
|
||||
ch "
|
||||
SELECT
|
||||
toStartOfMinute(timestamp) AS minute,
|
||||
count(*) AS packets,
|
||||
round(count(*) / 60, 1) AS per_sec
|
||||
FROM ${TRACES_TABLE}
|
||||
WHERE timestamp >= now() - INTERVAL ${window} MINUTE
|
||||
AND serviceName = 'nym-node'
|
||||
AND name = 'handle_received_nym_packet'
|
||||
GROUP BY minute
|
||||
ORDER BY minute
|
||||
FORMAT PrettyCompactNoEscapes
|
||||
"
|
||||
|
||||
divider
|
||||
|
||||
# 6. Latency spikes (potential TCP congestion / backpressure indicators)
|
||||
echo -e "\n${BOLD} [6] Latency Spikes (sphinx_unwrap p99 per minute)${NC}\n"
|
||||
ch "
|
||||
SELECT
|
||||
toStartOfMinute(timestamp) AS minute,
|
||||
round(quantile(0.99)(duration_nano / 1e6), 3) AS p99_ms,
|
||||
round(quantile(0.50)(duration_nano / 1e6), 3) AS p50_ms,
|
||||
round(p99_ms / greatest(p50_ms, 0.001), 1) AS spike_ratio,
|
||||
count(*) AS samples
|
||||
FROM ${TRACES_TABLE}
|
||||
WHERE timestamp >= now() - INTERVAL ${window} MINUTE
|
||||
AND serviceName = 'nym-node'
|
||||
AND name = 'mixnode.sphinx_full_unwrap'
|
||||
GROUP BY minute
|
||||
ORDER BY minute
|
||||
FORMAT PrettyCompactNoEscapes
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo -e " ${DIM}spike_ratio > 10x suggests backpressure or queue buildup.${NC}"
|
||||
echo -e " ${DIM}Sustained high p99 across minutes may indicate TCP meltdown.${NC}"
|
||||
|
||||
divider
|
||||
echo ""
|
||||
echo -e " ${BLUE}SigNoz UI:${NC} http://localhost:8080"
|
||||
echo -e " ${DIM}Traces tab -> Filter: serviceName = nym-node${NC}"
|
||||
echo ""
|
||||
}
|
||||
|
||||
if [ "$LIVE" = "true" ]; then
|
||||
while true; do
|
||||
clear
|
||||
print_report "$LOOKBACK_MIN"
|
||||
echo -e " ${DIM}Refreshing in 10s... (Ctrl+C to stop)${NC}"
|
||||
sleep 10
|
||||
done
|
||||
else
|
||||
print_report "$LOOKBACK_MIN"
|
||||
fi
|
||||
Reference in New Issue
Block a user