diff --git a/docker/localnet/Dockerfile.localnet b/docker/localnet/Dockerfile.localnet index 63829c77a3..a6eb260f1a 100644 --- a/docker/localnet/Dockerfile.localnet +++ b/docker/localnet/Dockerfile.localnet @@ -1,43 +1,22 @@ -# Multi-stage Dockerfile for Nym localnet -# Stage 1: Build Rust binaries -# Stage 2: Build wireguard-go (cached separately from Rust builds) -# Stage 3: Slim runtime with only the final binaries +# Single-stage Dockerfile for Nym localnet +# Builds: nym-node, nym-network-requester, nym-socks5-client +# Target: Apple Container Runtime with host networking -# --- Rust build stage --- -FROM rust:latest AS builder +FROM rust:latest WORKDIR /usr/src/nym COPY ./ ./ ENV CARGO_BUILD_JOBS=8 +# Build all required binaries in release mode RUN cargo build --release --locked \ - -p nym-node --features otel \ + -p nym-node \ -p nym-network-requester \ -p nym-socks5-client -# --- wireguard-go build stage (cached independently) --- -FROM debian:trixie-slim AS wireguard-builder - -ARG TARGETARCH - -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl git build-essential ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -RUN curl -fsSL "https://go.dev/dl/go1.23.6.linux-${TARGETARCH}.tar.gz" \ - | tar -C /usr/local -xz - -ENV PATH="/usr/local/go/bin:$PATH" - -RUN git clone https://git.zx2c4.com/wireguard-go && \ - cd wireguard-go && make - -# --- Runtime stage --- -FROM debian:trixie-slim - -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ +# Install runtime dependencies including Go for wireguard-go +RUN apt update && apt install -y \ python3 \ python3-pip \ netcat-openbsd \ @@ -45,21 +24,31 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ iproute2 \ net-tools \ wireguard-tools \ + golang-go \ + git \ iptables \ && rm -rf /var/lib/apt/lists/* +# Install wireguard-go (userspace WireGuard implementation) +RUN git clone https://git.zx2c4.com/wireguard-go && \ + cd wireguard-go && \ + make && \ + cp wireguard-go /usr/local/bin/ && \ + cd .. && \ + rm -rf wireguard-go + +# Install Python dependencies for build_topology.py RUN pip3 install --break-system-packages base58 -# Copy wireguard-go from its dedicated build stage -COPY --from=wireguard-builder /wireguard-go/wireguard-go /usr/local/bin/ - -# Copy only the compiled binaries from the Rust builder stage -COPY --from=builder /usr/src/nym/target/release/nym-node /usr/local/bin/ -COPY --from=builder /usr/src/nym/target/release/nym-network-requester /usr/local/bin/ -COPY --from=builder /usr/src/nym/target/release/nym-socks5-client /usr/local/bin/ +# Move binaries to /usr/local/bin for easy access +RUN cp target/release/nym-node /usr/local/bin/ && \ + cp target/release/nym-network-requester /usr/local/bin/ && \ + cp target/release/nym-socks5-client /usr/local/bin/ +# Copy supporting scripts COPY ./docker/localnet/build_topology.py /usr/local/bin/ WORKDIR /nym +# Default command CMD ["nym-node", "--help"] diff --git a/docker/localnet/README.md b/docker/localnet/README.md index 84c52cb1b9..415b38e01b 100644 --- a/docker/localnet/README.md +++ b/docker/localnet/README.md @@ -1,71 +1,35 @@ -# Nym Localnet +# Nym Localnet for Kata Container Runtimes -A complete Nym mixnet test environment with OpenTelemetry instrumentation. -Supports both Docker Desktop and Apple Container Runtime on macOS. +A complete Nym mixnet test environment running on Apple's container runtime for macOS (for now). ## Overview This localnet setup provides a fully functional Nym mixnet for local development and testing: - **3 mixnodes** (layer 1, 2, 3) -- **2 gateways** (entry + exit mode) +- **1 gateway** (entry + exit mode) - **1 network-requester** (service provider) - **1 SOCKS5 client** -- **OpenTelemetry tracing** via OTLP/gRPC to SigNoz (or any OTLP collector) All components run in isolated containers with proper networking and dynamic IP resolution. -When the `otel` feature is enabled (default), every nym-node exports traces covering -the full packet lifecycle: ingress, Sphinx processing, forwarding, and final-hop delivery. ## Prerequisites ### Required - **macOS** (tested on macOS Sequoia 15.0+) -- **Docker Desktop** (recommended) or **Apple Container Runtime** +- **Apple Container Runtime** - Built into macOS +- **Docker Desktop** (for building images only) - **Python 3** with `base58` library -### SigNoz (for trace viewing) - -SigNoz is an open-source APM that receives and visualises OpenTelemetry data. -Install it locally with Docker Compose -- this takes about 2 minutes: - -```bash -# Clone the SigNoz repository -git clone -b main https://github.com/SigNoz/signoz.git ~/signoz -cd ~/signoz/deploy - -# Start SigNoz (runs ClickHouse, otel-collector, query-service, frontend) -docker compose up -d - -# Verify it is running -docker ps --filter "name=signoz" --format "table {{.Names}}\t{{.Status}}" -``` - -Once running: -- **SigNoz UI**: http://localhost:8080 -- **OTLP gRPC collector**: localhost:4317 (used by nym-nodes) -- **OTLP HTTP collector**: localhost:4318 - -The localnet script auto-detects the SigNoz Docker network (`signoz-net`) and -routes OTel traffic directly to the collector container -- no manual endpoint -configuration needed. - -To stop SigNoz later: -```bash -cd ~/signoz/deploy && docker compose down -``` - ### Installation ```bash # Install Python dependencies pip3 install --break-system-packages base58 -# Verify Docker is installed -docker --version -``` - -If using Apple Container Runtime instead of Docker: -```bash +# Verify container runtime is available container --version + +# Verify Docker is installed (for building) +docker --version ``` ## Quick Start @@ -234,99 +198,54 @@ container logs nym-gateway --follow ### Status ```bash # List all containers -docker ps --filter "name=nym-" --format "table {{.Names}}\t{{.Status}}" +container list # Check specific container -docker logs nym-gateway +container logs nym-gateway # Inspect network -docker network inspect nym-localnet-network +container network inspect nym-localnet-network ``` ## Testing ### Basic SOCKS5 Test ```bash -# Simple HTTP request through the mixnet -curl -x socks5h://127.0.0.1:1080 https://httpbin.org/get +# Simple HTTP request with redirect following +curl -L --socks5 localhost:1080 http://example.com # HTTPS request -curl -x socks5h://127.0.0.1:1080 https://nymtech.net +curl -L --socks5 localhost:1080 https://nymtech.net # Download a file -curl -x socks5h://127.0.0.1:1080 \ +curl -L --socks5 localhost:1080 \ https://test-download-files-nym.s3.amazonaws.com/download-files/1MB.zip \ --output /tmp/test.zip ``` -### Load Testing - -A load test script is included to generate sustained traffic and populate SigNoz -with meaningful trace data: - -```bash -# Default: 10 concurrent workers, 60 seconds -./loadtest.sh - -# Heavier load: 20 workers for 2 minutes -./loadtest.sh -c 20 -d 120 - -# Light single-threaded test -./loadtest.sh -c 1 -d 10 - -# Target a specific URL -./loadtest.sh -c 5 -d 30 -u https://httpbin.org/bytes/4096 -``` - -The script reports live progress, then prints a summary with request counts, -throughput, and latency percentiles (p50/p95/p99). - ### Verify Network Topology ```bash # View the generated topology -docker exec nym-gateway cat /localnet/network.json | jq . +container exec nym-gateway cat /localnet/network.json | jq . -# Check container status -docker ps --filter "name=nym-" --format "table {{.Names}}\t{{.Status}}" +# Check container IPs +container list | grep nym- # Verify all bonding files exist -docker exec nym-gateway ls -la /localnet/ +container exec nym-gateway ls -la /localnet/ ``` ### Test Mixnet Routing ```bash -# All traffic flows through: client -> gateway -> mix1 -> mix2 -> mix3 -> gateway -> internet +# All traffic flows through: client → mix1 → mix2 → mix3 → gateway → internet # Watch logs to verify routing: -docker logs nym-mixnode1 --follow & -docker logs nym-mixnode2 --follow & -docker logs nym-mixnode3 --follow & -docker logs nym-gateway --follow & +container logs nym-mixnode1 --follow & +container logs nym-mixnode2 --follow & +container logs nym-mixnode3 --follow & +container logs nym-gateway --follow & # Make a request -curl -x socks5h://127.0.0.1:1080 https://nymtech.net -``` - -## OpenTelemetry - -OTel is enabled by default. Each nym-node exports traces via OTLP/gRPC covering -packet ingress, Sphinx processing, forwarding, and final-hop delivery. - -### Viewing Traces - -- **SigNoz UI**: http://localhost:8080 -- filter by `serviceName = nym-node` -- **Terminal report** (queries ClickHouse directly, no login needed): - -```bash -./otel-report.sh # last 15 minutes -./otel-report.sh 60 # last 60 minutes -./otel-report.sh live # auto-refresh every 10s -``` - -### Disabling OTel - -```bash -OTEL_ENABLE=0 ./localnet.sh start # disable -OTEL_ENDPOINT=http://my-collector:4317 ./localnet.sh start # custom collector +curl -L --socks5 localhost:1080 https://nymtech.com ``` ### LP (Lewes Protocol) Testing @@ -370,11 +289,8 @@ This makes localnet perfect for rapid LP protocol development and testing. docker/localnet/ ├── README.md # This file ├── localnet.sh # Main orchestration script -├── loadtest.sh # Load test / traffic generator -├── otel-report.sh # Terminal-based OTel metrics report -├── Dockerfile.localnet # Multi-stage Docker image (builder + slim runtime) -├── build_topology.py # Topology generator -└── localnet-logs.sh # Tmux-based multi-container log viewer +├── Dockerfile.localnet # Docker image definition +└── build_topology.py # Topology generator ``` ## How It Works diff --git a/docker/localnet/loadtest.sh b/docker/localnet/loadtest.sh deleted file mode 100755 index adfc5902a6..0000000000 --- a/docker/localnet/loadtest.sh +++ /dev/null @@ -1,313 +0,0 @@ -#!/bin/bash - -# Nym Localnet Load Test -# Generates sustained traffic through the mixnet SOCKS5 proxy to produce -# OTel traces and exercise the packet pipeline end-to-end. -# -# Usage: -# ./loadtest.sh # defaults: 10 concurrent, 60s, mixed sizes -# ./loadtest.sh -c 20 -d 120 # 20 concurrent, 120s -# ./loadtest.sh -s 64k # fixed 64KB responses (many Sphinx fragments) -# ./loadtest.sh -s 1k -c 5 -d 30 # small payloads, 5 workers -# -# Payload sizes (-s flag) map to Sphinx packet fragmentation: -# 1k = ~1 Sphinx packet (sub-MTU, minimal fragmentation) -# 4k = ~2-3 packets (small payload) -# 16k = ~8-10 packets (medium payload) -# 64k = ~32-35 packets (large payload, stresses forwarding) -# 256k = ~128-130 packets (heavy payload, stresses queues) -# 1m = ~512 packets (very heavy, potential backpressure) -# -# Prerequisites: -# - Localnet running (./localnet.sh start) -# - SOCKS5 proxy available on localhost:1080 - -set -e - -check_dependencies() { - local missing=() - for cmd in curl jq bc; do - command -v "$cmd" &>/dev/null || missing+=("$cmd") - done - if [ ${#missing[@]} -gt 0 ]; then - echo "ERROR: missing required commands: ${missing[*]}" >&2 - exit 1 - fi -} -check_dependencies - -CONCURRENCY=10 -DURATION=60 -PROXY="socks5h://127.0.0.1:1080" -PAYLOAD_SIZE="" -CUSTOM_URL="" -STATS_INTERVAL=5 - -# Default targets: mixed sizes for general testing -TARGETS=( - "https://httpbin.org/get" - "https://httpbin.org/bytes/1024" - "https://httpbin.org/delay/1" - "https://example.com" - "https://nym.com" -) - -# Convert human-readable size to bytes for httpbin -parse_size() { - local s - s=$(echo "$1" | tr '[:upper:]' '[:lower:]') - local num - num=$(echo "$s" | sed 's/[a-z]*$//') - case "$s" in - *m|*mb) echo $(( num * 1024 * 1024 )) ;; - *k|*kb) echo $(( num * 1024 )) ;; - *) echo "$num" ;; - esac -} - -usage() { - echo "Usage: $0 [-c concurrency] [-d duration_secs] [-s payload_size] [-u url] [-p proxy]" - echo "" - echo "Options:" - echo " -c Number of concurrent workers (default: $CONCURRENCY)" - echo " -d Test duration in seconds (default: $DURATION)" - echo " -s Response payload size: 1k, 4k, 16k, 64k, 256k, 1m (default: mixed)" - echo " -u Custom target URL (overrides -s and default targets)" - echo " -p SOCKS5 proxy address (default: $PROXY)" - echo "" - echo "Examples:" - echo " $0 # 10 workers, 60s, mixed targets/sizes" - echo " $0 -s 1k # small payloads (~1 Sphinx packet each)" - echo " $0 -s 64k -c 5 # large payloads, 5 workers" - echo " $0 -s 256k -c 2 -d 30 # very large payloads, observe queue pressure" - echo " $0 -c 20 -d 120 # heavier concurrency, 2 minutes" - exit 0 -} - -while getopts "c:d:s:u:p:h" opt; do - case $opt in - c) CONCURRENCY=$OPTARG ;; - d) DURATION=$OPTARG ;; - s) PAYLOAD_SIZE=$OPTARG ;; - u) CUSTOM_URL=$OPTARG ;; - p) PROXY=$OPTARG ;; - h) usage ;; - *) usage ;; - esac -done - -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -log_info() { echo -e "${BLUE}[INFO]${NC} $*"; } -log_ok() { echo -e "${GREEN}[OK]${NC} $*"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } -log_err() { echo -e "${RED}[ERROR]${NC} $*"; } - -# Build sized URL if -s was specified -SIZED_URL="" -SIZE_LABEL="mixed" -if [ -n "$PAYLOAD_SIZE" ]; then - PAYLOAD_BYTES=$(parse_size "$PAYLOAD_SIZE") - SIZED_URL="https://httpbin.org/bytes/${PAYLOAD_BYTES}" - SIZE_LABEL="${PAYLOAD_SIZE} (~${PAYLOAD_BYTES} bytes)" -fi - -# Preflight checks -if ! nc -z 127.0.0.1 1080 2>/dev/null; then - log_err "SOCKS5 proxy not reachable on localhost:1080. Is the localnet running?" - exit 1 -fi - -# Counters (written to temp files for cross-process aggregation) -STATS_DIR=$(mktemp -d) -cleanup() { - kill $(jobs -p) 2>/dev/null || true - rm -rf "$STATS_DIR" -} -trap cleanup INT TERM EXIT - -pick_url() { - if [ -n "$CUSTOM_URL" ]; then - echo "$CUSTOM_URL" - elif [ -n "$PAYLOAD_SIZE" ]; then - echo "$SIZED_URL" - else - local idx=$((RANDOM % ${#TARGETS[@]})) - echo "${TARGETS[$idx]}" - fi -} - -# Millisecond timestamp (works on both GNU and BSD/macOS date) -now_ms() { - if command -v python3 &>/dev/null; then - python3 -c 'import time; print(int(time.time()*1000))' - else - echo $(($(date +%s) * 1000)) - fi -} - -# Worker function: runs requests in a loop until duration expires -worker() { - local id=$1 - local end_time=$2 - local ok=0 - local fail=0 - - while [ "$(date +%s)" -lt "$end_time" ]; do - local url - url=$(pick_url) - local start_ms - start_ms=$(now_ms) - - if curl -x "$PROXY" -m 15 -sf -o /dev/null -w "" "$url" 2>/dev/null; then - ok=$((ok + 1)) - else - fail=$((fail + 1)) - fi - - local end_ms - end_ms=$(now_ms) - local latency=$((end_ms - start_ms)) - - echo "$latency" >> "$STATS_DIR/latencies_${id}.txt" - done - - echo "$ok" > "$STATS_DIR/ok_${id}.txt" - echo "$fail" > "$STATS_DIR/fail_${id}.txt" -} - -echo "" -log_info "=== Nym Localnet Load Test ===" -log_info "Concurrency: $CONCURRENCY workers" -log_info "Duration: ${DURATION}s" -log_info "Payload: $SIZE_LABEL" -if [ -n "$CUSTOM_URL" ]; then - log_info "Target: $CUSTOM_URL" -elif [ -n "$PAYLOAD_SIZE" ]; then - log_info "Target: $SIZED_URL" -else - log_info "Targets: ${#TARGETS[@]} rotating URLs" -fi -log_info "Proxy: $PROXY" -echo "" - -# Quick connectivity check -log_info "Preflight: testing SOCKS5 proxy..." -if curl -x "$PROXY" -m 15 -sf -o /dev/null "https://httpbin.org/get"; then - log_ok "SOCKS5 proxy is working" -else - log_err "SOCKS5 proxy test failed. Check localnet status." - exit 1 -fi - -END_TIME=$(( $(date +%s) + DURATION )) -START_TIME=$(date +%s) - -log_info "Starting $CONCURRENCY workers for ${DURATION}s..." -echo "" - -for i in $(seq 1 "$CONCURRENCY"); do - worker "$i" "$END_TIME" & -done - -# Progress reporter (counts completed latency entries as a proxy for request count) -while [ "$(date +%s)" -lt "$END_TIME" ]; do - sleep "$STATS_INTERVAL" - elapsed=$(( $(date +%s) - START_TIME )) - remaining=$(( END_TIME - $(date +%s) )) - if [ "$remaining" -lt 0 ]; then remaining=0; fi - - total=0 - for f in "$STATS_DIR"/latencies_*.txt; do - if [ -f "$f" ]; then - count=$(wc -l < "$f" 2>/dev/null || echo 0) - total=$((total + count)) - fi - done - - if [ "$elapsed" -gt 0 ]; then - rps=$(echo "scale=1; $total / $elapsed" | bc 2>/dev/null || echo "?") - else - rps="?" - fi - - printf "\r [%3ds / %3ds] requests: %d | ~%s req/s | remaining: %ds " \ - "$elapsed" "$DURATION" "$total" "$rps" "$remaining" -done - -echo "" -log_info "Waiting for workers to finish..." -wait 2>/dev/null || true - -# Final stats -echo "" -log_info "=== Results ===" -total_ok=0 -total_fail=0 -all_latencies="" - -for f in "$STATS_DIR"/ok_*.txt; do - [ -f "$f" ] && total_ok=$((total_ok + $(cat "$f" 2>/dev/null || echo 0))) -done -for f in "$STATS_DIR"/fail_*.txt; do - [ -f "$f" ] && total_fail=$((total_fail + $(cat "$f" 2>/dev/null || echo 0))) -done -for f in "$STATS_DIR"/latencies_*.txt; do - [ -f "$f" ] && all_latencies="$all_latencies $(cat "$f" 2>/dev/null | tr '\n' ' ')" -done - -total=$((total_ok + total_fail)) -actual_duration=$(( $(date +%s) - START_TIME )) - -echo "" -echo " Total requests: $total" -echo " Successful: $total_ok" -echo " Failed: $total_fail" -if [ "$actual_duration" -gt 0 ]; then - rps=$(echo "scale=2; $total / $actual_duration" | bc 2>/dev/null || echo "?") - echo " Duration: ${actual_duration}s" - echo " Throughput: ~${rps} req/s" -fi - -if [ -n "$all_latencies" ]; then - sorted=$(echo "$all_latencies" | tr ' ' '\n' | sort -n | grep -v '^$') - count=$(echo "$sorted" | wc -l | tr -d ' ') - if [ "$count" -gt 0 ]; then - p50_idx=$(( count * 50 / 100 )) - p95_idx=$(( count * 95 / 100 )) - p99_idx=$(( count * 99 / 100 )) - [ "$p50_idx" -lt 1 ] && p50_idx=1 - [ "$p95_idx" -lt 1 ] && p95_idx=1 - [ "$p99_idx" -lt 1 ] && p99_idx=1 - - min_lat=$(echo "$sorted" | head -1) - max_lat=$(echo "$sorted" | tail -1) - p50=$(echo "$sorted" | sed -n "${p50_idx}p") - p95=$(echo "$sorted" | sed -n "${p95_idx}p") - p99=$(echo "$sorted" | sed -n "${p99_idx}p") - - echo "" - echo " Latency (ms):" - echo " min: ${min_lat}ms" - echo " p50: ${p50}ms" - echo " p95: ${p95}ms" - echo " p99: ${p99}ms" - echo " max: ${max_lat}ms" - fi -fi - -echo "" -if [ "$total_fail" -gt 0 ] && [ "$total" -gt 0 ]; then - fail_pct=$(echo "scale=1; $total_fail * 100 / $total" | bc 2>/dev/null || echo "?") - log_warn "Failure rate: ${fail_pct}% -- ${total_fail} of ${total} failed" -else - log_ok "All requests succeeded" -fi -echo "" -log_info "View traces in SigNoz: http://localhost:8080/traces" -log_info "Filter by service: nym-node" -echo "" diff --git a/docker/localnet/localnet-logs.sh b/docker/localnet/localnet-logs.sh index 9dfcdb5291..3347943e09 100755 --- a/docker/localnet/localnet-logs.sh +++ b/docker/localnet/localnet-logs.sh @@ -5,13 +5,6 @@ SESSION_NAME="nym-localnet-logs" -# Detect runtime -if command -v container &> /dev/null; then - RUNTIME="container" -else - RUNTIME="docker" -fi - # Container names CONTAINERS=( "nym-mixnode1" @@ -24,9 +17,9 @@ CONTAINERS=( # Check if containers are running running_containers=() -for ctr in "${CONTAINERS[@]}"; do - if $RUNTIME inspect "$ctr" &>/dev/null; then - running_containers+=("$ctr") +for container in "${CONTAINERS[@]}"; do + if container inspect "$container" &>/dev/null; then + running_containers+=("$container") fi done @@ -39,11 +32,11 @@ fi # Check if we're already in tmux if [ -n "$TMUX" ]; then # Inside tmux - create new window - tmux new-window -n "logs" "$RUNTIME logs -f ${running_containers[0]}" + tmux new-window -n "logs" "container logs -f ${running_containers[0]}" # Split for remaining containers for ((i=1; i<${#running_containers[@]}; i++)); do - tmux split-window -t logs "$RUNTIME logs -f ${running_containers[$i]}" + tmux split-window -t logs "container logs -f ${running_containers[$i]}" tmux select-layout -t logs tiled done @@ -55,11 +48,11 @@ else exec tmux attach-session -t "$SESSION_NAME" else # Create new session - tmux new-session -d -s "$SESSION_NAME" -n "logs" "$RUNTIME logs -f ${running_containers[0]}" + tmux new-session -d -s "$SESSION_NAME" -n "logs" "container logs -f ${running_containers[0]}" # Split for remaining containers for ((i=1; i<${#running_containers[@]}; i++)); do - tmux split-window -t "$SESSION_NAME:logs" "$RUNTIME logs -f ${running_containers[$i]}" + tmux split-window -t "$SESSION_NAME:logs" "container logs -f ${running_containers[$i]}" tmux select-layout -t "$SESSION_NAME:logs" tiled done diff --git a/docker/localnet/localnet.sh b/docker/localnet/localnet.sh index 5b92158f76..e021310bc2 100755 --- a/docker/localnet/localnet.sh +++ b/docker/localnet/localnet.sh @@ -2,8 +2,8 @@ set -ex -# Nym Localnet Orchestration Script -# Supports both Docker and Apple Container Runtime +# Nym Localnet Orchestration Script for Apple Container Runtime +# Emulates docker-compose functionality SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" @@ -14,38 +14,6 @@ NYM_VOLUME_PATH="/tmp/nym-localnet-home-$$" SUFFIX=${NYM_NODE_SUFFIX:-localnet} -# Detect container runtime: prefer Apple 'container' if available, fall back to docker -if command -v container &> /dev/null; then - RUNTIME="container" - HOST_INTERNAL="host.containers.internal" -else - RUNTIME="docker" - HOST_INTERNAL="host.docker.internal" -fi - -# OpenTelemetry configuration -# Set OTEL_ENABLE=1 to enable OTel tracing on all nym-node instances. -# OTEL_ENDPOINT should point to the OTLP gRPC collector reachable from containers. -# When SigNoz runs in Docker (signoz-net), we route to its collector directly. -OTEL_ENABLE=${OTEL_ENABLE:-1} -if [ -z "${OTEL_ENDPOINT:-}" ]; then - SIGNOZ_NET=$(docker network ls --filter name=signoz-net --format '{{.Name}}' 2>/dev/null || true) - if [ "$RUNTIME" = "docker" ] && [ -n "$SIGNOZ_NET" ]; then - OTEL_ENDPOINT="http://signoz-otel-collector:4317" - OTEL_SIGNOZ_NET="$SIGNOZ_NET" - else - OTEL_ENDPOINT="http://${HOST_INTERNAL}:4317" - OTEL_SIGNOZ_NET="" - fi -fi - -# Build OTel flags for nym-node run commands -otel_flags() { - if [ "$OTEL_ENABLE" = "1" ]; then - echo "--otel --otel-endpoint $OTEL_ENDPOINT" - fi -} - # Container names INIT_CONTAINER="nym-localnet-init" MIXNODE1_CONTAINER="nym-mixnode1" @@ -96,13 +64,13 @@ cleanup_host_state() { done } -# Check prerequisites +# Check if container command exists check_prerequisites() { - if ! command -v docker &> /dev/null; then - log_error "Docker not found" + if ! command -v container &> /dev/null; then + log_error "Apple 'container' command not found" + log_error "Install from: https://github.com/apple/container" exit 1 fi - log_info "Using runtime: $RUNTIME" } # Build the Docker image @@ -112,6 +80,7 @@ build_image() { cd "$PROJECT_ROOT" + # Build with Docker log_info "Building with Docker..." if ! docker build \ -f "$SCRIPT_DIR/Dockerfile.localnet" \ @@ -121,24 +90,30 @@ build_image() { exit 1 fi - # If using Apple container runtime, transfer image from Docker - if [ "$RUNTIME" = "container" ]; then - log_info "Transferring image to Apple container runtime..." - TEMP_IMAGE="/tmp/nym-localnet-image-$$.tar" - if ! docker save -o "$TEMP_IMAGE" "$IMAGE_NAME"; then - log_error "Failed to save Docker image" - exit 1 - fi - if ! container image load --input "$TEMP_IMAGE"; then - rm -f "$TEMP_IMAGE" - log_error "Failed to load image into container runtime" - exit 1 - fi + # Transfer image to container runtime + log_info "Transferring image to container runtime..." + + # Save to temporary file (container image load doesn't support stdin) + TEMP_IMAGE="/tmp/nym-localnet-image-$$.tar" + if ! docker save -o "$TEMP_IMAGE" "$IMAGE_NAME"; then + log_error "Failed to save Docker image" + exit 1 + fi + + # Load into container runtime from file + if ! container image load --input "$TEMP_IMAGE"; then rm -f "$TEMP_IMAGE" - if ! container image inspect "$IMAGE_NAME" &>/dev/null; then - log_error "Image not found in container runtime after load" - exit 1 - fi + log_error "Failed to load image into container runtime" + exit 1 + fi + + # Clean up temporary file + rm -f "$TEMP_IMAGE" + + # Verify image is available + if ! container image inspect "$IMAGE_NAME" &>/dev/null; then + log_error "Image not found in container runtime after load" + exit 1 fi log_success "Image built and loaded: $IMAGE_NAME" @@ -180,7 +155,7 @@ NETWORK_NAME="nym-localnet-network" # Create container network create_network() { log_info "Creating container network: $NETWORK_NAME" - if $RUNTIME network create "$NETWORK_NAME" 2>/dev/null; then + if container network create "$NETWORK_NAME" 2>/dev/null; then log_success "Network created: $NETWORK_NAME" else log_info "Network $NETWORK_NAME already exists or creation failed" @@ -189,9 +164,9 @@ create_network() { # Remove container network remove_network() { - if $RUNTIME network list | grep -q "$NETWORK_NAME"; then + if container network list | grep -q "$NETWORK_NAME"; then log_info "Removing network: $NETWORK_NAME" - $RUNTIME network rm "$NETWORK_NAME" 2>/dev/null || true + container network rm "$NETWORK_NAME" 2>/dev/null || true log_success "Network removed" fi } @@ -208,10 +183,7 @@ start_mixnode() { local verloc_port="2000${node_id}" local http_port="3000${node_id}" - local otel_args - otel_args=$(otel_flags) - - $RUNTIME run \ + container run \ --name "$container_name" \ -m 2G \ --network "$NETWORK_NAME" \ @@ -243,7 +215,7 @@ start_mixnode() { sleep 2; done; echo "Starting mix'"${node_id}"'..."; - exec nym-node '"${otel_args}"' run --id mix'"${node_id}"'-localnet --unsafe-disable-replay-protection --local + exec nym-node run --id mix'"${node_id}"'-localnet --unsafe-disable-replay-protection --local ' log_success "$container_name started" @@ -252,14 +224,9 @@ start_mixnode() { start_gateway() { log_info "Starting $GATEWAY_CONTAINER..." - local otel_args - otel_args=$(otel_flags) - - $RUNTIME run \ + container run \ --name "$GATEWAY_CONTAINER" \ -m 2G \ - --cap-add=NET_ADMIN \ - --device /dev/net/tun \ --network "$NETWORK_NAME" \ -p 9000:9000 \ -p 10004:10004 \ @@ -288,9 +255,11 @@ start_gateway() { --http-bind-address=0.0.0.0:30004 \ --http-access-token=lala \ --public-ips $CONTAINER_IP \ + --enable-lp true \ --lp-use-mock-ecash true \ --output=json \ - --wireguard-enabled false \ + --wireguard-enabled true \ + --wireguard-userspace true \ --bonding-information-output="/localnet/gateway.json"; echo "Waiting for network.json..."; @@ -298,7 +267,7 @@ start_gateway() { sleep 2; done; echo "Starting gateway with LP listener (mock ecash)..."; - exec nym-node '"${otel_args}"' run --id gateway-localnet --unsafe-disable-replay-protection --local --wireguard-enabled false --lp-use-mock-ecash true + exec nym-node run --id gateway-localnet --unsafe-disable-replay-protection --local --wireguard-enabled true --wireguard-userspace true --lp-use-mock-ecash true ' log_success "$GATEWAY_CONTAINER started" @@ -322,14 +291,9 @@ start_gateway() { start_gateway2() { log_info "Starting $GATEWAY2_CONTAINER..." - local otel_args - otel_args=$(otel_flags) - - $RUNTIME run \ + container run \ --name "$GATEWAY2_CONTAINER" \ -m 2G \ - --cap-add=NET_ADMIN \ - --device /dev/net/tun \ --network "$NETWORK_NAME" \ -p 9001:9001 \ -p 10005:10005 \ @@ -358,9 +322,11 @@ start_gateway2() { --http-bind-address=0.0.0.0:30005 \ --http-access-token=lala \ --public-ips $CONTAINER_IP \ + --enable-lp true \ --lp-use-mock-ecash true \ --output=json \ - --wireguard-enabled false \ + --wireguard-enabled true \ + --wireguard-userspace true \ --bonding-information-output="/localnet/gateway2.json"; echo "Waiting for network.json..."; @@ -368,7 +334,7 @@ start_gateway2() { sleep 2; done; echo "Starting gateway2 with LP listener (mock ecash)..."; - exec nym-node '"${otel_args}"' run --id gateway2-localnet --unsafe-disable-replay-protection --local --wireguard-enabled false --lp-use-mock-ecash true + exec nym-node run --id gateway2-localnet --unsafe-disable-replay-protection --local --wireguard-enabled true --wireguard-userspace true --lp-use-mock-ecash true ' log_success "$GATEWAY2_CONTAINER started" @@ -392,12 +358,12 @@ start_gateway2() { start_network_requester() { log_info "Starting $REQUESTER_CONTAINER..." - # Get gateway IP address (first IP only, in case container has multiple networks) + # Get gateway IP address log_info "Getting gateway IP address..." - GATEWAY_IP=$($RUNTIME exec "$GATEWAY_CONTAINER" hostname -i | awk '{print $1}') + GATEWAY_IP=$(container exec "$GATEWAY_CONTAINER" hostname -i) log_info "Gateway IP: $GATEWAY_IP" - $RUNTIME run \ + container run \ --name "$REQUESTER_CONTAINER" \ --network "$NETWORK_NAME" \ -v "$VOLUME_PATH:/localnet" \ @@ -432,7 +398,7 @@ start_network_requester() { start_socks5_client() { log_info "Starting $SOCKS5_CONTAINER..." - $RUNTIME run \ + container run \ --name "$SOCKS5_CONTAINER" \ --network "$NETWORK_NAME" \ -p 1080:1080 \ @@ -485,15 +451,15 @@ stop_containers() { log_info "Stopping all containers..." for container_name in "${ALL_CONTAINERS[@]}"; do - if $RUNTIME inspect "$container_name" &>/dev/null; then + if container inspect "$container_name" &>/dev/null; then log_info "Stopping $container_name" - $RUNTIME stop "$container_name" 2>/dev/null || true - $RUNTIME rm "$container_name" 2>/dev/null || true + container stop "$container_name" 2>/dev/null || true + container rm "$container_name" 2>/dev/null || true fi done # Also clean up init container if it exists - $RUNTIME rm "$INIT_CONTAINER" 2>/dev/null || true + container rm "$INIT_CONTAINER" 2>/dev/null || true log_success "All containers stopped" @@ -501,7 +467,7 @@ stop_containers() { remove_network } -# Show $RUNTIME logs +# Show container logs show_logs() { local container_name=${1:-} @@ -512,8 +478,8 @@ show_logs() { fi # Show logs for specific container - if $RUNTIME inspect "$container_name" &>/dev/null; then - $RUNTIME logs -f "$container_name" + if container inspect "$container_name" &>/dev/null; then + container logs -f "$container_name" else log_error "Container not found: $container_name" log_info "Available containers:" @@ -530,8 +496,8 @@ show_status() { echo "" for container_name in "${ALL_CONTAINERS[@]}"; do - if $RUNTIME inspect "$container_name" &>/dev/null; then - local status=$($RUNTIME inspect "$container_name" 2>/dev/null | grep -o '"Status":"[^"]*"' | cut -d'"' -f4 || echo "unknown") + if container inspect "$container_name" &>/dev/null; then + local status=$(container inspect "$container_name" 2>/dev/null | grep -o '"Status":"[^"]*"' | cut -d'"' -f4 || echo "unknown") echo -e " ${GREEN}●${NC} $container_name - $status" else echo -e " ${RED}○${NC} $container_name - not running" @@ -586,13 +552,13 @@ build_topology() { log_success " $file created" done - # Get container IPs (first IP only, containers may be on multiple networks) + # Get container IPs log_info "Getting container IP addresses..." - MIX1_IP=$($RUNTIME exec "$MIXNODE1_CONTAINER" hostname -i | awk '{print $1}') - MIX2_IP=$($RUNTIME exec "$MIXNODE2_CONTAINER" hostname -i | awk '{print $1}') - MIX3_IP=$($RUNTIME exec "$MIXNODE3_CONTAINER" hostname -i | awk '{print $1}') - GATEWAY_IP=$($RUNTIME exec "$GATEWAY_CONTAINER" hostname -i | awk '{print $1}') - GATEWAY2_IP=$($RUNTIME exec "$GATEWAY2_CONTAINER" hostname -i | awk '{print $1}') + MIX1_IP=$(container exec "$MIXNODE1_CONTAINER" hostname -i) + MIX2_IP=$(container exec "$MIXNODE2_CONTAINER" hostname -i) + MIX3_IP=$(container exec "$MIXNODE3_CONTAINER" hostname -i) + GATEWAY_IP=$(container exec "$GATEWAY_CONTAINER" hostname -i) + GATEWAY2_IP=$(container exec "$GATEWAY2_CONTAINER" hostname -i) log_info "Container IPs:" echo " mix1: $MIX1_IP" @@ -602,7 +568,7 @@ build_topology() { echo " gateway2: $GATEWAY2_IP" # Run build_topology.py in a container with access to the volumes - $RUNTIME run \ + container run \ --name "nym-localnet-topology-builder" \ --network "$NETWORK_NAME" \ -v "$VOLUME_PATH:/localnet" \ @@ -641,32 +607,19 @@ start_all() { start_mixnode 3 "$MIXNODE3_CONTAINER" start_gateway start_gateway2 - - # Connect nym containers to SigNoz network for direct OTLP routing - if [ -n "${OTEL_SIGNOZ_NET:-}" ]; then - log_info "Connecting containers to SigNoz network ($OTEL_SIGNOZ_NET)..." - for c in "$MIXNODE1_CONTAINER" "$MIXNODE2_CONTAINER" "$MIXNODE3_CONTAINER" \ - "$GATEWAY_CONTAINER" "$GATEWAY2_CONTAINER"; do - docker network connect "$OTEL_SIGNOZ_NET" "$c" 2>/dev/null && \ - log_success " $c connected to $OTEL_SIGNOZ_NET" || true - done - fi - build_topology # Configure networking for two-hop WireGuard routing on both gateways - # Note: Requires --privileged or --cap-add=NET_ADMIN on the containers. - # Non-fatal: only needed for WireGuard VPN routing, not mixnet packet testing. + # Note: Runs after build_topology to ensure gateways have finished WireGuard setup log_info "Configuring gateway networking (IP forwarding, NAT)..." for gw in "$GATEWAY_CONTAINER" "$GATEWAY2_CONTAINER"; do - if $RUNTIME exec "$gw" sh -c " - echo 1 > /proc/sys/net/ipv4/ip_forward 2>/dev/null - iptables-legacy -t nat -A POSTROUTING -o eth0 -j MASQUERADE 2>/dev/null - " 2>/dev/null; then - log_success "Configured $gw" - else - log_warn "Could not configure NAT on $gw (needs --privileged). WireGuard VPN routing will not work." - fi + container exec "$gw" sh -c " + # Enable IP forwarding + echo 1 > /proc/sys/net/ipv4/ip_forward + # Add NAT masquerade for outbound traffic + iptables-legacy -t nat -A POSTROUTING -o eth0 -j MASQUERADE + " + log_success "Configured $gw" done start_network_requester diff --git a/docker/localnet/otel-report.sh b/docker/localnet/otel-report.sh deleted file mode 100755 index 7930603382..0000000000 --- a/docker/localnet/otel-report.sh +++ /dev/null @@ -1,239 +0,0 @@ -#!/bin/bash - -# Nym Localnet OTel Report -# Queries ClickHouse directly to produce a terminal-based summary of -# the core metrics captured by the OTel-instrumented nym-nodes. -# -# Usage: -# ./otel-report.sh # last 15 minutes -# ./otel-report.sh 60 # last 60 minutes -# ./otel-report.sh live # live mode: refresh every 10s -# -# Prerequisites: localnet + SigNoz running - -set -e - -check_dependencies() { - local missing=() - for cmd in docker jq; do - command -v "$cmd" &>/dev/null || missing+=("$cmd") - done - if [ ${#missing[@]} -gt 0 ]; then - echo "ERROR: missing required commands: ${missing[*]}" >&2 - exit 1 - fi -} -check_dependencies - -CH_CONTAINER="signoz-clickhouse" -TRACES_TABLE="signoz_traces.distributed_signoz_index_v3" -LOOKBACK_MIN=${1:-15} -LIVE=false - -if [ "$1" = "live" ]; then - LIVE=true - LOOKBACK_MIN=5 -fi - -BLUE='\033[0;34m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -BOLD='\033[1m' -DIM='\033[2m' -NC='\033[0m' - -if ! docker ps --filter "name=$CH_CONTAINER" --format '{{.Names}}' | grep -q "$CH_CONTAINER"; then - echo "ERROR: ClickHouse container '$CH_CONTAINER' not running. Is SigNoz started?" >&2 - exit 1 -fi - -ch() { - docker exec "$CH_CONTAINER" clickhouse-client --query "$1" 2>/dev/null -} - -divider() { - echo -e "${DIM}$(printf '%.0s-' {1..78})${NC}" -} - -print_report() { - local window="$1" - - echo "" - echo -e "${BOLD} Nym Localnet -- OTel Packet Pipeline Report${NC}" - echo -e " ${DIM}Window: last ${window} minutes | $(date '+%Y-%m-%d %H:%M:%S')${NC}" - divider - - # 1. Throughput per operation - echo -e "\n${BOLD} [1] Packet Throughput (packets/sec by operation)${NC}\n" - ch " - SELECT - name AS operation, - count(*) AS total, - round(count(*) / (${window} * 60), 1) AS per_sec - FROM ${TRACES_TABLE} - WHERE timestamp >= now() - INTERVAL ${window} MINUTE - AND serviceName = 'nym-node' - AND name IN ( - 'handle_received_nym_packet', - 'mixnode.sphinx_full_unwrap', - 'mixnode.forward_packet', - 'mixnode.final_hop' - ) - GROUP BY name - ORDER BY total DESC - FORMAT PrettyCompactNoEscapes - " - - divider - - # 2. Latency per operation - echo -e "\n${BOLD} [2] Processing Latency (milliseconds)${NC}\n" - ch " - SELECT - name AS operation, - round(quantile(0.50)(duration_nano / 1e6), 3) AS p50_ms, - round(quantile(0.95)(duration_nano / 1e6), 3) AS p95_ms, - round(quantile(0.99)(duration_nano / 1e6), 3) AS p99_ms, - round(quantile(0.999)(duration_nano / 1e6), 3) AS p999_ms, - round(avg(duration_nano / 1e6), 3) AS avg_ms - FROM ${TRACES_TABLE} - WHERE timestamp >= now() - INTERVAL ${window} MINUTE - AND serviceName = 'nym-node' - AND name IN ( - 'handle_received_nym_packet', - 'mixnode.sphinx_full_unwrap', - 'mixnode.forward_packet', - 'mixnode.final_hop' - ) - AND duration_nano < 60000000000 - GROUP BY name - ORDER BY p50_ms DESC - FORMAT PrettyCompactNoEscapes - " - - divider - - # 3. Error rate - echo -e "\n${BOLD} [3] Error Rate${NC}\n" - local errors - errors=$(ch " - SELECT - name, - countIf(has_error = true) AS errors, - count(*) AS total, - round(100.0 * countIf(has_error = true) / count(*), 3) AS error_pct - FROM ${TRACES_TABLE} - WHERE timestamp >= now() - INTERVAL ${window} MINUTE - AND serviceName = 'nym-node' - AND name IN ( - 'handle_received_nym_packet', - 'mixnode.sphinx_full_unwrap', - 'mixnode.forward_packet', - 'mixnode.final_hop' - ) - GROUP BY name - HAVING errors > 0 - ORDER BY errors DESC - FORMAT PrettyCompactNoEscapes - ") - - if [ -z "$errors" ]; then - echo -e " ${GREEN}No errors detected across all operations${NC}" - else - echo "$errors" - fi - - divider - - # 4. Forwarding ratio (are packets being dropped between stages?) - echo -e "\n${BOLD} [4] Pipeline Funnel (packet drop detection)${NC}\n" - ch " - SELECT - name AS stage, - count(*) AS packets, - round(100.0 * count(*) / max(total_ingress), 1) AS pct_of_ingress - FROM ${TRACES_TABLE} - CROSS JOIN ( - SELECT count(*) AS total_ingress - FROM ${TRACES_TABLE} - WHERE timestamp >= now() - INTERVAL ${window} MINUTE - AND serviceName = 'nym-node' - AND name = 'handle_received_nym_packet' - ) AS t - WHERE timestamp >= now() - INTERVAL ${window} MINUTE - AND serviceName = 'nym-node' - AND name IN ( - 'handle_received_nym_packet', - 'mixnode.sphinx_full_unwrap', - 'mixnode.forward_packet', - 'mixnode.final_hop' - ) - GROUP BY name - ORDER BY packets DESC - FORMAT PrettyCompactNoEscapes - " - - echo "" - echo -e " ${DIM}Expected ratios: sphinx_unwrap ~ 100%, forward ~ 75% (3 of 4 hops forward),${NC}" - echo -e " ${DIM}final_hop ~ 25% (1 of 4 hops is the last one). Significantly lower = drops.${NC}" - - divider - - # 5. Throughput over time (1-minute buckets) - echo -e "\n${BOLD} [5] Throughput Timeline (1-min buckets, ingress packets)${NC}\n" - ch " - SELECT - toStartOfMinute(timestamp) AS minute, - count(*) AS packets, - round(count(*) / 60, 1) AS per_sec - FROM ${TRACES_TABLE} - WHERE timestamp >= now() - INTERVAL ${window} MINUTE - AND serviceName = 'nym-node' - AND name = 'handle_received_nym_packet' - GROUP BY minute - ORDER BY minute - FORMAT PrettyCompactNoEscapes - " - - divider - - # 6. Latency spikes (potential TCP congestion / backpressure indicators) - echo -e "\n${BOLD} [6] Latency Spikes (sphinx_unwrap p99 per minute)${NC}\n" - ch " - SELECT - toStartOfMinute(timestamp) AS minute, - round(quantile(0.99)(duration_nano / 1e6), 3) AS p99_ms, - round(quantile(0.50)(duration_nano / 1e6), 3) AS p50_ms, - round(p99_ms / greatest(p50_ms, 0.001), 1) AS spike_ratio, - count(*) AS samples - FROM ${TRACES_TABLE} - WHERE timestamp >= now() - INTERVAL ${window} MINUTE - AND serviceName = 'nym-node' - AND name = 'mixnode.sphinx_full_unwrap' - GROUP BY minute - ORDER BY minute - FORMAT PrettyCompactNoEscapes - " - - echo "" - echo -e " ${DIM}spike_ratio > 10x suggests backpressure or queue buildup.${NC}" - echo -e " ${DIM}Sustained high p99 across minutes may indicate TCP meltdown.${NC}" - - divider - echo "" - echo -e " ${BLUE}SigNoz UI:${NC} http://localhost:8080" - echo -e " ${DIM}Traces tab -> Filter: serviceName = nym-node${NC}" - echo "" -} - -if [ "$LIVE" = "true" ]; then - while true; do - clear - print_report "$LOOKBACK_MIN" - echo -e " ${DIM}Refreshing in 10s... (Ctrl+C to stop)${NC}" - sleep 10 - done -else - print_report "$LOOKBACK_MIN" -fi