From dd7a43dd159deb38a3dff0dee7d58de8a02e2315 Mon Sep 17 00:00:00 2001 From: durch Date: Tue, 16 Dec 2025 23:17:44 +0100 Subject: [PATCH] Various smaller fixes --- Cargo.lock | 8 - common/nym-kcp/Cargo.toml | 1 - common/nym-kcp/src/lib.rs | 2 + common/nym-kcp/src/session.rs | 9 +- common/nym-lp/src/codec.rs | 8 +- common/nym-lp/src/message.rs | 28 +- common/nym-lp/src/replay/validator.rs | 12 +- common/nym-lp/src/session.rs | 4 +- common/wireguard/Cargo.toml | 1 - docs/LP_DEPLOYMENT.md | 845 ------------------ gateway/src/node/lp_listener/handler.rs | 19 +- .../src/lp_client/client.rs | 7 + .../src/lp_client/nested_session.rs | 7 + 13 files changed, 72 insertions(+), 879 deletions(-) delete mode 100644 docs/LP_DEPLOYMENT.md diff --git a/Cargo.lock b/Cargo.lock index 07d760df9a..6140b6cd8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1000,12 +1000,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" -[[package]] -name = "byte_string" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11aade7a05aa8c3a351cedc44c3fc45806430543382fcc4743a9b757a2a0b4ed" - [[package]] name = "bytecodec" version = "0.4.15" @@ -6627,7 +6621,6 @@ name = "nym-kcp" version = "0.1.0" dependencies = [ "ansi_term", - "byte_string", "bytes", "env_logger", "log", @@ -8084,7 +8077,6 @@ dependencies = [ "futures", "ip_network", "ipnetwork", - "log", "nym-credential-verification", "nym-credentials-interface", "nym-crypto", diff --git a/common/nym-kcp/Cargo.toml b/common/nym-kcp/Cargo.toml index beafab5ee9..73cf7edfcb 100644 --- a/common/nym-kcp/Cargo.toml +++ b/common/nym-kcp/Cargo.toml @@ -18,7 +18,6 @@ path = "bin/session/main.rs" [dependencies] tokio-util = { workspace = true, features = ["codec"] } -byte_string = "1.0" bytes = { workspace = true } thiserror = { workspace = true } log = { workspace = true } diff --git a/common/nym-kcp/src/lib.rs b/common/nym-kcp/src/lib.rs index 27c71df515..c3663a9e30 100644 --- a/common/nym-kcp/src/lib.rs +++ b/common/nym-kcp/src/lib.rs @@ -3,3 +3,5 @@ pub mod driver; pub mod error; pub mod packet; pub mod session; + +pub const MAX_RTO: u32 = 60000; // Same as used in update_rtt \ No newline at end of file diff --git a/common/nym-kcp/src/session.rs b/common/nym-kcp/src/session.rs index 2d9b844cee..e6924c5be7 100644 --- a/common/nym-kcp/src/session.rs +++ b/common/nym-kcp/src/session.rs @@ -6,9 +6,11 @@ use std::{ use ansi_term::Color::Yellow; use bytes::{Buf, BytesMut}; -use log::{debug, error, info, warn}; +use log::{debug, error, warn}; use std::thread; +use crate::MAX_RTO; + use super::error::KcpError; use super::packet::{KcpCommand, KcpPacket}; @@ -430,12 +432,11 @@ impl KcpSession { // Exponential backoff: double RTO for this segment seg.rto *= 2; // Clamp to the session's maximum RTO (hardcoded as 60s for now) - const MAX_RTO: u32 = 60000; // Same as used in update_rtt if seg.rto > MAX_RTO { seg.rto = MAX_RTO; } seg.resendts = self.current + seg.rto; - info!( + debug!( "{}", Yellow.paint(format!( "Retransmit conv_id: {}, sn: {}, frg: {}", @@ -685,7 +686,7 @@ impl KcpSession { } } let rto = self.rx_srtt + cmp::max(self.interval, 4 * self.rx_rttval); - self.rx_rto = rto.clamp(self.rx_minrto, 60000); + self.rx_rto = rto.clamp(self.rx_minrto, MAX_RTO); } } diff --git a/common/nym-lp/src/codec.rs b/common/nym-lp/src/codec.rs index c1e52b17f7..d8eac1fbb8 100644 --- a/common/nym-lp/src/codec.rs +++ b/common/nym-lp/src/codec.rs @@ -393,6 +393,8 @@ impl LpError { #[cfg(test)] mod tests { + use std::time::{SystemTime, UNIX_EPOCH}; + // Import standalone functions use super::{OuterAeadKey, parse_lp_packet, serialize_lp_packet}; // Keep necessary imports @@ -734,11 +736,15 @@ mod tests { use crate::message::ClientHelloData; let mut dst = BytesMut::new(); + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("System time before UNIX epoch") + .as_secs(); // Create ClientHelloData with fresh salt let client_key = [7u8; 32]; let client_ed25519_key = [8u8; 32]; - let hello_data = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key); + let hello_data = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key, timestamp); // Create a ClientHello message packet let packet = LpPacket { diff --git a/common/nym-lp/src/message.rs b/common/nym-lp/src/message.rs index aac66e0910..59ce473a4d 100644 --- a/common/nym-lp/src/message.rs +++ b/common/nym-lp/src/message.rs @@ -31,17 +31,13 @@ impl ClientHelloData { pub fn new_with_fresh_salt( client_lp_public_key: [u8; 32], client_ed25519_public_key: [u8; 32], + timestamp: u64, ) -> Self { - use std::time::{SystemTime, UNIX_EPOCH}; // Generate salt: timestamp + nonce let mut salt = [0u8; 32]; // First 8 bytes: current timestamp as u64 little-endian - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("System time before UNIX epoch") - .as_secs(); salt[..8].copy_from_slice(×tamp.to_le_bytes()); // Last 24 bytes: random nonce @@ -331,6 +327,8 @@ impl LpMessage { #[cfg(test)] mod tests { + use std::time::{SystemTime, UNIX_EPOCH}; + use super::*; use crate::LpPacket; use crate::packet::{LpHeader, TRAILER_LEN}; @@ -369,10 +367,14 @@ mod tests { #[test] fn test_client_hello_salt_generation() { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("System time before UNIX epoch") + .as_secs(); let client_key = [1u8; 32]; let client_ed25519_key = [2u8; 32]; - let hello1 = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key); - let hello2 = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key); + let hello1 = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key, timestamp); + let hello2 = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key, timestamp); // Different salts should be generated assert_ne!(hello1.salt, hello2.salt); @@ -385,9 +387,13 @@ mod tests { #[test] fn test_client_hello_timestamp_extraction() { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("System time before UNIX epoch") + .as_secs(); let client_key = [2u8; 32]; let client_ed25519_key = [3u8; 32]; - let hello = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key); + let hello = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key, timestamp); let timestamp = hello.extract_timestamp(); let now = std::time::SystemTime::now() @@ -401,9 +407,13 @@ mod tests { #[test] fn test_client_hello_salt_format() { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("System time before UNIX epoch") + .as_secs(); let client_key = [3u8; 32]; let client_ed25519_key = [4u8; 32]; - let hello = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key); + let hello = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key, timestamp); // First 8 bytes should be non-zero timestamp let timestamp_bytes = &hello.salt[..8]; diff --git a/common/nym-lp/src/replay/validator.rs b/common/nym-lp/src/replay/validator.rs index 1d842b4913..a2599e7286 100644 --- a/common/nym-lp/src/replay/validator.rs +++ b/common/nym-lp/src/replay/validator.rs @@ -114,19 +114,17 @@ impl ReceivingKeyCounterValidator { let duplicate = self.check_bit_branchless(counter); - // Using Option to avoid early returns let result = if is_growing { - Some(Ok(())) + Ok(()) } else if too_far_back { - Some(Err(ReplayError::OutOfWindow)) + Err(ReplayError::OutOfWindow) } else if duplicate { - Some(Err(ReplayError::DuplicateCounter)) + Err(ReplayError::DuplicateCounter) } else { - Some(Ok(())) + Ok(()) }; - // Unwrap the option (always Some) - result.unwrap() + result } /// Special case function for clearing the entire bitmap diff --git a/common/nym-lp/src/session.rs b/common/nym-lp/src/session.rs index 7399bb8e39..9275e9be95 100644 --- a/common/nym-lp/src/session.rs +++ b/common/nym-lp/src/session.rs @@ -723,8 +723,8 @@ impl LpSession { /// /// # Returns /// - /// * `Ok(None)` if no message needs to be sent currently (e.g., waiting for peer, or handshake complete). - /// * `Err(LpError)` if there's an error within the Noise protocol or PSQ. + /// * `None` if no message needs to be sent currently (e.g., waiting for peer, or handshake complete). + /// * `Some(LpError)` if there's an error within the Noise protocol or PSQ. pub fn prepare_handshake_message(&self) -> Option> { let mut noise_state = self.noise_state.lock(); diff --git a/common/wireguard/Cargo.toml b/common/wireguard/Cargo.toml index 7d887b7394..8e1d63fd89 100644 --- a/common/wireguard/Cargo.toml +++ b/common/wireguard/Cargo.toml @@ -16,7 +16,6 @@ defguard_wireguard_rs = { workspace = true } futures = { workspace = true } ip_network = { workspace = true } ipnetwork = { workspace = true } -log.workspace = true rand = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "net", "io-util"] } diff --git a/docs/LP_DEPLOYMENT.md b/docs/LP_DEPLOYMENT.md deleted file mode 100644 index a3b88c79fd..0000000000 --- a/docs/LP_DEPLOYMENT.md +++ /dev/null @@ -1,845 +0,0 @@ -# LP (Lewes Protocol) Deployment Guide - -## Prerequisites - -### System Requirements - -**Minimum:** -- CPU: 2 cores (x86_64 or ARM64) -- RAM: 4 GB -- Network: 100 Mbps -- Disk: 20 GB SSD - -**Recommended:** -- CPU: 4+ cores with AVX2/NEON support (for SIMD optimizations) -- RAM: 8+ GB -- Network: 1 Gbps -- Disk: 50+ GB NVMe SSD - -### Software Dependencies - -```bash -# Ubuntu/Debian -sudo apt-get update -sudo apt-get install -y \ - build-essential \ - pkg-config \ - libssl-dev \ - postgresql \ - wireguard - -# macOS -brew install \ - postgresql \ - wireguard-tools -``` - -## Gateway Setup - -### 1. Enable LP in Configuration - -Edit your gateway configuration file (typically `~/.nym/gateways//config/config.toml`): - -```toml -[lp] -# Enable the LP listener -enabled = true - -# Bind address (0.0.0.0 for all interfaces, 127.0.0.1 for localhost only) -bind_address = "0.0.0.0" - -# Control port for LP handshake and registration -control_port = 41264 - -# Data port (reserved for future use, not currently used) -data_port = 51264 - -# Maximum concurrent LP connections -# Adjust based on expected load and available memory (~5 KB per connection) -max_connections = 10000 - -# Timestamp tolerance in seconds -# ClientHello messages with timestamps outside this window are rejected -# Balance security (smaller window) vs clock skew tolerance (larger window) -timestamp_tolerance_secs = 30 - -# IMPORTANT: ONLY for testing! Never enable in production -use_mock_ecash = false -``` - -### 2. Network Configuration - -#### Firewall Rules - -```bash -# Allow LP control port -sudo ufw allow 41264/tcp comment 'Nym LP control port' - -# Optional: Rate limiting using iptables -sudo iptables -A INPUT -p tcp --dport 41264 -m state --state NEW \ - -m recent --set --name LP_CONN_LIMIT - -sudo iptables -A INPUT -p tcp --dport 41264 -m state --state NEW \ - -m recent --update --seconds 60 --hitcount 100 --name LP_CONN_LIMIT \ - -j DROP -``` - -#### NAT/Port Forwarding - -If your gateway is behind NAT, forward port 41264: - -```bash -# Example for router at 192.168.1.1 -# Forward external:41264 -> internal:41264 (TCP) - -# Verify with: -nc -zv 41264 -``` - -### 3. LP Keypair Generation - -LP uses separate keypairs from the gateway's main identity. Generate on first run: - -```bash -# Start gateway (will auto-generate LP keypair if missing) -./nym-node run --mode gateway --id - -# LP keypair stored at: -# ~/.nym/gateways//keys/lp_x25519.pem -``` - -**Key Storage Security:** - -```bash -# Restrict key file permissions -chmod 600 ~/.nym/gateways//keys/lp_x25519.pem - -# Backup keys securely (encrypted) -gpg -c ~/.nym/gateways//keys/lp_x25519.pem -# Store lp_x25519.pem.gpg in secure location -``` - -### 4. Database Configuration - -LP requires PostgreSQL for credential tracking: - -```bash -# Create database -sudo -u postgres createdb nym_gateway - -# Create user -sudo -u postgres psql -c "CREATE USER nym_gateway WITH PASSWORD 'strong_password';" -sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE nym_gateway TO nym_gateway;" - -# Configure in gateway config -[storage] -database_url = "postgresql://nym_gateway:strong_password@localhost/nym_gateway" -``` - -**Database Maintenance:** - -```sql --- Index for nullifier lookups (critical for performance) -CREATE INDEX idx_nullifiers ON spent_credentials(nullifier); - --- Periodic cleanup of old nullifiers (run daily via cron) -DELETE FROM spent_credentials WHERE expiry < NOW() - INTERVAL '30 days'; - --- Vacuum to reclaim space -VACUUM ANALYZE spent_credentials; -``` - -### 5. WireGuard Configuration (for dVPN mode) - -```bash -# Enable WireGuard kernel module -sudo modprobe wireguard - -# Verify loaded -lsmod | grep wireguard - -# Generate gateway WireGuard keys -wg genkey | tee wg_private.key | wg pubkey > wg_public.key -chmod 600 wg_private.key - -# Configure in gateway config -[wireguard] -enabled = true -private_key_path = "/path/to/wg_private.key" -listen_port = 51820 -interface_name = "wg-nym" -subnet = "10.0.0.0/8" -``` - -**WireGuard Interface Setup:** - -```bash -# Create interface -sudo ip link add dev wg-nym type wireguard - -# Configure interface -sudo ip addr add 10.0.0.1/8 dev wg-nym -sudo ip link set wg-nym up - -# Enable IP forwarding -sudo sysctl -w net.ipv4.ip_forward=1 -echo "net.ipv4.ip_forward=1" | sudo tee -a /etc/sysctl.conf - -# NAT for WireGuard clients -sudo iptables -t nat -A POSTROUTING -s 10.0.0.0/8 -o eth0 -j MASQUERADE -``` - -### 6. Monitoring Setup - -#### Prometheus Metrics - -LP exposes metrics on the gateway's metrics endpoint (default: `:8080/metrics`): - -```yaml -# prometheus.yml -scrape_configs: - - job_name: 'nym-gateway-lp' - static_configs: - - targets: ['gateway-host:8080'] - metric_relabel_configs: - # Focus on LP metrics - - source_labels: [__name__] - regex: 'lp_.*' - action: keep -``` - -**Key Metrics:** - -```promql -# Connection metrics -nym_gateway_active_lp_connections # Current active connections -rate(nym_gateway_lp_connections_total[5m]) # Connection rate -rate(nym_gateway_lp_connections_completed_with_error[5m]) # Error rate - -# Handshake metrics -rate(nym_gateway_lp_handshakes_success[5m]) -rate(nym_gateway_lp_handshakes_failed[5m]) -histogram_quantile(0.95, nym_gateway_lp_handshake_duration_seconds) - -# Registration metrics -rate(nym_gateway_lp_registration_success_total[5m]) -rate(nym_gateway_lp_registration_failed_total[5m]) -histogram_quantile(0.95, nym_gateway_lp_registration_duration_seconds) - -# Credential metrics -rate(nym_gateway_lp_credential_verification_failed[5m]) -nym_gateway_lp_bandwidth_allocated_bytes_total - -# Error metrics -rate(nym_gateway_lp_errors_handshake[5m]) -rate(nym_gateway_lp_errors_timestamp_too_old[5m]) -rate(nym_gateway_lp_errors_wg_peer_registration[5m]) -``` - -#### Grafana Dashboard - -Import dashboard JSON (create and export after setup): - -```json -{ - "dashboard": { - "title": "Nym Gateway - LP Protocol", - "panels": [ - { - "title": "Active Connections", - "targets": [ - { - "expr": "nym_gateway_active_lp_connections" - } - ] - }, - { - "title": "Registration Success Rate", - "targets": [ - { - "expr": "rate(nym_gateway_lp_registration_success_total[5m]) / (rate(nym_gateway_lp_registration_success_total[5m]) + rate(nym_gateway_lp_registration_failed_total[5m]))" - } - ] - } - ] - } -} -``` - -#### Alert Rules - -```yaml -# alerting_rules.yml -groups: - - name: lp_alerts - interval: 30s - rules: - # High connection rejection rate - - alert: LPHighRejectionRate - expr: rate(nym_gateway_lp_connections_completed_with_error[5m]) > 10 - for: 5m - labels: - severity: warning - annotations: - summary: "High LP connection rejection rate" - description: "Gateway {{ $labels.instance }} rejecting {{ $value }} connections/sec" - - # Handshake failure rate > 5% - - alert: LPHandshakeFailures - expr: | - rate(nym_gateway_lp_handshakes_failed[5m]) / - (rate(nym_gateway_lp_handshakes_success[5m]) + rate(nym_gateway_lp_handshakes_failed[5m])) - > 0.05 - for: 10m - labels: - severity: warning - annotations: - summary: "High LP handshake failure rate" - - # Credential verification issues - - alert: LPCredentialVerificationFailures - expr: rate(nym_gateway_lp_credential_verification_failed[5m]) > 50 - for: 5m - labels: - severity: critical - annotations: - summary: "High credential verification failure rate" - - # High latency - - alert: LPHighLatency - expr: histogram_quantile(0.95, nym_gateway_lp_registration_duration_seconds) > 5 - for: 10m - labels: - severity: warning - annotations: - summary: "LP registration latency is high" -``` - -## Client Configuration - -### 1. Obtain Gateway LP Public Key - -```bash -# Query gateway descriptor -curl https://validator.nymtech.net/api/v1/gateways/ - -# Extract LP public key from response -{ - "gateway": { - "identity_key": "...", - "lp_public_key": "base64-encoded-x25519-public-key", - "host": "1.2.3.4", - "lp_port": 41264 - } -} -``` - -### 2. Initialize Registration Client - -```rust -use nym_registration_client::{RegistrationClient, RegistrationMode}; - -// Create client -let mut client = RegistrationClient::builder() - .gateway_identity("gateway-identity-key") - .gateway_lp_public_key(gateway_lp_pubkey) - .gateway_lp_address("1.2.3.4:41264") - .mode(RegistrationMode::Lp) - .build()?; - -// Perform registration -let result = client.register_lp( - credential, // E-cash credential - RegistrationMode::Dvpn { - wg_public_key: client_wg_pubkey, - } -).await?; - -match result { - LpRegistrationResult::Success { gateway_data, .. } => { - // Use gateway_data to configure WireGuard tunnel - } - LpRegistrationResult::Error { code, message } => { - eprintln!("Registration failed: {}", message); - } -} -``` - -## Testing - -### Local Testing Environment - -#### 1. Start Mock Gateway - -```bash -# Use mock e-cash verifier (accepts any credential) -export LP_USE_MOCK_ECASH=true - -# Start gateway in dev mode -./nym-node run --mode gateway --id test-gateway -``` - -#### 2. Test LP Connection - -```bash -# Test TCP connectivity -nc -zv localhost 41264 - -# Test with openssl (basic TLS check - won't work as LP uses Noise) -timeout 5 openssl s_client -connect localhost:41264 < /dev/null -# Expected: Connection closes (Noise != TLS) -``` - -#### 3. Run Integration Tests - -```bash -# Run full LP registration test suite -cargo test --test lp_integration -- --nocapture - -# Run specific test -cargo test --test lp_integration test_dvpn_registration_success -``` - -### Production Testing - -#### Health Check Script - -```bash -#!/bin/bash -# lp_health_check.sh - -GATEWAY_HOST="${1:-localhost}" -GATEWAY_PORT="${2:-41264}" - -# Check TCP connectivity -if ! timeout 5 nc -zv "$GATEWAY_HOST" "$GATEWAY_PORT" 2>&1 | grep -q succeeded; then - echo "CRITICAL: Cannot connect to LP port $GATEWAY_PORT" - exit 2 -fi - -# Check metrics endpoint -ACTIVE_CONNS=$(curl -s "http://$GATEWAY_HOST:8080/metrics" | \ - grep "^nym_gateway_active_lp_connections" | awk '{print $2}') - -if [ -z "$ACTIVE_CONNS" ]; then - echo "WARNING: Cannot read metrics" - exit 1 -fi - -echo "OK: LP listener responding, $ACTIVE_CONNS active connections" -exit 0 -``` - -#### Load Testing - -```bash -# Install tool -cargo install --git https://github.com/nymtech/nym tools/nym-lp-load-test - -# Run load test (1000 concurrent registrations) -nym-lp-load-test \ - --gateway "1.2.3.4:41264" \ - --gateway-pubkey "base64-key" \ - --concurrent 1000 \ - --duration 60s -``` - -## Troubleshooting - -### Connection Refused - -**Symptom:** `Connection refused` when connecting to port 41264 - -**Diagnosis:** -```bash -# Check if LP listener is running -sudo netstat -tlnp | grep 41264 - -# Check gateway logs -journalctl -u nym-gateway -f | grep LP - -# Check firewall -sudo ufw status | grep 41264 -``` - -**Solutions:** -1. Ensure `lp.enabled = true` in config -2. Check bind address (`0.0.0.0` vs `127.0.0.1`) -3. Open firewall port: `sudo ufw allow 41264/tcp` -4. Restart gateway after config changes - -### Handshake Failures - -**Symptom:** `lp_handshakes_failed` metric increasing - -**Diagnosis:** -```bash -# Check error logs -journalctl -u nym-gateway | grep "LP.*handshake.*failed" - -# Common errors: -# - "Noise decryption error" → Wrong keys or MITM -# - "Timestamp too old" → Clock skew > 30s -# - "Replay detected" → Duplicate connection attempt -``` - -**Solutions:** -1. **Noise errors**: Verify client has correct gateway LP public key -2. **Timestamp errors**: Sync clocks with NTP - ```bash - sudo timedatectl set-ntp true - sudo timedatectl status - ``` -3. **Replay errors**: Check for connection retry logic creating duplicates - -### Credential Verification Failures - -**Symptom:** `lp_credential_verification_failed` metric high - -**Diagnosis:** -```bash -# Check database connectivity -psql -U nym_gateway -d nym_gateway -c "SELECT COUNT(*) FROM spent_credentials;" - -# Check ecash manager logs -journalctl -u nym-gateway | grep -i credential -``` - -**Solutions:** -1. **Database errors**: Check PostgreSQL is running and accessible -2. **Signature errors**: Verify ecash contract address is correct -3. **Expired credentials**: Client needs to obtain fresh credentials -4. **Nullifier collision**: Credential already used (check `spent_credentials` table) - -### High Latency - -**Symptom:** `lp_registration_duration_seconds` p95 > 5 seconds - -**Diagnosis:** -```bash -# Check database query performance -psql -U nym_gateway -d nym_gateway -c "EXPLAIN ANALYZE SELECT * FROM spent_credentials WHERE nullifier = 'test';" - -# Check system load -top -bn1 | head -20 -iostat -x 1 5 -``` - -**Solutions:** -1. **Database slow**: Add index on nullifier column - ```sql - CREATE INDEX CONCURRENTLY idx_nullifiers ON spent_credentials(nullifier); - ``` -2. **CPU bound**: Check if SIMD is enabled - ```bash - # Check for AVX2 support - grep avx2 /proc/cpuinfo - # Rebuild with target-cpu=native - RUSTFLAGS="-C target-cpu=native" cargo build --release - ``` -3. **Network latency**: Check RTT to gateway - ```bash - ping -c 10 gateway-host - mtr gateway-host - ``` - -### Connection Limit Reached - -**Symptom:** `lp_connections_completed_with_error` high, logs show "connection limit exceeded" - -**Diagnosis:** -```bash -# Check active connections -curl -s http://localhost:8080/metrics | grep active_lp_connections - -# Check system limits -ulimit -n # File descriptors per process -sysctl net.ipv4.ip_local_port_range -``` - -**Solutions:** -1. **Increase max_connections** in config: - ```toml - [lp] - max_connections = 20000 # Increased from 10000 - ``` -2. **Increase system limits**: - ```bash - # /etc/security/limits.conf - nym-gateway soft nofile 65536 - nym-gateway hard nofile 65536 - - # /etc/sysctl.conf - net.ipv4.ip_local_port_range = 1024 65535 - net.core.somaxconn = 4096 - - # Apply - sudo sysctl -p - ``` -3. **Check for connection leaks**: - ```bash - # Connections in CLOSE_WAIT (indicates app not closing properly) - netstat -an | grep 41264 | grep CLOSE_WAIT | wc -l - ``` - -## Performance Tuning - -### TCP Tuning - -```bash -# /etc/sysctl.conf - Optimize for many concurrent connections - -# Increase max backlog -net.core.somaxconn = 4096 -net.ipv4.tcp_max_syn_backlog = 8192 - -# Faster TCP timeouts -net.ipv4.tcp_fin_timeout = 15 -net.ipv4.tcp_keepalive_time = 300 -net.ipv4.tcp_keepalive_probes = 5 -net.ipv4.tcp_keepalive_intvl = 15 - -# Optimize buffer sizes -net.core.rmem_max = 134217728 -net.core.wmem_max = 134217728 -net.ipv4.tcp_rmem = 4096 87380 67108864 -net.ipv4.tcp_wmem = 4096 65536 67108864 - -# Enable TCP Fast Open -net.ipv4.tcp_fastopen = 3 - -# Apply -sudo sysctl -p -``` - -### SIMD Optimization - -Ensure gateway is built with CPU-specific optimizations: - -```bash -# Check current CPU features -rustc --print target-features - -# Build with native CPU features (enables AVX2, SSE4, etc.) -RUSTFLAGS="-C target-cpu=native" cargo build --release -p nym-node - -# Verify SIMD is used (check binary for AVX2 instructions) -objdump -d target/release/nym-node | grep vpmovzxbw | wc -l -# Non-zero result means AVX2 is being used -``` - -### Database Optimization - -```sql --- Analyze query performance -EXPLAIN ANALYZE SELECT * FROM spent_credentials WHERE nullifier = 'xyz'; - --- Essential indexes -CREATE INDEX CONCURRENTLY idx_spent_credentials_nullifier ON spent_credentials(nullifier); -CREATE INDEX CONCURRENTLY idx_spent_credentials_expiry ON spent_credentials(expiry); - --- Optimize PostgreSQL config (postgresql.conf) --- Adjust based on available RAM -shared_buffers = 2GB # 25% of RAM -effective_cache_size = 6GB # 75% of RAM -maintenance_work_mem = 512MB -work_mem = 64MB -max_connections = 200 - --- Enable query planning optimizations -random_page_cost = 1.1 # SSD-optimized -effective_io_concurrency = 200 # SSD-optimized - --- Restart PostgreSQL after config changes -sudo systemctl restart postgresql -``` - -## Security Hardening - -### 1. Principle of Least Privilege - -```bash -# Run gateway as dedicated user (not root) -sudo useradd -r -s /bin/false nym-gateway - -# Set file ownership -sudo chown -R nym-gateway:nym-gateway /home/nym-gateway/.nym - -# Systemd service with restrictions -[Service] -User=nym-gateway -Group=nym-gateway -NoNewPrivileges=true -PrivateTmp=true -ProtectSystem=strict -ProtectHome=true -ReadWritePaths=/home/nym-gateway/.nym -``` - -### 2. TLS for Metrics Endpoint - -```bash -# Use reverse proxy (nginx) for metrics -server { - listen 443 ssl http2; - server_name metrics.your-gateway.com; - - ssl_certificate /etc/letsencrypt/live/metrics.your-gateway.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/metrics.your-gateway.com/privkey.pem; - - location /metrics { - proxy_pass http://127.0.0.1:8080/metrics; - # Authentication - auth_basic "Metrics"; - auth_basic_user_file /etc/nginx/.htpasswd; - } -} -``` - -### 3. Key Rotation - -```bash -# Generate new LP keypair -./nym-node generate-lp-keypair --output new_lp_key.pem - -# Atomic key swap (minimizes downtime) -# 1. Stop gateway gracefully -systemctl stop nym-gateway - -# 2. Backup old key -cp ~/.nym/gateways//keys/lp_x25519.pem ~/.nym/gateways//keys/lp_x25519.pem.backup - -# 3. Install new key -mv new_lp_key.pem ~/.nym/gateways//keys/lp_x25519.pem -chmod 600 ~/.nym/gateways//keys/lp_x25519.pem - -# 4. Restart gateway -systemctl start nym-gateway - -# 5. Update gateway descriptor (publishes new public key) -# This happens automatically on restart -``` - -## Maintenance - -### Regular Tasks - -**Daily:** -- Monitor metrics for anomalies -- Check error logs for new patterns -- Verify disk space for database growth - -**Weekly:** -- Vacuum database to reclaim space - ```sql - VACUUM ANALYZE spent_credentials; - ``` -- Review and archive old logs - ```bash - journalctl --vacuum-time=7d - ``` - -**Monthly:** -- Update dependencies (security patches) - ```bash - cargo update - cargo audit - cargo build --release - ``` -- Backup configuration and keys -- Review and update alert thresholds based on traffic patterns - -**Quarterly:** -- Key rotation (if security policy requires) -- Performance review and capacity planning -- Security audit of configuration - -### Backup Procedure - -```bash -#!/bin/bash -# backup_lp.sh - -BACKUP_DIR="/backup/nym-gateway/$(date +%Y%m%d)" -mkdir -p "$BACKUP_DIR" - -# Backup keys -cp -r ~/.nym/gateways//keys "$BACKUP_DIR/" - -# Backup config -cp ~/.nym/gateways//config/config.toml "$BACKUP_DIR/" - -# Backup database -pg_dump -U nym_gateway nym_gateway | gzip > "$BACKUP_DIR/database.sql.gz" - -# Encrypt and upload -tar -czf - "$BACKUP_DIR" | gpg -c | aws s3 cp - s3://backups/nym-gateway-$(date +%Y%m%d).tar.gz.gpg -``` - -### Upgrade Procedure - -```bash -# 1. Backup current installation -./backup_lp.sh - -# 2. Download new version -wget https://github.com/nymtech/nym/releases/download/vX.Y.Z/nym-node - -# 3. Stop gateway -systemctl stop nym-gateway - -# 4. Replace binary -sudo mv nym-node /usr/local/bin/nym-node -sudo chmod +x /usr/local/bin/nym-node - -# 5. Run migrations (if any) -nym-node migrate --config ~/.nym/gateways//config/config.toml - -# 6. Start gateway -systemctl start nym-gateway - -# 7. Verify -curl http://localhost:8080/metrics | grep lp_connections_total -journalctl -u nym-gateway -f -``` - -## Reference - -### Default Ports - -| Port | Protocol | Purpose | -|------|----------|---------| -| 41264 | TCP | LP control plane (handshake + registration) | -| 51264 | Reserved | LP data plane (future use) | -| 51820 | UDP | WireGuard (for dVPN mode) | -| 8080 | HTTP | Metrics endpoint | - -### File Locations - -| File | Location | Purpose | -|------|----------|---------| -| Config | `~/.nym/gateways//config/config.toml` | Main configuration | -| LP Private Key | `~/.nym/gateways//keys/lp_x25519.pem` | LP static private key | -| WG Private Key | `~/.nym/gateways//keys/wg_private.key` | WireGuard private key | -| Database | PostgreSQL database | Nullifier tracking | -| Logs | `journalctl -u nym-gateway` | System logs | - -### Useful Commands - -```bash -# Check LP listener status -sudo netstat -tlnp | grep 41264 - -# View real-time logs -journalctl -u nym-gateway -f | grep LP - -# Query metrics -curl -s http://localhost:8080/metrics | grep "^lp_" - -# Check active connections -ss -tn sport = :41264 | wc -l - -# Test credential verification -psql -U nym_gateway -d nym_gateway -c \ - "SELECT COUNT(*) FROM spent_credentials WHERE created_at > NOW() - INTERVAL '1 hour';" -``` diff --git a/gateway/src/node/lp_listener/handler.rs b/gateway/src/node/lp_listener/handler.rs index 2cc8a1f162..5b3eb64e3b 100644 --- a/gateway/src/node/lp_listener/handler.rs +++ b/gateway/src/node/lp_listener/handler.rs @@ -1482,12 +1482,17 @@ mod tests { use nym_lp::message::ClientHelloData; use tokio::net::{TcpListener, TcpStream}; + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("System time before UNIX epoch") + .as_secs(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let client_key = [7u8; 32]; let client_ed25519_key = [8u8; 32]; - let hello_data = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key); + let hello_data = ClientHelloData::new_with_fresh_salt(client_key, client_ed25519_key, timestamp); let expected_salt = hello_data.salt; // Clone salt before moving hello_data let server_task = tokio::spawn(async move { @@ -1530,6 +1535,11 @@ mod tests { async fn test_receive_client_hello_valid() { use tokio::net::{TcpListener, TcpStream}; + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("System time before UNIX epoch") + .as_secs(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -1553,6 +1563,7 @@ mod tests { let hello_data = ClientHelloData::new_with_fresh_salt( client_x25519_public.to_bytes(), client_ed25519_keypair.public_key().to_bytes(), + timestamp ); let packet = LpPacket::new( LpHeader { @@ -1585,6 +1596,11 @@ mod tests { use std::time::{SystemTime, UNIX_EPOCH}; use tokio::net::{TcpListener, TcpStream}; + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("System time before UNIX epoch") + .as_secs(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -1608,6 +1624,7 @@ mod tests { let mut hello_data = ClientHelloData::new_with_fresh_salt( client_x25519_public.to_bytes(), client_ed25519_keypair.public_key().to_bytes(), + timestamp, ); // Manually set timestamp to be very old (100 seconds ago) diff --git a/nym-registration-client/src/lp_client/client.rs b/nym-registration-client/src/lp_client/client.rs index e20795fd33..074e3cfef1 100644 --- a/nym-registration-client/src/lp_client/client.rs +++ b/nym-registration-client/src/lp_client/client.rs @@ -17,6 +17,7 @@ use nym_registration_common::{GatewayData, LpRegistrationRequest, LpRegistration use nym_wireguard_types::PeerPublicKey; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; @@ -180,10 +181,16 @@ impl LpRegistrationClient { LpClientError::Crypto(format!("Failed to derive X25519 public key: {}", e)) })?; + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("System time before UNIX epoch") + .as_secs(); + // Step 2: Generate ClientHelloData with fresh salt and both public keys let client_hello_data = nym_lp::ClientHelloData::new_with_fresh_salt( client_x25519_public.to_bytes(), self.local_ed25519_keypair.public_key().to_bytes(), + timestamp, ); let salt = client_hello_data.salt; let receiver_index = client_hello_data.receiver_index; diff --git a/nym-registration-client/src/lp_client/nested_session.rs b/nym-registration-client/src/lp_client/nested_session.rs index e9fcbebab2..3b07d5f1af 100644 --- a/nym-registration-client/src/lp_client/nested_session.rs +++ b/nym-registration-client/src/lp_client/nested_session.rs @@ -32,6 +32,7 @@ use nym_registration_common::{GatewayData, LpRegistrationRequest, LpRegistration use nym_wireguard_types::PeerPublicKey; use std::net::IpAddr; use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; /// Create explicit bincode options for consistent serialization across versions. /// @@ -131,10 +132,16 @@ impl NestedLpSession { LpClientError::Crypto(format!("Failed to derive X25519 public key: {}", e)) })?; + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("System time before UNIX epoch") + .as_secs(); + // Step 2: Generate ClientHello for exit gateway let client_hello_data = nym_lp::ClientHelloData::new_with_fresh_salt( client_x25519_public.to_bytes(), self.client_keypair.public_key().to_bytes(), + timestamp, ); let salt = client_hello_data.salt; let receiver_index = client_hello_data.receiver_index;