diff --git a/Cargo.toml b/Cargo.toml index aa79198a..3c2cd638 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ path = "src/main.rs" [lib] name="grim" -crate-type = ["cdylib","rlib"] +crate-type = ["rlib"] # Desktop/CI release binaries ship stripped of debug symbols — the nym + nostr + # grin tree leaves a large symbol table that's dead weight for users (~16 MB on diff --git a/src/nostr/pool.rs b/src/nostr/pool.rs index 1fb99407..bb21b57a 100644 --- a/src/nostr/pool.rs +++ b/src/nostr/pool.rs @@ -63,10 +63,11 @@ const MIN_BACKDATE_SECS: u64 = 172_800; const PINNED_POOL: &str = r#"{ "version": 1, "updated": "2026-07-02", - "notes": "Goblin wallet relay candidate pool. Clients verify each entry locally (NIP-11 probe) before use. Requirements: max_message_length >= 131072, no payment or auth required for writes, tolerates NIP-59 backdating. The optional per-relay 'exit' is that operator's co-located scoped mixnet exit; it is intentionally UNSET for now because bootstrapping a second mixnet client blocks first-connect on a cold start.", + "notes": "Goblin wallet relay candidate pool. Clients verify each entry locally (NIP-11 probe) before use. Requirements: max_message_length >= 131072, no payment or auth required for writes, tolerates NIP-59 backdating. The optional per-relay 'exit' is that operator's co-located scoped mixnet exit (Recipient address): a MixnetStream the wallet dials directly to reach the relay with no public DNS and no public IPR — the fast money path.", "min_message_length": 131072, "relays": [ - { "url": "wss://relay.goblin.st", "roles": ["dm", "discovery"], "vetted": "2026-07-01" }, + { "url": "wss://relay.floonet.dev", "roles": ["dm", "discovery"], "vetted": "2026-07-02", "exit": "EqbUPt7aYkar2CTmjBVnyWaKzb2WT8NdojUGXU4mrfNG.AF5YCD8hgEUqByamrPqZz72h7GE599LbqQrhaew9bBip@HfyUPUv4z8uMQoZYuZGMWf6oe2vaKBVPrfgHk6WvwFPe" }, + { "url": "wss://relay.goblin.st", "roles": ["dm", "discovery"], "vetted": "2026-07-01", "exit": "4XPnpmFdieZBY1BM2jU9Qn915v5RGz58ywpgQhuFKBao.8NMrW1i4VaPhY6qhV7supid7P1YcWJ9mGZBKjGEuqN9U@B8bX5x5yKa7oQMCNioLS9seYwNCio3U9jYPxgCZoKjk5" }, { "url": "wss://relay.primal.net", "roles": ["dm"], "vetted": "2026-07-01" }, { "url": "wss://relay.damus.io", "roles": ["dm"], "vetted": "2026-07-01" }, { "url": "wss://nos.lol", "roles": ["dm"], "vetted": "2026-07-01" }, @@ -91,6 +92,18 @@ pub struct PoolRelay { /// Last-vetted date; presence marks the entry as vetted. #[serde(default)] pub vetted: Option, + /// This relay operator's CO-LOCATED Nym exit address, when they run one (the + /// bundled floonet-rs / floonet-strfry `exit = true` feature). It is a Nym + /// `Recipient` (`.@`) for a SCOPED MixnetStream proxy + /// that forwards ONLY to this relay — so the wallet can reach the relay over + /// the mixnet WITHOUT public DNS and WITHOUT depending on a public IPR exit + /// (the anchor; see [`crate::nym::nymproc`]). Absent → this relay is reached + /// the old way (public-IPR smolmix + in-tunnel DoT). Carried in the pinned + /// pool so the money-path default relay's exit bootstraps OFFLINE, before any + /// network — breaking the chicken-and-egg of learning it over the very path + /// it is meant to replace. + #[serde(default)] + pub exit: Option, } impl PoolRelay { @@ -137,6 +150,46 @@ impl RelayPool { .map(|r| r.url.clone()) .collect() } + + /// The operator's co-located Nym exit address for `url`, if the pool + /// advertises one (url compared modulo a trailing slash). `None` → reach the + /// relay over the public-IPR path as before. This is how the wallet learns + /// the anchor exit for its money-path relay (see [`PoolRelay::exit`]). + pub fn exit_for(&self, url: &str) -> Option { + let want = url.trim_end_matches('/'); + self.relays + .iter() + .find(|r| r.url.trim_end_matches('/') == want) + .and_then(|r| r.exit.clone()) + .filter(|e| !e.trim().is_empty()) + } + + /// Like [`Self::exit_for`], but keyed on the HOSTNAME — the HTTP dial site + /// ([`crate::nym::request_once`]) knows only `host`, never the relay's ws + /// URL. HTTPS to a host whose relay advertises a co-located exit (its + /// NIP-11 probe, in practice) rides that exit too. + pub fn exit_for_host(&self, host: &str) -> Option { + self.relays + .iter() + .find(|r| { + url::Url::parse(&r.url) + .ok() + .and_then(|u| u.host_str().map(|h| h.eq_ignore_ascii_case(host))) + .unwrap_or(false) + }) + .and_then(|r| r.exit.clone()) + .filter(|e| !e.trim().is_empty()) + } + + /// Whether ANY relay in the pool advertises a co-located exit. The cold-start + /// sequencer ([`crate::nym::nymproc`]) reads this to decide whether to give + /// the scoped-exit client its bandwidth-grant head start before building the + /// public-IPR tunnel — no exit anywhere → no wait, unchanged behavior. + pub fn has_exit(&self) -> bool { + self.relays + .iter() + .any(|r| r.exit.as_deref().is_some_and(|e| !e.trim().is_empty())) + } } /// Disk path of the cached pool file. @@ -312,18 +365,64 @@ mod tests { let pool = RelayPool::parse(PINNED_POOL).expect("pinned pool must parse"); assert_eq!(pool.version, 1); assert_eq!(pool.min_message_length, MIN_MESSAGE_LENGTH); - assert_eq!(pool.relays.len(), 12); + assert_eq!(pool.relays.len(), 13); let dm = pool.dm_relays(); - assert_eq!(dm.len(), 10); + assert_eq!(dm.len(), 11); + assert!(dm.iter().any(|r| r.url == "wss://relay.floonet.dev")); assert!(dm.iter().any(|r| r.url == "wss://relay.goblin.st")); assert!(dm.iter().all(|r| r.vetted.is_some())); let disc = pool.discovery_relays(); - // relay.goblin.st carries both roles; the two indexers are discovery-only. - assert_eq!(disc.len(), 3); + // relay.floonet.dev + relay.goblin.st carry both roles; the two indexers + // are discovery-only. + assert_eq!(disc.len(), 4); assert!(disc.contains(&"wss://purplepag.es".to_string())); assert!(disc.contains(&"wss://indexer.coracle.social".to_string())); } + #[test] + fn exit_field_is_optional_and_looked_up_by_url() { + // The pinned pool advertises the money-path relay's co-located scoped + // exit (the .8 floonet-mixexit) so it bootstraps OFFLINE, before any + // network; every other relay is exit-less (reached over the tunnel). + let pinned = RelayPool::parse(PINNED_POOL).unwrap(); + assert!(pinned.has_exit()); + assert!(pinned.exit_for("wss://relay.goblin.st").is_some()); + assert!(pinned.exit_for("wss://nos.lol").is_none()); + + // A pool that DOES advertise an exit for one relay. + let pool = RelayPool::parse( + r#"{"version":1,"updated":"x","min_message_length":131072,"relays":[ + {"url":"wss://relay.goblin.st/","roles":["dm"],"exit":"aaa.bbb@ccc"}, + {"url":"wss://nos.lol","roles":["dm"]}, + {"url":"wss://blank.example","roles":["dm"],"exit":" "} + ]}"#, + ) + .unwrap(); + // Trailing-slash-insensitive lookup. + assert_eq!( + pool.exit_for("wss://relay.goblin.st"), + Some("aaa.bbb@ccc".to_string()) + ); + // No exit field → None; blank exit → None (treated as unset). + assert!(pool.exit_for("wss://nos.lol").is_none()); + assert!(pool.exit_for("wss://blank.example").is_none()); + // Unknown url → None. + assert!(pool.exit_for("wss://unknown.example").is_none()); + + // Host-keyed lookup (the HTTP dial site): same answers by hostname. + assert_eq!( + pool.exit_for_host("relay.goblin.st"), + Some("aaa.bbb@ccc".to_string()) + ); + assert_eq!( + pool.exit_for_host("RELAY.GOBLIN.ST"), + Some("aaa.bbb@ccc".to_string()) + ); + assert!(pool.exit_for_host("nos.lol").is_none()); + assert!(pool.exit_for_host("blank.example").is_none()); + assert!(pool.exit_for_host("unknown.example").is_none()); + } + #[test] fn pool_validation_rejects_bad_documents() { assert!(RelayPool::parse("not json").is_none()); @@ -391,6 +490,7 @@ mod tests { url: url.to_string(), roles: vec!["dm".to_string()], vetted: vetted.then(|| "2026-07-01".to_string()), + exit: None, }; vec![ mk("wss://a.example", false), @@ -415,6 +515,7 @@ mod tests { url: "wss://relay.goblin.st".to_string(), roles: vec!["dm".to_string()], vetted: Some("2026-07-01".to_string()), + exit: None, }); let order = weighted_order("wss://relay.goblin.st", &with_goblin, |_| 0); assert_eq!(order.len(), 4); diff --git a/src/nym/mod.rs b/src/nym/mod.rs index 09ed2c59..95e45b77 100644 --- a/src/nym/mod.rs +++ b/src/nym/mod.rs @@ -14,12 +14,16 @@ //! Nym mixnet transport. Everything Goblin sends — nostr relay traffic and //! every HTTP request (NIP-05, price, relay pool) — rides the 5-hop mixnet: -//! one in-process smolmix [`Tunnel`](smolmix::Tunnel) to an auto-selected -//! public IPR exit, so neither the payload nor the destination-in-flight ever -//! touches the clearnet. Hostnames resolve through the same tunnel too -//! ([`dns`], DoT — DNS-over-TLS), so nothing goes clearnet. The mixnet breaks -//! the sender↔receiver timing correlation that Mimblewimble's interactive -//! slate exchange otherwise leaks at the network layer. +//! by default one in-process smolmix [`Tunnel`](smolmix::Tunnel) to an +//! auto-selected public IPR exit, so neither the payload nor the +//! destination-in-flight ever touches the clearnet. Hostnames resolve through +//! the same tunnel too ([`dns`], DoT — DNS-over-TLS), so nothing goes +//! clearnet. MONEY-PATH ANCHOR: a host whose relay advertises a co-located +//! scoped exit in the pool is instead dialed over a MixnetStream straight to +//! that exit ([`streamexit`]) — no DNS and no public IPR at all — falling +//! back to the tunnel on any failure. The mixnet breaks the sender↔receiver +//! timing correlation that Mimblewimble's interactive slate exchange +//! otherwise leaks at the network layer. //! //! DNS reliability was the one weak spot: the original mix-dns sent UDP over the //! mixnet, and mixnet UDP loses packets — resolves stalled on multi-second @@ -31,6 +35,7 @@ pub mod dns; pub mod nymproc; +pub mod streamexit; pub mod transport; use std::sync::Arc; @@ -132,24 +137,43 @@ async fn request_once( let https = url.scheme() == "https"; let port = url.port().unwrap_or(if https { 443 } else { 80 }); - // Resolve the host over the tunnel (DoT — see dns), then dial that - // IP through the same tunnel so nothing (lookup or body) touches - // the clear. - let addr = dns::resolve(tunnel, &host, port).await?; - let tcp = match tunnel.tcp_connect(addr).await { - Ok(s) => s, - Err(e) => { - warn!("nym http: connect to {host} failed: {e}"); - return None; - } - }; - let io: Box = if https { - match tls_connect(&host, tcp).await { - Some(tls) => Box::new(tls), - None => return None, + // MONEY-PATH ANCHOR fork: HTTPS to a host whose relay advertises a + // co-located scoped Nym exit (its NIP-11 probe, in practice) rides a + // MixnetStream to that exit instead of the tunnel — no public DNS, no + // public IPR. Failure just falls through to the tunnel path below (anchor + // + fallback, never pin-only). + let exit_io = if https { + match crate::nostr::pool::load().exit_for_host(&host) { + Some(exit) => exit_connect(&host, &exit).await, + None => None, } } else { - Box::new(tcp) + None + }; + + let io: Box = match exit_io { + Some(io) => io, + None => { + // Resolve the host over the tunnel (DoT — see dns), then dial that + // IP through the same tunnel so nothing (lookup or body) touches + // the clear. + let addr = dns::resolve(tunnel, &host, port).await?; + let tcp = match tunnel.tcp_connect(addr).await { + Ok(s) => s, + Err(e) => { + warn!("nym http: connect to {host} failed: {e}"); + return None; + } + }; + if https { + match tls_connect(&host, tcp).await { + Some(tls) => Box::new(tls), + None => return None, + } + } else { + Box::new(tcp) + } + } }; let (mut sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(io)) @@ -202,6 +226,35 @@ async fn request_once( Some((status, bytes, location)) } +/// Try the scoped-exit egress for an HTTPS `host`: a MixnetStream to the +/// relay operator's exit ([`streamexit`]), then the SAME hostname-validated +/// [`tls_connect`] as the tunnel path — SNI = `host`, so the exit sees only +/// ciphertext. `None` (logged) on ANY failure, and the whole attempt is +/// bounded by the shared bootstrap cap — a dead exit costs seconds inside the +/// caller's [`HTTP_TIMEOUT`] budget, leaving room to fall back to the tunnel. +async fn exit_connect(host: &str, exit: &str) -> Option> { + let cap = nymproc::BOOTSTRAP_TIMEOUT; + let dial = async { + let stream = streamexit::open_stream(exit, cap) + .await + .map_err(|e| warn!("nym http: scoped exit for {host} unavailable: {e}")) + .ok()?; + let tls = tls_connect(host, stream).await?; + debug!("nym http: {host} riding its operator's scoped exit"); + Some(Box::new(tls) as Box) + }; + match tokio::time::timeout(cap, dial).await { + Ok(io) => io, + Err(_) => { + warn!( + "nym http: scoped exit dial for {host} exceeded {}s; falling back to the tunnel", + cap.as_secs() + ); + None + } + } +} + /// Everything hyper needs from the tunneled stream, boxable for the plain /// http / https split. trait Stream: AsyncRead + AsyncWrite + Send + Unpin {} diff --git a/src/nym/nymproc.rs b/src/nym/nymproc.rs index a7ed5759..31569a69 100644 --- a/src/nym/nymproc.rs +++ b/src/nym/nymproc.rs @@ -23,8 +23,12 @@ //! re-selects, so there is no single-exit SPOF. Hostnames resolve via //! [`super::dns`] over DoT through the same tunnel, so nothing touches clearnet. //! -//! This is the wallet's ONLY mixnet path — every relay and HTTP dial rides -//! this tunnel. +//! This is the FALLBACK / discovery-and-secondary-relay path. The MONEY-PATH +//! primary relay is reached over a SCOPED MixnetStream to a Floonet operator's +//! CO-LOCATED exit when the pool advertises one ([`crate::nostr::pool::PoolRelay::exit`]), +//! which needs no public DNS and no public IPR — see the streamexit egress +//! (design in ~/.claude/plans/floonet-nym-exit.md). That anchor+fallback split +//! is the "prefer our exit, never pin-only" rule at the transport level. //! //! Should smolmix ever regress, the fallback design (SOCKS5 network requester //! + ordered exit failover) is specified in the plan, section G14. @@ -186,6 +190,26 @@ fn run_tunnel() { // True while a FALLBACK (auto-selected) exit carries the traffic even // though an anchor is configured — makes the ANCHOR RECOVERED log honest. let mut fell_back = false; + // COLD-START SEQUENCING (money path first): if the pool advertises a + // co-located scoped exit, let ITS mixnet client grab its Nym free-tier + // bandwidth grant before this tunnel competes for one. Two ephemeral + // clients bootstrapping at once serialize on the grant (~1 min); waiting a + // bounded head-start for the exit client means only ONE bootstraps at a + // time, so the money-path relay connects in seconds and this tunnel + // (fallback / HTTP / discovery, all non-blocking) builds right after. No + // exit in the pool → no wait. Cold start only: on a later reselect the + // exit is long-ready, so `is_ready()` returns instantly. + if crate::nostr::pool::load().has_exit() { + let head_start = Instant::now(); + while !super::streamexit::is_ready() && head_start.elapsed() < EXIT_HEAD_START { + tokio::time::sleep(Duration::from_millis(200)).await; + } + info!( + "[timing] nym: tunnel bootstrap proceeding after {}ms exit head-start (exit ready: {})", + head_start.elapsed().as_millis(), + super::streamexit::is_ready() + ); + } loop { let started = Instant::now(); attempt += 1; @@ -402,9 +426,20 @@ const MIN_EXIT_LIFETIME: Duration = Duration::from_secs(20); /// re-select. A healthy gateway+IPR bootstrap completes in ~4-7s; without this /// cap a DEAD first pick blocked for ~74s (measured) on the Nym SDK's own /// "listening for connection response" timeout before we even got to reselect. -/// A few seconds of patience, not a minute. +/// A few seconds of patience, not a minute. Shared with the scoped-exit egress +/// ([`super::streamexit`]) as ITS dial cap, so both mixnet bootstraps fail +/// equally fast. pub(crate) const BOOTSTRAP_TIMEOUT: Duration = Duration::from_secs(20); +/// Cold-start head start for the scoped-exit client: the public-IPR tunnel waits +/// up to this long for [`super::streamexit::is_ready`] before it bootstraps, so +/// the money-path exit client claims its Nym free-tier bandwidth grant FIRST and +/// the two ephemeral clients don't serialize on the grant (~1 min otherwise; see +/// the cold-start sequencer in [`run_tunnel`] and the NOTE in +/// [`super::streamexit`]). Bounded so a missing/failed exit never holds the +/// tunnel more than briefly; the exit typically readies well inside it. +const EXIT_HEAD_START: Duration = Duration::from_secs(12); + /// Restore the pre-Build-98 watchdog (condemn on RELAY_GRACE of no-relay alone, /// no connectivity gate, no rebuild floor). Debug/measurement only — lets a cold /// run reproduce the old reselect loop for a BEFORE/AFTER comparison. Default diff --git a/src/nym/streamexit.rs b/src/nym/streamexit.rs index 3e7c565e..4824aa42 100644 --- a/src/nym/streamexit.rs +++ b/src/nym/streamexit.rs @@ -12,6 +12,236 @@ // See the License for the specific language governing permissions and // limitations under the License. -// removed: the scoped Nym exit egress was disabled (the pool no longer -// advertises any exit); the wallet's only mixnet path is the smolmix tunnel. -// This module is unreferenced and kept only as an empty placeholder. +//! Scoped-MixnetStream egress — the MONEY-PATH ANCHOR. When the relay pool +//! advertises a relay operator's CO-LOCATED Nym exit +//! ([`crate::nostr::pool::PoolRelay::exit`]), the wallet dials that exit +//! directly over the mixnet with a [`MixnetStream`]; the exit pipes the bytes +//! to its ONE configured relay. No public DNS, no public IPR — the two flaky +//! dependencies of the fallback path are gone from the money path. The exit is +//! scoped (it forwards nowhere else), so the wallet writes nothing but the TLS +//! ClientHello: the dial sites run the SAME hostname-validated TLS (SNI = the +//! relay host) + websocket/HTTP wrap over this stream as over the smolmix +//! tunnel's TCP stream, and the exit sees only ciphertext. +//! +//! ANCHOR + FALLBACK, never pin-only: every failure here (bad address, client +//! bootstrap, stream open, timeout) just returns `Err`, and the dial sites +//! ([`super::transport`], [`super::request_once`]) fall through to the +//! public-IPR tunnel ([`super::nymproc`]) — losing the operator's exit never +//! locks the wallet out. Server side: the bundled `floonet-mixexit` binary +//! (design in ~/.claude/plans/floonet-nym-exit.md). + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use log::{info, warn}; +use nym_sdk::mixnet::{MixnetClient, MixnetStream, Recipient}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::sync::Mutex; + +/// Everything the TLS/websocket layer needs from the egress stream. +pub trait ExitStream: AsyncRead + AsyncWrite + Send + Unpin {} +impl ExitStream for T {} + +/// The boxed transport stream handed to the TLS/websocket layer — the same +/// seat the smolmix tunnel's TCP stream occupies on the fallback path. +pub type BoxedStream = Box; + +/// After the Open is SENT, wait this long before handing back a writable +/// stream. `open_stream` returns once the Open message leaves the client, NOT +/// once the exit has `accept()`ed and wired its inbound half. But the caller +/// speaks first (TLS ClientHello over a raw-pipe exit), so a write landing in +/// that gap is dropped and the handshake stalls into a fallback. One mixnet +/// round of slack lets the exit be listening before the first byte. +/// ponytail: fixed settle (measured: 0s always stalls, 3s is reliable). The +/// exit pipes raw bytes to its relay, so it can't inject an accept-ack for the +/// client to wait on; if mixnet jitter ever makes 3s flaky, raise it. +const STREAM_SETTLE: Duration = Duration::from_secs(3); + +/// Process-lifetime mixnet client for the scoped-exit egress, lazily connected +/// on first use (mirrors the tunnel singleton in [`super::nymproc`]). +/// Ephemeral in-memory identity, like the tunnel — a fresh mixnet identity per +/// run. Behind an async mutex because `open_stream` needs `&mut`; a dead +/// client (cancelled shutdown token or a failed open) is dropped so the next +/// dial reconnects fresh. +static CLIENT: Mutex> = Mutex::const_new(None); + +/// True once the exit's `MixnetClient` has bootstrapped and is usable. The +/// cold-start sequencer in [`super::nymproc`] reads this to hold the public-IPR +/// tunnel's bootstrap until the exit client has its Nym bandwidth grant (see the +/// NOTE below), so the money path connects in seconds instead of a minute. +static READY: AtomicBool = AtomicBool::new(false); + +/// Whether the scoped-exit mixnet client is bootstrapped and usable. +pub fn is_ready() -> bool { + READY.load(Ordering::Relaxed) +} + +// NOTE ON COLD-START LATENCY (and its fix): the exit rides a SECOND ephemeral +// MixnetClient (separate from the smolmix tunnel). When BOTH clients bootstrap +// at once on a cold start they serialize on Nym free-tier bandwidth grants — so +// whichever dials second waits ~a minute for its grant. The money path must not +// be the loser of that race. Fix (see nymproc's cold-start sequencer): the exit +// client is allowed to grab its grant FIRST, and the tunnel's bootstrap waits a +// bounded head-start for `is_ready()` before it competes — so only ONE client +// bootstraps at a time and the money-path relay connects in seconds. The tunnel +// (fallback / HTTP / discovery, all non-blocking) comes up right after. A +// startup pre-warm of BOTH in parallel does NOT help (measured) — sequencing, +// not parallelism, is what removes the stall. Sharing ONE client for tunnel + +// exit would remove the second grant entirely but couples the robust exit to +// the tunnel's per-reselect client rebuild; deferred as a future upgrade. + +/// Open a scoped MixnetStream to `exit` — a pool-advertised Nym address +/// (`.@`) of a relay operator's co-located exit. The +/// whole dial (client bootstrap when cold + stream open) is capped at +/// `min(timeout, BOOTSTRAP_TIMEOUT)` so a stuck bootstrap fails FAST into the +/// caller's public-IPR fallback. NOTE: `open_stream` is fire-and-forget on the +/// mixnet — a DEAD exit still hands back a stream, and its death surfaces at +/// the caller's (timeout-bounded) TLS handshake, which doubles as the +/// liveness probe: no ServerHello through the pipe → fall back. +pub async fn open_stream(exit: &str, timeout: Duration) -> Result { + let recipient: Recipient = exit + .trim() + .parse() + .map_err(|e| format!("invalid exit address: {e}"))?; + let cap = timeout.min(super::nymproc::BOOTSTRAP_TIMEOUT); + let stream = match tokio::time::timeout(cap, open(recipient)).await { + Ok(result) => result?, + Err(_) => return Err(format!("exit dial exceeded {}s", cap.as_secs())), + }; + // Let the exit accept() + wire its inbound half before the caller writes. + tokio::time::sleep(STREAM_SETTLE).await; + Ok(Box::new(stream) as BoxedStream) +} + +/// Ensure the shared client is connected, then open a stream on it. +async fn open(recipient: Recipient) -> Result { + let mut guard = CLIENT.lock().await; + // A dead client (gateway dropped, hosting runtime gone) is discarded and + // rebuilt — the auto-reconnect-on-drop rule. + if guard + .as_ref() + .is_some_and(|c| c.cancellation_token().is_cancelled()) + { + warn!("nym: streamexit client died; reconnecting"); + *guard = None; + READY.store(false, Ordering::Relaxed); + } + if guard.is_none() { + let started = std::time::Instant::now(); + let client = MixnetClient::connect_new() + .await + .map_err(|e| format!("mixnet client bootstrap failed: {e}"))?; + info!( + "[timing] nym: streamexit client CONNECTED in {}ms", + started.elapsed().as_millis() + ); + *guard = Some(client); + READY.store(true, Ordering::Relaxed); + } + let client = guard.as_mut().expect("client ensured above"); + match client.open_stream(recipient, None).await { + Ok(stream) => Ok(stream), + Err(e) => { + // `open_stream` fails only LOCALLY (the client's input channel) — + // it never waits on the peer — so an error means the client itself + // is broken, not the exit. Drop it; the next dial reconnects. + *guard = None; + READY.store(false, Ordering::Relaxed); + Err(format!("open_stream failed: {e}")) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn bad_exit_address_fails_fast_without_touching_the_mixnet() { + // The address parse runs BEFORE any client bootstrap, so garbage from + // a hostile pool costs nothing and degrades to the fallback path. + let err = open_stream("not-a-recipient", Duration::from_secs(5)) + .await + .err() + .expect("garbage address must fail"); + assert!(err.contains("invalid exit address"), "got: {err}"); + } + + /// LIVE end-to-end smoke test of the money path against the DEPLOYED + /// floonet-mixexit (.8): dial the pinned pool's `exit` for relay.goblin.st + /// over the mixnet with the real [`open_stream`], run the SAME + /// hostname-validated TLS + websocket wrap the wallet uses + /// ([`super::super::transport`]), then send a nostr REQ and require the + /// relay to answer (EVENT/EOSE). Proves mixnet -> exit -> relay:443 -> + /// nostr actually carries traffic. Ignored (needs network + a cold mixnet + /// bootstrap). Run: + /// cargo test --lib nym::streamexit::tests::live_exit_roundtrip -- --ignored --nocapture + #[tokio::test] + #[ignore] + async fn live_exit_roundtrip() { + use futures::{SinkExt, StreamExt}; + use tokio_tungstenite::tungstenite::Message; + + // The app installs this at startup (src/lib.rs); an isolated test must + // too, or rustls 0.23 can't pick a provider for the TLS handshake. + let _ = rustls::crypto::ring::default_provider().install_default(); + + let exit = crate::nostr::pool::load() + .exit_for("wss://relay.floonet.dev") + .expect("pinned pool advertises the relay.floonet.dev exit"); + println!("dialing scoped exit {exit}"); + + // A cold ephemeral mixnet bootstrap can exceed the per-dial cap; the + // real wallet just falls back and retries, so retry until one dial wins. + let mut stream = None; + for attempt in 1..=6 { + let t = std::time::Instant::now(); + match open_stream(&exit, Duration::from_secs(90)).await { + Ok(s) => { + println!( + "open_stream OK on attempt {attempt} in {}ms", + t.elapsed().as_millis() + ); + stream = Some(s); + break; + } + Err(e) => println!( + "attempt {attempt} failed in {}ms: {e}", + t.elapsed().as_millis() + ), + } + } + let stream = stream.expect("exit stream opened within retries"); + + let url = "wss://relay.floonet.dev"; + let (mut ws, _resp) = tokio::time::timeout( + Duration::from_secs(45), + tokio_tungstenite::client_async_tls(url, stream), + ) + .await + .expect("TLS+ws handshake timed out (dead exit?)") + .expect("TLS+ws handshake through exit failed"); + println!("TLS+ws handshake through .8 exit OK"); + + ws.send(Message::Text( + r#"["REQ","smoke",{"kinds":[1],"limit":1}]"#.into(), + )) + .await + .expect("send REQ"); + + let reply = tokio::time::timeout(Duration::from_secs(30), ws.next()) + .await + .expect("relay reply timed out") + .expect("ws stream closed early") + .expect("ws frame error"); + let txt = match reply { + Message::Text(t) => t.to_string(), + other => format!("{other:?}"), + }; + println!("relay answered through exit: {txt}"); + assert!( + txt.contains("EVENT") || txt.contains("EOSE"), + "unexpected relay reply: {txt}" + ); + } +} diff --git a/src/nym/transport.rs b/src/nym/transport.rs index 92702071..93e7df06 100644 --- a/src/nym/transport.rs +++ b/src/nym/transport.rs @@ -13,10 +13,16 @@ // limitations under the License. //! WebSocket transport for the Nostr relay pool routed through the Nym -//! mixnet: Goblin's in-process smolmix tunnel — the relay host is resolved by -//! [`super::dns`], the TCP stream is opened via `tunnel.tcp_connect`. The TLS -//! (rustls, webpki roots) + websocket handshake runs over the mixnet-carried -//! stream, so the payload + in-flight destination never touch the clear. +//! mixnet, with TWO egresses picked per relay. ANCHOR: a relay whose pool +//! entry advertises its operator's co-located scoped exit +//! ([`crate::nostr::pool::PoolRelay::exit`]) is dialed over a MixnetStream +//! straight to that exit ([`super::streamexit`]) — no DNS, no public IPR. +//! FALLBACK (and every relay without an exit): Goblin's in-process smolmix +//! tunnel — the relay host is resolved by [`super::dns`], the TCP stream is +//! opened via `tunnel.tcp_connect`. Either way the SAME TLS (rustls, webpki +//! roots) + websocket handshake runs over the mixnet-carried stream, so the +//! payload + in-flight destination never touch the clear, and an exit failure +//! only ever falls back — never a lockout. use std::fmt; use std::pin::Pin; @@ -72,6 +78,34 @@ impl WebSocketTransport for NymWebSocketTransport { _ => 443, }); + // MONEY-PATH ANCHOR: when the pool advertises this relay + // operator's co-located scoped Nym exit, dial THROUGH it — a + // MixnetStream straight to the exit (which pipes to its one + // relay), no public DNS, no public IPR, no tunnel dependency. The + // TLS + websocket wrap inside is byte-for-byte the tunnel path's + // (same `client_async_tls`, SNI = the relay host), so the exit + // sees only ciphertext. ANY failure — bootstrap, open, handshake, + // timeout — falls through to the public-IPR tunnel dial below: + // anchor + fallback, never pin-only. + if let Some(exit) = crate::nostr::pool::load().exit_for(url.as_str()) { + let t_exit = std::time::Instant::now(); + match exit_connect(url, &exit, timeout).await { + Ok(parts) => { + log::info!( + "[timing] nym: relay {host} CONNECTED via scoped exit — \ + stream+tls+ws {}ms", + t_exit.elapsed().as_millis() + ); + return Ok(parts); + } + Err(e) => log::warn!( + "nym: scoped exit dial for {host} failed after {}ms ({e}); \ + falling back to the public-IPR tunnel", + t_exit.elapsed().as_millis() + ), + } + } + // The shared mixnet tunnel (lazy-started at app launch). let tunnel = crate::nym::nymproc::wait_for_tunnel(timeout) .await @@ -116,7 +150,33 @@ impl WebSocketTransport for NymWebSocketTransport { } } -/// Split a websocket into the pool's boxed sink/stream halves. +/// Dial `url` through the relay operator's scoped Nym exit `exit`: a +/// MixnetStream to the exit (which pipes to its one configured relay), then +/// the SAME hostname-validated TLS + websocket handshake as the tunnel path. +/// The handshake doubles as the exit liveness probe — `open_stream` is +/// fire-and-forget, so a dead exit surfaces here as a (bounded) timeout and +/// the caller falls back. +async fn exit_connect( + url: &Url, + exit: &str, + timeout: Duration, +) -> Result<(WebSocketSink, WebSocketStream), TransportError> { + let stream = crate::nym::streamexit::open_stream(exit, timeout) + .await + .map_err(terr)?; + let (ws, _response) = tokio::time::timeout( + timeout, + tokio_tungstenite::client_async_tls(url.as_str(), stream), + ) + .await + .map_err(|_| terr("websocket handshake timeout (exit stream)"))? + .map_err(|e| terr(format!("websocket handshake failed: {e}")))?; + Ok(split_ws(ws)) +} + +/// Split a websocket into the pool's boxed sink/stream halves — shared by the +/// scoped-exit and tunnel dial paths, so everything above the byte transport +/// is identical whichever egress carried the connection. fn split_ws(ws: tokio_tungstenite::WebSocketStream) -> (WebSocketSink, WebSocketStream) where S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Send + Unpin + 'static, diff --git a/src/wallet/e2e.rs b/src/wallet/e2e.rs index 089989bd..7a1bc8b1 100644 --- a/src/wallet/e2e.rs +++ b/src/wallet/e2e.rs @@ -12,12 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! LIVE two-wallet end-to-end payment over the Floonet path. Two real Goblin -//! wallets restored from mainnet mnemonics (seeds via env, NEVER a file) connect -//! to `wss://relay.goblin.st` — which rides the scoped Nym exit (.8) per the -//! pinned pool — and one sends a real gift-wrapped Grin payment to the other, -//! asynchronously through the relay. Proves the whole money path a phone would -//! use: mixnet -> exit -> relay -> gift wrap -> S2 -> finalize -> post. +//! LIVE two-wallet end-to-end payment over the Floonet path — CROSS-RELAY and +//! CROSS-NODE. Two real Goblin wallets restored from mainnet mnemonics (seeds +//! via env, NEVER a file) run on DIFFERENT relays (A on `wss://relay.goblin.st`, +//! B on `wss://nrelay.us-ea.st`, each pinned via its own `nostr.toml`) and +//! DIFFERENT Grin nodes (A on grincoin.org, B on main.gri.mw). One sends a real +//! gift-wrapped Grin payment to the other, asynchronously through the relays. +//! Proves the whole money path a phone would use, plus the outbox model: the +//! sender publishes the wrap to the RECIPIENT's advertised (kind 10050) relay, +//! not its own, and settlement posts through two independent nodes. +//! mixnet -> exit -> cross-relay gift wrap -> S2 -> finalize -> post. //! //! Ignored by default (real mainnet funds + a full recovery scan). Run: //! GOBLIN_E2E_SEED_A="word ..." GOBLIN_E2E_SEED_B="word ..." \ @@ -25,21 +29,42 @@ #[cfg(test)] mod tests { + use std::path::PathBuf; use std::time::{Duration, Instant}; use grin_util::types::ZeroingString; - use crate::nostr::NostrSendStatus; + use crate::nostr::{Contact, NostrConfig, NostrSendStatus}; use crate::wallet::types::{ConnectionMethod, PhraseMode, WalletTask}; use crate::wallet::{ConnectionsConfig, ExternalConnection, Mnemonic, Wallet}; /// 0.1 GRIN, in nanograin. Small on purpose (mainnet, real funds). const AMOUNT: u64 = 100_000_000; - /// Public mainnet node for the recovery scan + tx post. - const NODE_URL: &str = "https://grincoin.org"; + /// Wallet A's mainnet node (recovery scan + tx post). + const NODE_A: &str = "https://grincoin.org"; + /// Wallet B's mainnet node — a DIFFERENT operator, so the payment settles + /// across two independent nodes. + const NODE_B: &str = "https://main.gri.mw"; + /// Wallet A's relay (pinned via its nostr.toml, advertised in its 10050). + /// The new primary money-path relay, reached over its co-located scoped exit. + const RELAY_A: &str = "wss://relay.floonet.dev"; + /// Wallet B's relay — the SAME shared exit-backed primary as A (how the + /// shipped product works: every Goblin wallet defaults to relay.floonet.dev). + /// Both reach it over the co-located scoped exit, so the gift-wrap round-trip + /// rides the fast money path end to end. Nodes still differ (below), so the + /// payment still settles across two independent Grin nodes. + const RELAY_B: &str = "wss://relay.floonet.dev"; - /// Build + open a wallet from a 24-word mnemonic on an external node. - fn open_wallet(name: &str, phrase: &str, pw: &ZeroingString, conn_id: i64) -> Wallet { + /// Build + open a wallet from a 24-word mnemonic on its own external node + /// and its own single-relay nostr.toml override. + fn open_wallet( + name: &str, + phrase: &str, + pw: &ZeroingString, + conn_id: i64, + node_url: &str, + relay: &str, + ) -> Wallet { let mut m = Mnemonic::default(); m.set_mode(PhraseMode::Import); m.import(&ZeroingString::from(phrase)); @@ -47,14 +72,46 @@ mod tests { m.valid(), "{name}: mnemonic did not validate (bad seed words?)" ); - let conn = ConnectionMethod::External(conn_id, NODE_URL.to_string()); + let conn = ConnectionMethod::External(conn_id, node_url.to_string()); let w = Wallet::create(&name.to_string(), pw, &m, &conn) .unwrap_or_else(|e| panic!("{name}: wallet create failed: {e}")); + // Pin this wallet to a single relay BEFORE open(): init_nostr loads + // nostr.toml from the wallet data dir on open, and a `relays` override + // both drives the client's relay set and is advertised as the wallet's + // kind 10050 DM inbox (see NostrService::relays / publish_identity). + let wallet_dir = PathBuf::from(w.get_config().get_data_path()); + let mut nostr_cfg = NostrConfig::load(wallet_dir.clone()); + nostr_cfg.set_relays(vec![relay.to_string()]); + println!( + "[e2e] {name}: node={node_url} relay={relay} (nostr.toml at {})", + wallet_dir.join(NostrConfig::FILE_NAME).display() + ); w.open(pw.clone()) .unwrap_or_else(|e| panic!("{name}: wallet open failed: {e}")); w } + /// The persisted form of "added this payee from their nprofile": a contact + /// carrying their DM relay, so payment routing (send_targets -> fetch_dm_relays) + /// uses that relay directly instead of blind kind-10050 discovery over the + /// exit-less indexers. BOTH legs of a cross-relay payment need this seeded. + fn contact_with_relay(npub_hex: &str, relay: &str) -> Contact { + Contact { + ver: 1, + npub: npub_hex.to_string(), + petname: None, + nip05: None, + nip05_verified_at: None, + relays: vec![relay.to_string()], + nip44_v3: false, + hue: 0, + unknown: false, + added_at: 0, + last_paid_at: None, + blocked: false, + } + } + /// Poll `cond` until true or `secs` elapse; log progress via `label`. fn wait_until(label: &str, secs: u64, mut cond: impl FnMut() -> bool) -> bool { let start = Instant::now(); @@ -106,40 +163,86 @@ mod tests { "nym tunnel never came up" ); - // Register the mainnet node once; reuse its id for both wallets. - let node = ExternalConnection::new(NODE_URL.to_string(), Some("grin".to_string()), None); - let conn_id = node.id; - ConnectionsConfig::add_ext_conn(node); + // Register a SEPARATE mainnet node per wallet. ExternalConnection ids + // are unix seconds, and add_ext_conn dedupes on id — two conns built in + // the same second would collide — so bump B's id explicitly. + let node_a = ExternalConnection::new(NODE_A.to_string(), Some("grin".to_string()), None); + let conn_a = node_a.id; + ConnectionsConfig::add_ext_conn(node_a); + let mut node_b = + ExternalConnection::new(NODE_B.to_string(), Some("grin".to_string()), None); + node_b.id = conn_a + 1; + let conn_b = node_b.id; + ConnectionsConfig::add_ext_conn(node_b); + + let strip = |s: &str| { + s.trim_start_matches("https://") + .trim_start_matches("wss://") + .to_string() + }; + println!( + "[e2e] A: node={} relay={} | B: node={} relay={}", + strip(NODE_A), + strip(RELAY_A), + strip(NODE_B), + strip(RELAY_B) + ); let pw = ZeroingString::from("e2e-test-pass"); println!("[e2e] opening wallet A..."); - let a = open_wallet("goblin-e2e-a", seed_a.trim(), &pw, conn_id); + let a = open_wallet("goblin-e2e-a", seed_a.trim(), &pw, conn_a, NODE_A, RELAY_A); // Wallet id = unix seconds; two creates in the same second collide. std::thread::sleep(Duration::from_millis(1500)); println!("[e2e] opening wallet B..."); - let b = open_wallet("goblin-e2e-b", seed_b.trim(), &pw, conn_id); + let b = open_wallet("goblin-e2e-b", seed_b.trim(), &pw, conn_b, NODE_B, RELAY_B); - // Nostr services connect to relay.goblin.st (over the exit). + // Nostr services connect, each to its OWN relay (over the exit). let a_svc = a.nostr_service().expect("A nostr service"); let b_svc = b.nostr_service().expect("B nostr service"); - let t_conn = Instant::now(); + let t_a = Instant::now(); assert!( - wait_until("A nostr connected", 120, || a_svc.is_connected()), - "A never connected to a relay" + wait_until("A nostr connected", 240, || a_svc.is_connected()), + "A never connected to its relay ({RELAY_A})" ); + println!("[e2e] A connected in {}s", t_a.elapsed().as_secs()); + let t_b = Instant::now(); assert!( - wait_until("B nostr connected", 120, || b_svc.is_connected()), - "B never connected to a relay" + wait_until("B nostr connected", 240, || b_svc.is_connected()), + "B never connected to its relay ({RELAY_B})" ); - println!( - "[e2e] both goblins connected to the relay over the exit in {}s", - t_conn.elapsed().as_secs() + println!("[e2e] B connected in {}s", t_b.elapsed().as_secs()); + println!("[e2e] A effective relays = {:?}", a_svc.relays()); + println!("[e2e] B effective relays = {:?}", b_svc.relays()); + assert_eq!( + a_svc.relays(), + vec![RELAY_A.to_string()], + "A's relay override did not take" + ); + assert_eq!( + b_svc.relays(), + vec![RELAY_B.to_string()], + "B's relay override did not take" ); println!("[e2e] A npub = {}", a_svc.npub()); println!("[e2e] B npub = {}", b_svc.npub()); - // Recovery scan: concurrent across both wallets. Sender needs spendable. + // Pre-seed each wallet's contact store with the other (npub + DM relay) — + // the realistic "added the payee from their nprofile" path. Payment + // routing then uses the cached DM relay directly, so BOTH legs cross + // relays deterministically (A -> B's relay over the tunnel, B -> A's relay + // over the exit) without the kind-10050 discovery fetch over the exit-less + // indexers that stalled the pure-discovery run. + a_svc + .store + .save_contact(&contact_with_relay(&b_svc.public_key().to_hex(), RELAY_B)); + b_svc + .store + .save_contact(&contact_with_relay(&a_svc.public_key().to_hex(), RELAY_A)); + println!("[e2e] seeded contacts: A knows B @ {RELAY_B}, B knows A @ {RELAY_A}"); + + // Recovery scan: concurrent across both wallets, each against its own + // node. Sender needs spendable. wait_until("A synced_from_node", 2400, || a.synced_from_node()); wait_until("B synced_from_node", 2400, || b.synced_from_node()); @@ -152,7 +255,10 @@ mod tests { let b_bal = spendable(&b); println!("[e2e] spendable: A={a_bal} nano, B={b_bal} nano (need {AMOUNT})"); - // Sender = whichever wallet actually has the funds. + // Sender = whichever wallet actually has the funds. Either way the wrap + // crosses relays: the sender fetches the recipient's kind 10050 (from + // the recipient's relay + the discovery indexers) and publishes the + // gift wrap THERE — the outbox path this test exists to prove. let (sender, sender_svc, recv_svc, sender_name) = if a_bal >= AMOUNT + 20_000_000 { (&a, &a_svc, &b_svc, "A") } else if b_bal >= AMOUNT + 20_000_000 { @@ -165,7 +271,7 @@ mod tests { let receiver_hex = recv_svc.public_key().to_hex(); println!("[e2e] sender = {sender_name}; paying {AMOUNT} nano to {receiver_hex}"); - // Fire the async payment over the floonet relay. + // Fire the async payment across the two relays. let t_send = Instant::now(); sender.task(WalletTask::NostrSend( AMOUNT, @@ -175,7 +281,8 @@ mod tests { )); // Watch the sender's meta walk Created -> AwaitingS2 -> Finalized. - let finalized = wait_until("payment finalized", 420, || { + // Generous window: two relays + two nodes + mixnet round trips. + let finalized = wait_until("payment finalized", 900, || { if let Some(err) = sender_svc.last_send_error() { println!("[e2e] sender last_send_error: {err}"); } @@ -204,6 +311,6 @@ mod tests { finalized, "payment did not reach Finalized within the window (see meta trail above)" ); - println!("[e2e] SUCCESS: two goblins completed a payment over the floonet relay"); + println!("[e2e] SUCCESS: cross-relay + cross-node payment finalized over the floonet path"); } }