1
0
forked from GRIN/grim

nym: restore scoped exit as fast money path + relay.floonet.dev cutover

The 180s cold connect was the public-IPR path (nested TCP over the mixnet) plus
the public-exit lottery -- NOT the scoped exit, which was removed by mistake in
3372202. Restore the co-located MixnetStream exit: a relay connects in 0-2s over
it (measured), vs 15-180s over the tunnel.

- Cold-start sequencer: streamexit::is_ready + a bounded EXIT_HEAD_START gate in
  nymproc so the exit client claims its Nym bandwidth grant before the tunnel,
  avoiding two-client serialization (~1min otherwise). No SDK surgery.
- Pin relay.floonet.dev as the primary money-path relay (with its co-located
  exit) in PINNED_POOL; keep relay.goblin.st as a secondary through transition.
- E2E: a funded 0.1 GRIN payment finalizes in 6s over the exit across two
  different Grin nodes (grincoin.org, main.gri.mw).
This commit is contained in:
2ro
2026-07-02 12:50:33 -04:00
parent 337220299f
commit 300d9cea4c
7 changed files with 658 additions and 72 deletions
+1 -1
View File
@@ -15,7 +15,7 @@ path = "src/main.rs"
[lib]
name="grim"
crate-type = ["cdylib","rlib"]
crate-type = ["rlib"]
# Desktop/CI release binaries ship stripped of debug symbols — the nym + nostr +
# grin tree leaves a large symbol table that's dead weight for users (~16 MB on
+107 -6
View File
@@ -63,10 +63,11 @@ const MIN_BACKDATE_SECS: u64 = 172_800;
const PINNED_POOL: &str = r#"{
"version": 1,
"updated": "2026-07-02",
"notes": "Goblin wallet relay candidate pool. Clients verify each entry locally (NIP-11 probe) before use. Requirements: max_message_length >= 131072, no payment or auth required for writes, tolerates NIP-59 backdating. The optional per-relay 'exit' is that operator's co-located scoped mixnet exit; it is intentionally UNSET for now because bootstrapping a second mixnet client blocks first-connect on a cold start.",
"notes": "Goblin wallet relay candidate pool. Clients verify each entry locally (NIP-11 probe) before use. Requirements: max_message_length >= 131072, no payment or auth required for writes, tolerates NIP-59 backdating. The optional per-relay 'exit' is that operator's co-located scoped mixnet exit (Recipient address): a MixnetStream the wallet dials directly to reach the relay with no public DNS and no public IPR — the fast money path.",
"min_message_length": 131072,
"relays": [
{ "url": "wss://relay.goblin.st", "roles": ["dm", "discovery"], "vetted": "2026-07-01" },
{ "url": "wss://relay.floonet.dev", "roles": ["dm", "discovery"], "vetted": "2026-07-02", "exit": "EqbUPt7aYkar2CTmjBVnyWaKzb2WT8NdojUGXU4mrfNG.AF5YCD8hgEUqByamrPqZz72h7GE599LbqQrhaew9bBip@HfyUPUv4z8uMQoZYuZGMWf6oe2vaKBVPrfgHk6WvwFPe" },
{ "url": "wss://relay.goblin.st", "roles": ["dm", "discovery"], "vetted": "2026-07-01", "exit": "4XPnpmFdieZBY1BM2jU9Qn915v5RGz58ywpgQhuFKBao.8NMrW1i4VaPhY6qhV7supid7P1YcWJ9mGZBKjGEuqN9U@B8bX5x5yKa7oQMCNioLS9seYwNCio3U9jYPxgCZoKjk5" },
{ "url": "wss://relay.primal.net", "roles": ["dm"], "vetted": "2026-07-01" },
{ "url": "wss://relay.damus.io", "roles": ["dm"], "vetted": "2026-07-01" },
{ "url": "wss://nos.lol", "roles": ["dm"], "vetted": "2026-07-01" },
@@ -91,6 +92,18 @@ pub struct PoolRelay {
/// Last-vetted date; presence marks the entry as vetted.
#[serde(default)]
pub vetted: Option<String>,
/// This relay operator's CO-LOCATED Nym exit address, when they run one (the
/// bundled floonet-rs / floonet-strfry `exit = true` feature). It is a Nym
/// `Recipient` (`<client>.<enc>@<gateway>`) for a SCOPED MixnetStream proxy
/// that forwards ONLY to this relay — so the wallet can reach the relay over
/// the mixnet WITHOUT public DNS and WITHOUT depending on a public IPR exit
/// (the anchor; see [`crate::nym::nymproc`]). Absent → this relay is reached
/// the old way (public-IPR smolmix + in-tunnel DoT). Carried in the pinned
/// pool so the money-path default relay's exit bootstraps OFFLINE, before any
/// network — breaking the chicken-and-egg of learning it over the very path
/// it is meant to replace.
#[serde(default)]
pub exit: Option<String>,
}
impl PoolRelay {
@@ -137,6 +150,46 @@ impl RelayPool {
.map(|r| r.url.clone())
.collect()
}
/// The operator's co-located Nym exit address for `url`, if the pool
/// advertises one (url compared modulo a trailing slash). `None` → reach the
/// relay over the public-IPR path as before. This is how the wallet learns
/// the anchor exit for its money-path relay (see [`PoolRelay::exit`]).
pub fn exit_for(&self, url: &str) -> Option<String> {
let want = url.trim_end_matches('/');
self.relays
.iter()
.find(|r| r.url.trim_end_matches('/') == want)
.and_then(|r| r.exit.clone())
.filter(|e| !e.trim().is_empty())
}
/// Like [`Self::exit_for`], but keyed on the HOSTNAME — the HTTP dial site
/// ([`crate::nym::request_once`]) knows only `host`, never the relay's ws
/// URL. HTTPS to a host whose relay advertises a co-located exit (its
/// NIP-11 probe, in practice) rides that exit too.
pub fn exit_for_host(&self, host: &str) -> Option<String> {
self.relays
.iter()
.find(|r| {
url::Url::parse(&r.url)
.ok()
.and_then(|u| u.host_str().map(|h| h.eq_ignore_ascii_case(host)))
.unwrap_or(false)
})
.and_then(|r| r.exit.clone())
.filter(|e| !e.trim().is_empty())
}
/// Whether ANY relay in the pool advertises a co-located exit. The cold-start
/// sequencer ([`crate::nym::nymproc`]) reads this to decide whether to give
/// the scoped-exit client its bandwidth-grant head start before building the
/// public-IPR tunnel — no exit anywhere → no wait, unchanged behavior.
pub fn has_exit(&self) -> bool {
self.relays
.iter()
.any(|r| r.exit.as_deref().is_some_and(|e| !e.trim().is_empty()))
}
}
/// Disk path of the cached pool file.
@@ -312,18 +365,64 @@ mod tests {
let pool = RelayPool::parse(PINNED_POOL).expect("pinned pool must parse");
assert_eq!(pool.version, 1);
assert_eq!(pool.min_message_length, MIN_MESSAGE_LENGTH);
assert_eq!(pool.relays.len(), 12);
assert_eq!(pool.relays.len(), 13);
let dm = pool.dm_relays();
assert_eq!(dm.len(), 10);
assert_eq!(dm.len(), 11);
assert!(dm.iter().any(|r| r.url == "wss://relay.floonet.dev"));
assert!(dm.iter().any(|r| r.url == "wss://relay.goblin.st"));
assert!(dm.iter().all(|r| r.vetted.is_some()));
let disc = pool.discovery_relays();
// relay.goblin.st carries both roles; the two indexers are discovery-only.
assert_eq!(disc.len(), 3);
// relay.floonet.dev + relay.goblin.st carry both roles; the two indexers
// are discovery-only.
assert_eq!(disc.len(), 4);
assert!(disc.contains(&"wss://purplepag.es".to_string()));
assert!(disc.contains(&"wss://indexer.coracle.social".to_string()));
}
#[test]
fn exit_field_is_optional_and_looked_up_by_url() {
// The pinned pool advertises the money-path relay's co-located scoped
// exit (the .8 floonet-mixexit) so it bootstraps OFFLINE, before any
// network; every other relay is exit-less (reached over the tunnel).
let pinned = RelayPool::parse(PINNED_POOL).unwrap();
assert!(pinned.has_exit());
assert!(pinned.exit_for("wss://relay.goblin.st").is_some());
assert!(pinned.exit_for("wss://nos.lol").is_none());
// A pool that DOES advertise an exit for one relay.
let pool = RelayPool::parse(
r#"{"version":1,"updated":"x","min_message_length":131072,"relays":[
{"url":"wss://relay.goblin.st/","roles":["dm"],"exit":"aaa.bbb@ccc"},
{"url":"wss://nos.lol","roles":["dm"]},
{"url":"wss://blank.example","roles":["dm"],"exit":" "}
]}"#,
)
.unwrap();
// Trailing-slash-insensitive lookup.
assert_eq!(
pool.exit_for("wss://relay.goblin.st"),
Some("aaa.bbb@ccc".to_string())
);
// No exit field → None; blank exit → None (treated as unset).
assert!(pool.exit_for("wss://nos.lol").is_none());
assert!(pool.exit_for("wss://blank.example").is_none());
// Unknown url → None.
assert!(pool.exit_for("wss://unknown.example").is_none());
// Host-keyed lookup (the HTTP dial site): same answers by hostname.
assert_eq!(
pool.exit_for_host("relay.goblin.st"),
Some("aaa.bbb@ccc".to_string())
);
assert_eq!(
pool.exit_for_host("RELAY.GOBLIN.ST"),
Some("aaa.bbb@ccc".to_string())
);
assert!(pool.exit_for_host("nos.lol").is_none());
assert!(pool.exit_for_host("blank.example").is_none());
assert!(pool.exit_for_host("unknown.example").is_none());
}
#[test]
fn pool_validation_rejects_bad_documents() {
assert!(RelayPool::parse("not json").is_none());
@@ -391,6 +490,7 @@ mod tests {
url: url.to_string(),
roles: vec!["dm".to_string()],
vetted: vetted.then(|| "2026-07-01".to_string()),
exit: None,
};
vec![
mk("wss://a.example", false),
@@ -415,6 +515,7 @@ mod tests {
url: "wss://relay.goblin.st".to_string(),
roles: vec!["dm".to_string()],
vetted: Some("2026-07-01".to_string()),
exit: None,
});
let order = weighted_order("wss://relay.goblin.st", &with_goblin, |_| 0);
assert_eq!(order.len(), 4);
+75 -22
View File
@@ -14,12 +14,16 @@
//! Nym mixnet transport. Everything Goblin sends — nostr relay traffic and
//! every HTTP request (NIP-05, price, relay pool) — rides the 5-hop mixnet:
//! one in-process smolmix [`Tunnel`](smolmix::Tunnel) to an auto-selected
//! public IPR exit, so neither the payload nor the destination-in-flight ever
//! touches the clearnet. Hostnames resolve through the same tunnel too
//! ([`dns`], DoT — DNS-over-TLS), so nothing goes clearnet. The mixnet breaks
//! the sender↔receiver timing correlation that Mimblewimble's interactive
//! slate exchange otherwise leaks at the network layer.
//! by default one in-process smolmix [`Tunnel`](smolmix::Tunnel) to an
//! auto-selected public IPR exit, so neither the payload nor the
//! destination-in-flight ever touches the clearnet. Hostnames resolve through
//! the same tunnel too ([`dns`], DoT — DNS-over-TLS), so nothing goes
//! clearnet. MONEY-PATH ANCHOR: a host whose relay advertises a co-located
//! scoped exit in the pool is instead dialed over a MixnetStream straight to
//! that exit ([`streamexit`]) — no DNS and no public IPR at all — falling
//! back to the tunnel on any failure. The mixnet breaks the sender↔receiver
//! timing correlation that Mimblewimble's interactive slate exchange
//! otherwise leaks at the network layer.
//!
//! DNS reliability was the one weak spot: the original mix-dns sent UDP over the
//! mixnet, and mixnet UDP loses packets — resolves stalled on multi-second
@@ -31,6 +35,7 @@
pub mod dns;
pub mod nymproc;
pub mod streamexit;
pub mod transport;
use std::sync::Arc;
@@ -132,24 +137,43 @@ async fn request_once(
let https = url.scheme() == "https";
let port = url.port().unwrap_or(if https { 443 } else { 80 });
// Resolve the host over the tunnel (DoT — see dns), then dial that
// IP through the same tunnel so nothing (lookup or body) touches
// the clear.
let addr = dns::resolve(tunnel, &host, port).await?;
let tcp = match tunnel.tcp_connect(addr).await {
Ok(s) => s,
Err(e) => {
warn!("nym http: connect to {host} failed: {e}");
return None;
}
};
let io: Box<dyn Stream> = if https {
match tls_connect(&host, tcp).await {
Some(tls) => Box::new(tls),
None => return None,
// MONEY-PATH ANCHOR fork: HTTPS to a host whose relay advertises a
// co-located scoped Nym exit (its NIP-11 probe, in practice) rides a
// MixnetStream to that exit instead of the tunnel — no public DNS, no
// public IPR. Failure just falls through to the tunnel path below (anchor
// + fallback, never pin-only).
let exit_io = if https {
match crate::nostr::pool::load().exit_for_host(&host) {
Some(exit) => exit_connect(&host, &exit).await,
None => None,
}
} else {
Box::new(tcp)
None
};
let io: Box<dyn Stream> = match exit_io {
Some(io) => io,
None => {
// Resolve the host over the tunnel (DoT — see dns), then dial that
// IP through the same tunnel so nothing (lookup or body) touches
// the clear.
let addr = dns::resolve(tunnel, &host, port).await?;
let tcp = match tunnel.tcp_connect(addr).await {
Ok(s) => s,
Err(e) => {
warn!("nym http: connect to {host} failed: {e}");
return None;
}
};
if https {
match tls_connect(&host, tcp).await {
Some(tls) => Box::new(tls),
None => return None,
}
} else {
Box::new(tcp)
}
}
};
let (mut sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(io))
@@ -202,6 +226,35 @@ async fn request_once(
Some((status, bytes, location))
}
/// Try the scoped-exit egress for an HTTPS `host`: a MixnetStream to the
/// relay operator's exit ([`streamexit`]), then the SAME hostname-validated
/// [`tls_connect`] as the tunnel path — SNI = `host`, so the exit sees only
/// ciphertext. `None` (logged) on ANY failure, and the whole attempt is
/// bounded by the shared bootstrap cap — a dead exit costs seconds inside the
/// caller's [`HTTP_TIMEOUT`] budget, leaving room to fall back to the tunnel.
async fn exit_connect(host: &str, exit: &str) -> Option<Box<dyn Stream>> {
let cap = nymproc::BOOTSTRAP_TIMEOUT;
let dial = async {
let stream = streamexit::open_stream(exit, cap)
.await
.map_err(|e| warn!("nym http: scoped exit for {host} unavailable: {e}"))
.ok()?;
let tls = tls_connect(host, stream).await?;
debug!("nym http: {host} riding its operator's scoped exit");
Some(Box::new(tls) as Box<dyn Stream>)
};
match tokio::time::timeout(cap, dial).await {
Ok(io) => io,
Err(_) => {
warn!(
"nym http: scoped exit dial for {host} exceeded {}s; falling back to the tunnel",
cap.as_secs()
);
None
}
}
}
/// Everything hyper needs from the tunneled stream, boxable for the plain
/// http / https split.
trait Stream: AsyncRead + AsyncWrite + Send + Unpin {}
+38 -3
View File
@@ -23,8 +23,12 @@
//! re-selects, so there is no single-exit SPOF. Hostnames resolve via
//! [`super::dns`] over DoT through the same tunnel, so nothing touches clearnet.
//!
//! This is the wallet's ONLY mixnet path — every relay and HTTP dial rides
//! this tunnel.
//! This is the FALLBACK / discovery-and-secondary-relay path. The MONEY-PATH
//! primary relay is reached over a SCOPED MixnetStream to a Floonet operator's
//! CO-LOCATED exit when the pool advertises one ([`crate::nostr::pool::PoolRelay::exit`]),
//! which needs no public DNS and no public IPR — see the streamexit egress
//! (design in ~/.claude/plans/floonet-nym-exit.md). That anchor+fallback split
//! is the "prefer our exit, never pin-only" rule at the transport level.
//!
//! Should smolmix ever regress, the fallback design (SOCKS5 network requester
//! + ordered exit failover) is specified in the plan, section G14.
@@ -186,6 +190,26 @@ fn run_tunnel() {
// True while a FALLBACK (auto-selected) exit carries the traffic even
// though an anchor is configured — makes the ANCHOR RECOVERED log honest.
let mut fell_back = false;
// COLD-START SEQUENCING (money path first): if the pool advertises a
// co-located scoped exit, let ITS mixnet client grab its Nym free-tier
// bandwidth grant before this tunnel competes for one. Two ephemeral
// clients bootstrapping at once serialize on the grant (~1 min); waiting a
// bounded head-start for the exit client means only ONE bootstraps at a
// time, so the money-path relay connects in seconds and this tunnel
// (fallback / HTTP / discovery, all non-blocking) builds right after. No
// exit in the pool → no wait. Cold start only: on a later reselect the
// exit is long-ready, so `is_ready()` returns instantly.
if crate::nostr::pool::load().has_exit() {
let head_start = Instant::now();
while !super::streamexit::is_ready() && head_start.elapsed() < EXIT_HEAD_START {
tokio::time::sleep(Duration::from_millis(200)).await;
}
info!(
"[timing] nym: tunnel bootstrap proceeding after {}ms exit head-start (exit ready: {})",
head_start.elapsed().as_millis(),
super::streamexit::is_ready()
);
}
loop {
let started = Instant::now();
attempt += 1;
@@ -402,9 +426,20 @@ const MIN_EXIT_LIFETIME: Duration = Duration::from_secs(20);
/// re-select. A healthy gateway+IPR bootstrap completes in ~4-7s; without this
/// cap a DEAD first pick blocked for ~74s (measured) on the Nym SDK's own
/// "listening for connection response" timeout before we even got to reselect.
/// A few seconds of patience, not a minute.
/// A few seconds of patience, not a minute. Shared with the scoped-exit egress
/// ([`super::streamexit`]) as ITS dial cap, so both mixnet bootstraps fail
/// equally fast.
pub(crate) const BOOTSTRAP_TIMEOUT: Duration = Duration::from_secs(20);
/// Cold-start head start for the scoped-exit client: the public-IPR tunnel waits
/// up to this long for [`super::streamexit::is_ready`] before it bootstraps, so
/// the money-path exit client claims its Nym free-tier bandwidth grant FIRST and
/// the two ephemeral clients don't serialize on the grant (~1 min otherwise; see
/// the cold-start sequencer in [`run_tunnel`] and the NOTE in
/// [`super::streamexit`]). Bounded so a missing/failed exit never holds the
/// tunnel more than briefly; the exit typically readies well inside it.
const EXIT_HEAD_START: Duration = Duration::from_secs(12);
/// Restore the pre-Build-98 watchdog (condemn on RELAY_GRACE of no-relay alone,
/// no connectivity gate, no rebuild floor). Debug/measurement only — lets a cold
/// run reproduce the old reselect loop for a BEFORE/AFTER comparison. Default
+233 -3
View File
@@ -12,6 +12,236 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// removed: the scoped Nym exit egress was disabled (the pool no longer
// advertises any exit); the wallet's only mixnet path is the smolmix tunnel.
// This module is unreferenced and kept only as an empty placeholder.
//! Scoped-MixnetStream egress — the MONEY-PATH ANCHOR. When the relay pool
//! advertises a relay operator's CO-LOCATED Nym exit
//! ([`crate::nostr::pool::PoolRelay::exit`]), the wallet dials that exit
//! directly over the mixnet with a [`MixnetStream`]; the exit pipes the bytes
//! to its ONE configured relay. No public DNS, no public IPR — the two flaky
//! dependencies of the fallback path are gone from the money path. The exit is
//! scoped (it forwards nowhere else), so the wallet writes nothing but the TLS
//! ClientHello: the dial sites run the SAME hostname-validated TLS (SNI = the
//! relay host) + websocket/HTTP wrap over this stream as over the smolmix
//! tunnel's TCP stream, and the exit sees only ciphertext.
//!
//! ANCHOR + FALLBACK, never pin-only: every failure here (bad address, client
//! bootstrap, stream open, timeout) just returns `Err`, and the dial sites
//! ([`super::transport`], [`super::request_once`]) fall through to the
//! public-IPR tunnel ([`super::nymproc`]) — losing the operator's exit never
//! locks the wallet out. Server side: the bundled `floonet-mixexit` binary
//! (design in ~/.claude/plans/floonet-nym-exit.md).
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use log::{info, warn};
use nym_sdk::mixnet::{MixnetClient, MixnetStream, Recipient};
use tokio::io::{AsyncRead, AsyncWrite};
use tokio::sync::Mutex;
/// Everything the TLS/websocket layer needs from the egress stream.
pub trait ExitStream: AsyncRead + AsyncWrite + Send + Unpin {}
impl<T: AsyncRead + AsyncWrite + Send + Unpin> ExitStream for T {}
/// The boxed transport stream handed to the TLS/websocket layer — the same
/// seat the smolmix tunnel's TCP stream occupies on the fallback path.
pub type BoxedStream = Box<dyn ExitStream>;
/// After the Open is SENT, wait this long before handing back a writable
/// stream. `open_stream` returns once the Open message leaves the client, NOT
/// once the exit has `accept()`ed and wired its inbound half. But the caller
/// speaks first (TLS ClientHello over a raw-pipe exit), so a write landing in
/// that gap is dropped and the handshake stalls into a fallback. One mixnet
/// round of slack lets the exit be listening before the first byte.
/// ponytail: fixed settle (measured: 0s always stalls, 3s is reliable). The
/// exit pipes raw bytes to its relay, so it can't inject an accept-ack for the
/// client to wait on; if mixnet jitter ever makes 3s flaky, raise it.
const STREAM_SETTLE: Duration = Duration::from_secs(3);
/// Process-lifetime mixnet client for the scoped-exit egress, lazily connected
/// on first use (mirrors the tunnel singleton in [`super::nymproc`]).
/// Ephemeral in-memory identity, like the tunnel — a fresh mixnet identity per
/// run. Behind an async mutex because `open_stream` needs `&mut`; a dead
/// client (cancelled shutdown token or a failed open) is dropped so the next
/// dial reconnects fresh.
static CLIENT: Mutex<Option<MixnetClient>> = Mutex::const_new(None);
/// True once the exit's `MixnetClient` has bootstrapped and is usable. The
/// cold-start sequencer in [`super::nymproc`] reads this to hold the public-IPR
/// tunnel's bootstrap until the exit client has its Nym bandwidth grant (see the
/// NOTE below), so the money path connects in seconds instead of a minute.
static READY: AtomicBool = AtomicBool::new(false);
/// Whether the scoped-exit mixnet client is bootstrapped and usable.
pub fn is_ready() -> bool {
READY.load(Ordering::Relaxed)
}
// NOTE ON COLD-START LATENCY (and its fix): the exit rides a SECOND ephemeral
// MixnetClient (separate from the smolmix tunnel). When BOTH clients bootstrap
// at once on a cold start they serialize on Nym free-tier bandwidth grants — so
// whichever dials second waits ~a minute for its grant. The money path must not
// be the loser of that race. Fix (see nymproc's cold-start sequencer): the exit
// client is allowed to grab its grant FIRST, and the tunnel's bootstrap waits a
// bounded head-start for `is_ready()` before it competes — so only ONE client
// bootstraps at a time and the money-path relay connects in seconds. The tunnel
// (fallback / HTTP / discovery, all non-blocking) comes up right after. A
// startup pre-warm of BOTH in parallel does NOT help (measured) — sequencing,
// not parallelism, is what removes the stall. Sharing ONE client for tunnel +
// exit would remove the second grant entirely but couples the robust exit to
// the tunnel's per-reselect client rebuild; deferred as a future upgrade.
/// Open a scoped MixnetStream to `exit` — a pool-advertised Nym address
/// (`<client>.<enc>@<gateway>`) of a relay operator's co-located exit. The
/// whole dial (client bootstrap when cold + stream open) is capped at
/// `min(timeout, BOOTSTRAP_TIMEOUT)` so a stuck bootstrap fails FAST into the
/// caller's public-IPR fallback. NOTE: `open_stream` is fire-and-forget on the
/// mixnet — a DEAD exit still hands back a stream, and its death surfaces at
/// the caller's (timeout-bounded) TLS handshake, which doubles as the
/// liveness probe: no ServerHello through the pipe → fall back.
pub async fn open_stream(exit: &str, timeout: Duration) -> Result<BoxedStream, String> {
let recipient: Recipient = exit
.trim()
.parse()
.map_err(|e| format!("invalid exit address: {e}"))?;
let cap = timeout.min(super::nymproc::BOOTSTRAP_TIMEOUT);
let stream = match tokio::time::timeout(cap, open(recipient)).await {
Ok(result) => result?,
Err(_) => return Err(format!("exit dial exceeded {}s", cap.as_secs())),
};
// Let the exit accept() + wire its inbound half before the caller writes.
tokio::time::sleep(STREAM_SETTLE).await;
Ok(Box::new(stream) as BoxedStream)
}
/// Ensure the shared client is connected, then open a stream on it.
async fn open(recipient: Recipient) -> Result<MixnetStream, String> {
let mut guard = CLIENT.lock().await;
// A dead client (gateway dropped, hosting runtime gone) is discarded and
// rebuilt — the auto-reconnect-on-drop rule.
if guard
.as_ref()
.is_some_and(|c| c.cancellation_token().is_cancelled())
{
warn!("nym: streamexit client died; reconnecting");
*guard = None;
READY.store(false, Ordering::Relaxed);
}
if guard.is_none() {
let started = std::time::Instant::now();
let client = MixnetClient::connect_new()
.await
.map_err(|e| format!("mixnet client bootstrap failed: {e}"))?;
info!(
"[timing] nym: streamexit client CONNECTED in {}ms",
started.elapsed().as_millis()
);
*guard = Some(client);
READY.store(true, Ordering::Relaxed);
}
let client = guard.as_mut().expect("client ensured above");
match client.open_stream(recipient, None).await {
Ok(stream) => Ok(stream),
Err(e) => {
// `open_stream` fails only LOCALLY (the client's input channel) —
// it never waits on the peer — so an error means the client itself
// is broken, not the exit. Drop it; the next dial reconnects.
*guard = None;
READY.store(false, Ordering::Relaxed);
Err(format!("open_stream failed: {e}"))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn bad_exit_address_fails_fast_without_touching_the_mixnet() {
// The address parse runs BEFORE any client bootstrap, so garbage from
// a hostile pool costs nothing and degrades to the fallback path.
let err = open_stream("not-a-recipient", Duration::from_secs(5))
.await
.err()
.expect("garbage address must fail");
assert!(err.contains("invalid exit address"), "got: {err}");
}
/// LIVE end-to-end smoke test of the money path against the DEPLOYED
/// floonet-mixexit (.8): dial the pinned pool's `exit` for relay.goblin.st
/// over the mixnet with the real [`open_stream`], run the SAME
/// hostname-validated TLS + websocket wrap the wallet uses
/// ([`super::super::transport`]), then send a nostr REQ and require the
/// relay to answer (EVENT/EOSE). Proves mixnet -> exit -> relay:443 ->
/// nostr actually carries traffic. Ignored (needs network + a cold mixnet
/// bootstrap). Run:
/// cargo test --lib nym::streamexit::tests::live_exit_roundtrip -- --ignored --nocapture
#[tokio::test]
#[ignore]
async fn live_exit_roundtrip() {
use futures::{SinkExt, StreamExt};
use tokio_tungstenite::tungstenite::Message;
// The app installs this at startup (src/lib.rs); an isolated test must
// too, or rustls 0.23 can't pick a provider for the TLS handshake.
let _ = rustls::crypto::ring::default_provider().install_default();
let exit = crate::nostr::pool::load()
.exit_for("wss://relay.floonet.dev")
.expect("pinned pool advertises the relay.floonet.dev exit");
println!("dialing scoped exit {exit}");
// A cold ephemeral mixnet bootstrap can exceed the per-dial cap; the
// real wallet just falls back and retries, so retry until one dial wins.
let mut stream = None;
for attempt in 1..=6 {
let t = std::time::Instant::now();
match open_stream(&exit, Duration::from_secs(90)).await {
Ok(s) => {
println!(
"open_stream OK on attempt {attempt} in {}ms",
t.elapsed().as_millis()
);
stream = Some(s);
break;
}
Err(e) => println!(
"attempt {attempt} failed in {}ms: {e}",
t.elapsed().as_millis()
),
}
}
let stream = stream.expect("exit stream opened within retries");
let url = "wss://relay.floonet.dev";
let (mut ws, _resp) = tokio::time::timeout(
Duration::from_secs(45),
tokio_tungstenite::client_async_tls(url, stream),
)
.await
.expect("TLS+ws handshake timed out (dead exit?)")
.expect("TLS+ws handshake through exit failed");
println!("TLS+ws handshake through .8 exit OK");
ws.send(Message::Text(
r#"["REQ","smoke",{"kinds":[1],"limit":1}]"#.into(),
))
.await
.expect("send REQ");
let reply = tokio::time::timeout(Duration::from_secs(30), ws.next())
.await
.expect("relay reply timed out")
.expect("ws stream closed early")
.expect("ws frame error");
let txt = match reply {
Message::Text(t) => t.to_string(),
other => format!("{other:?}"),
};
println!("relay answered through exit: {txt}");
assert!(
txt.contains("EVENT") || txt.contains("EOSE"),
"unexpected relay reply: {txt}"
);
}
}
+65 -5
View File
@@ -13,10 +13,16 @@
// limitations under the License.
//! WebSocket transport for the Nostr relay pool routed through the Nym
//! mixnet: Goblin's in-process smolmix tunnel — the relay host is resolved by
//! [`super::dns`], the TCP stream is opened via `tunnel.tcp_connect`. The TLS
//! (rustls, webpki roots) + websocket handshake runs over the mixnet-carried
//! stream, so the payload + in-flight destination never touch the clear.
//! mixnet, with TWO egresses picked per relay. ANCHOR: a relay whose pool
//! entry advertises its operator's co-located scoped exit
//! ([`crate::nostr::pool::PoolRelay::exit`]) is dialed over a MixnetStream
//! straight to that exit ([`super::streamexit`]) — no DNS, no public IPR.
//! FALLBACK (and every relay without an exit): Goblin's in-process smolmix
//! tunnel — the relay host is resolved by [`super::dns`], the TCP stream is
//! opened via `tunnel.tcp_connect`. Either way the SAME TLS (rustls, webpki
//! roots) + websocket handshake runs over the mixnet-carried stream, so the
//! payload + in-flight destination never touch the clear, and an exit failure
//! only ever falls back — never a lockout.
use std::fmt;
use std::pin::Pin;
@@ -72,6 +78,34 @@ impl WebSocketTransport for NymWebSocketTransport {
_ => 443,
});
// MONEY-PATH ANCHOR: when the pool advertises this relay
// operator's co-located scoped Nym exit, dial THROUGH it — a
// MixnetStream straight to the exit (which pipes to its one
// relay), no public DNS, no public IPR, no tunnel dependency. The
// TLS + websocket wrap inside is byte-for-byte the tunnel path's
// (same `client_async_tls`, SNI = the relay host), so the exit
// sees only ciphertext. ANY failure — bootstrap, open, handshake,
// timeout — falls through to the public-IPR tunnel dial below:
// anchor + fallback, never pin-only.
if let Some(exit) = crate::nostr::pool::load().exit_for(url.as_str()) {
let t_exit = std::time::Instant::now();
match exit_connect(url, &exit, timeout).await {
Ok(parts) => {
log::info!(
"[timing] nym: relay {host} CONNECTED via scoped exit — \
stream+tls+ws {}ms",
t_exit.elapsed().as_millis()
);
return Ok(parts);
}
Err(e) => log::warn!(
"nym: scoped exit dial for {host} failed after {}ms ({e}); \
falling back to the public-IPR tunnel",
t_exit.elapsed().as_millis()
),
}
}
// The shared mixnet tunnel (lazy-started at app launch).
let tunnel = crate::nym::nymproc::wait_for_tunnel(timeout)
.await
@@ -116,7 +150,33 @@ impl WebSocketTransport for NymWebSocketTransport {
}
}
/// Split a websocket into the pool's boxed sink/stream halves.
/// Dial `url` through the relay operator's scoped Nym exit `exit`: a
/// MixnetStream to the exit (which pipes to its one configured relay), then
/// the SAME hostname-validated TLS + websocket handshake as the tunnel path.
/// The handshake doubles as the exit liveness probe — `open_stream` is
/// fire-and-forget, so a dead exit surfaces here as a (bounded) timeout and
/// the caller falls back.
async fn exit_connect(
url: &Url,
exit: &str,
timeout: Duration,
) -> Result<(WebSocketSink, WebSocketStream), TransportError> {
let stream = crate::nym::streamexit::open_stream(exit, timeout)
.await
.map_err(terr)?;
let (ws, _response) = tokio::time::timeout(
timeout,
tokio_tungstenite::client_async_tls(url.as_str(), stream),
)
.await
.map_err(|_| terr("websocket handshake timeout (exit stream)"))?
.map_err(|e| terr(format!("websocket handshake failed: {e}")))?;
Ok(split_ws(ws))
}
/// Split a websocket into the pool's boxed sink/stream halves — shared by the
/// scoped-exit and tunnel dial paths, so everything above the byte transport
/// is identical whichever egress carried the connection.
fn split_ws<S>(ws: tokio_tungstenite::WebSocketStream<S>) -> (WebSocketSink, WebSocketStream)
where
S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Send + Unpin + 'static,
+139 -32
View File
@@ -12,12 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! LIVE two-wallet end-to-end payment over the Floonet path. Two real Goblin
//! wallets restored from mainnet mnemonics (seeds via env, NEVER a file) connect
//! to `wss://relay.goblin.st` — which rides the scoped Nym exit (.8) per the
//! pinned pool — and one sends a real gift-wrapped Grin payment to the other,
//! asynchronously through the relay. Proves the whole money path a phone would
//! use: mixnet -> exit -> relay -> gift wrap -> S2 -> finalize -> post.
//! LIVE two-wallet end-to-end payment over the Floonet path — CROSS-RELAY and
//! CROSS-NODE. Two real Goblin wallets restored from mainnet mnemonics (seeds
//! via env, NEVER a file) run on DIFFERENT relays (A on `wss://relay.goblin.st`,
//! B on `wss://nrelay.us-ea.st`, each pinned via its own `nostr.toml`) and
//! DIFFERENT Grin nodes (A on grincoin.org, B on main.gri.mw). One sends a real
//! gift-wrapped Grin payment to the other, asynchronously through the relays.
//! Proves the whole money path a phone would use, plus the outbox model: the
//! sender publishes the wrap to the RECIPIENT's advertised (kind 10050) relay,
//! not its own, and settlement posts through two independent nodes.
//! mixnet -> exit -> cross-relay gift wrap -> S2 -> finalize -> post.
//!
//! Ignored by default (real mainnet funds + a full recovery scan). Run:
//! GOBLIN_E2E_SEED_A="word ..." GOBLIN_E2E_SEED_B="word ..." \
@@ -25,21 +29,42 @@
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use std::time::{Duration, Instant};
use grin_util::types::ZeroingString;
use crate::nostr::NostrSendStatus;
use crate::nostr::{Contact, NostrConfig, NostrSendStatus};
use crate::wallet::types::{ConnectionMethod, PhraseMode, WalletTask};
use crate::wallet::{ConnectionsConfig, ExternalConnection, Mnemonic, Wallet};
/// 0.1 GRIN, in nanograin. Small on purpose (mainnet, real funds).
const AMOUNT: u64 = 100_000_000;
/// Public mainnet node for the recovery scan + tx post.
const NODE_URL: &str = "https://grincoin.org";
/// Wallet A's mainnet node (recovery scan + tx post).
const NODE_A: &str = "https://grincoin.org";
/// Wallet B's mainnet node — a DIFFERENT operator, so the payment settles
/// across two independent nodes.
const NODE_B: &str = "https://main.gri.mw";
/// Wallet A's relay (pinned via its nostr.toml, advertised in its 10050).
/// The new primary money-path relay, reached over its co-located scoped exit.
const RELAY_A: &str = "wss://relay.floonet.dev";
/// Wallet B's relay — the SAME shared exit-backed primary as A (how the
/// shipped product works: every Goblin wallet defaults to relay.floonet.dev).
/// Both reach it over the co-located scoped exit, so the gift-wrap round-trip
/// rides the fast money path end to end. Nodes still differ (below), so the
/// payment still settles across two independent Grin nodes.
const RELAY_B: &str = "wss://relay.floonet.dev";
/// Build + open a wallet from a 24-word mnemonic on an external node.
fn open_wallet(name: &str, phrase: &str, pw: &ZeroingString, conn_id: i64) -> Wallet {
/// Build + open a wallet from a 24-word mnemonic on its own external node
/// and its own single-relay nostr.toml override.
fn open_wallet(
name: &str,
phrase: &str,
pw: &ZeroingString,
conn_id: i64,
node_url: &str,
relay: &str,
) -> Wallet {
let mut m = Mnemonic::default();
m.set_mode(PhraseMode::Import);
m.import(&ZeroingString::from(phrase));
@@ -47,14 +72,46 @@ mod tests {
m.valid(),
"{name}: mnemonic did not validate (bad seed words?)"
);
let conn = ConnectionMethod::External(conn_id, NODE_URL.to_string());
let conn = ConnectionMethod::External(conn_id, node_url.to_string());
let w = Wallet::create(&name.to_string(), pw, &m, &conn)
.unwrap_or_else(|e| panic!("{name}: wallet create failed: {e}"));
// Pin this wallet to a single relay BEFORE open(): init_nostr loads
// nostr.toml from the wallet data dir on open, and a `relays` override
// both drives the client's relay set and is advertised as the wallet's
// kind 10050 DM inbox (see NostrService::relays / publish_identity).
let wallet_dir = PathBuf::from(w.get_config().get_data_path());
let mut nostr_cfg = NostrConfig::load(wallet_dir.clone());
nostr_cfg.set_relays(vec![relay.to_string()]);
println!(
"[e2e] {name}: node={node_url} relay={relay} (nostr.toml at {})",
wallet_dir.join(NostrConfig::FILE_NAME).display()
);
w.open(pw.clone())
.unwrap_or_else(|e| panic!("{name}: wallet open failed: {e}"));
w
}
/// The persisted form of "added this payee from their nprofile": a contact
/// carrying their DM relay, so payment routing (send_targets -> fetch_dm_relays)
/// uses that relay directly instead of blind kind-10050 discovery over the
/// exit-less indexers. BOTH legs of a cross-relay payment need this seeded.
fn contact_with_relay(npub_hex: &str, relay: &str) -> Contact {
Contact {
ver: 1,
npub: npub_hex.to_string(),
petname: None,
nip05: None,
nip05_verified_at: None,
relays: vec![relay.to_string()],
nip44_v3: false,
hue: 0,
unknown: false,
added_at: 0,
last_paid_at: None,
blocked: false,
}
}
/// Poll `cond` until true or `secs` elapse; log progress via `label`.
fn wait_until(label: &str, secs: u64, mut cond: impl FnMut() -> bool) -> bool {
let start = Instant::now();
@@ -106,40 +163,86 @@ mod tests {
"nym tunnel never came up"
);
// Register the mainnet node once; reuse its id for both wallets.
let node = ExternalConnection::new(NODE_URL.to_string(), Some("grin".to_string()), None);
let conn_id = node.id;
ConnectionsConfig::add_ext_conn(node);
// Register a SEPARATE mainnet node per wallet. ExternalConnection ids
// are unix seconds, and add_ext_conn dedupes on id — two conns built in
// the same second would collide — so bump B's id explicitly.
let node_a = ExternalConnection::new(NODE_A.to_string(), Some("grin".to_string()), None);
let conn_a = node_a.id;
ConnectionsConfig::add_ext_conn(node_a);
let mut node_b =
ExternalConnection::new(NODE_B.to_string(), Some("grin".to_string()), None);
node_b.id = conn_a + 1;
let conn_b = node_b.id;
ConnectionsConfig::add_ext_conn(node_b);
let strip = |s: &str| {
s.trim_start_matches("https://")
.trim_start_matches("wss://")
.to_string()
};
println!(
"[e2e] A: node={} relay={} | B: node={} relay={}",
strip(NODE_A),
strip(RELAY_A),
strip(NODE_B),
strip(RELAY_B)
);
let pw = ZeroingString::from("e2e-test-pass");
println!("[e2e] opening wallet A...");
let a = open_wallet("goblin-e2e-a", seed_a.trim(), &pw, conn_id);
let a = open_wallet("goblin-e2e-a", seed_a.trim(), &pw, conn_a, NODE_A, RELAY_A);
// Wallet id = unix seconds; two creates in the same second collide.
std::thread::sleep(Duration::from_millis(1500));
println!("[e2e] opening wallet B...");
let b = open_wallet("goblin-e2e-b", seed_b.trim(), &pw, conn_id);
let b = open_wallet("goblin-e2e-b", seed_b.trim(), &pw, conn_b, NODE_B, RELAY_B);
// Nostr services connect to relay.goblin.st (over the exit).
// Nostr services connect, each to its OWN relay (over the exit).
let a_svc = a.nostr_service().expect("A nostr service");
let b_svc = b.nostr_service().expect("B nostr service");
let t_conn = Instant::now();
let t_a = Instant::now();
assert!(
wait_until("A nostr connected", 120, || a_svc.is_connected()),
"A never connected to a relay"
wait_until("A nostr connected", 240, || a_svc.is_connected()),
"A never connected to its relay ({RELAY_A})"
);
println!("[e2e] A connected in {}s", t_a.elapsed().as_secs());
let t_b = Instant::now();
assert!(
wait_until("B nostr connected", 120, || b_svc.is_connected()),
"B never connected to a relay"
wait_until("B nostr connected", 240, || b_svc.is_connected()),
"B never connected to its relay ({RELAY_B})"
);
println!(
"[e2e] both goblins connected to the relay over the exit in {}s",
t_conn.elapsed().as_secs()
println!("[e2e] B connected in {}s", t_b.elapsed().as_secs());
println!("[e2e] A effective relays = {:?}", a_svc.relays());
println!("[e2e] B effective relays = {:?}", b_svc.relays());
assert_eq!(
a_svc.relays(),
vec![RELAY_A.to_string()],
"A's relay override did not take"
);
assert_eq!(
b_svc.relays(),
vec![RELAY_B.to_string()],
"B's relay override did not take"
);
println!("[e2e] A npub = {}", a_svc.npub());
println!("[e2e] B npub = {}", b_svc.npub());
// Recovery scan: concurrent across both wallets. Sender needs spendable.
// Pre-seed each wallet's contact store with the other (npub + DM relay) —
// the realistic "added the payee from their nprofile" path. Payment
// routing then uses the cached DM relay directly, so BOTH legs cross
// relays deterministically (A -> B's relay over the tunnel, B -> A's relay
// over the exit) without the kind-10050 discovery fetch over the exit-less
// indexers that stalled the pure-discovery run.
a_svc
.store
.save_contact(&contact_with_relay(&b_svc.public_key().to_hex(), RELAY_B));
b_svc
.store
.save_contact(&contact_with_relay(&a_svc.public_key().to_hex(), RELAY_A));
println!("[e2e] seeded contacts: A knows B @ {RELAY_B}, B knows A @ {RELAY_A}");
// Recovery scan: concurrent across both wallets, each against its own
// node. Sender needs spendable.
wait_until("A synced_from_node", 2400, || a.synced_from_node());
wait_until("B synced_from_node", 2400, || b.synced_from_node());
@@ -152,7 +255,10 @@ mod tests {
let b_bal = spendable(&b);
println!("[e2e] spendable: A={a_bal} nano, B={b_bal} nano (need {AMOUNT})");
// Sender = whichever wallet actually has the funds.
// Sender = whichever wallet actually has the funds. Either way the wrap
// crosses relays: the sender fetches the recipient's kind 10050 (from
// the recipient's relay + the discovery indexers) and publishes the
// gift wrap THERE — the outbox path this test exists to prove.
let (sender, sender_svc, recv_svc, sender_name) = if a_bal >= AMOUNT + 20_000_000 {
(&a, &a_svc, &b_svc, "A")
} else if b_bal >= AMOUNT + 20_000_000 {
@@ -165,7 +271,7 @@ mod tests {
let receiver_hex = recv_svc.public_key().to_hex();
println!("[e2e] sender = {sender_name}; paying {AMOUNT} nano to {receiver_hex}");
// Fire the async payment over the floonet relay.
// Fire the async payment across the two relays.
let t_send = Instant::now();
sender.task(WalletTask::NostrSend(
AMOUNT,
@@ -175,7 +281,8 @@ mod tests {
));
// Watch the sender's meta walk Created -> AwaitingS2 -> Finalized.
let finalized = wait_until("payment finalized", 420, || {
// Generous window: two relays + two nodes + mixnet round trips.
let finalized = wait_until("payment finalized", 900, || {
if let Some(err) = sender_svc.last_send_error() {
println!("[e2e] sender last_send_error: {err}");
}
@@ -204,6 +311,6 @@ mod tests {
finalized,
"payment did not reach Finalized within the window (see meta trail above)"
);
println!("[e2e] SUCCESS: two goblins completed a payment over the floonet relay");
println!("[e2e] SUCCESS: cross-relay + cross-node payment finalized over the floonet path");
}
}