Add mock ecash support and fix PSK injection timing in LP sessions

Key changes:
- Add outer_aead_key_for_sending() to gate outer encryption on PSQ completion
  (fixes bug where initiator encrypted msg 1 before responder could decrypt)
- Add handshake_and_register_with_credential() to NestedLpSession for mock ecash
- Update PSQState::InitiatorWaiting to store PSK instead of ciphertext
- Add probe-localnet.sh script for two-hop localnet testing
- Update gateway handler with connection lifecycle statistics

The PSK timing fix ensures the first Noise message is sent in cleartext
because the responder hasn't derived the PSK yet from the PSQ payload.
This commit is contained in:
durch
2025-12-04 18:13:05 +01:00
parent 2a2f511333
commit bf8e061310
6 changed files with 411 additions and 86 deletions
+66 -30
View File
@@ -117,8 +117,12 @@ pub enum PSQState {
NotStarted,
/// Initiator has sent PSQ ciphertext and is waiting for confirmation.
/// Stores the ciphertext that was sent.
InitiatorWaiting { ciphertext: Vec<u8> },
/// PSK is already derived but we don't encrypt outgoing packets yet
/// because the responder may not have processed our message yet.
InitiatorWaiting {
/// The derived PSK, stored until we transition to Completed
psk: [u8; 32],
},
/// Responder is ready to receive and decapsulate PSQ ciphertext.
ResponderWaiting,
@@ -280,10 +284,40 @@ impl LpSession {
///
/// Callers should use `None` for packet encryption/decryption during
/// the handshake phase, and use the returned key for transport phase.
///
/// Note: For sending packets during handshake, use `outer_aead_key_for_sending()`
/// which checks PSQ state to avoid encrypting before the responder can decrypt.
pub fn outer_aead_key(&self) -> Option<OuterAeadKey> {
self.outer_aead_key.lock().clone()
}
/// Returns the outer AEAD key only if it's safe to use for sending.
///
/// This method gates the key based on PSQ handshake state:
/// - Returns `None` if PSQ is NotStarted, InitiatorWaiting, or ResponderWaiting
/// - Returns `Some(key)` only if PSQ is Completed
///
/// # Why This Matters
///
/// The first Noise handshake message (containing PSQ payload from initiator)
/// must be sent in cleartext because the responder hasn't derived the PSK yet.
/// Only after the responder processes the PSQ and both sides have the PSK
/// can outer encryption be used for sending.
///
/// For receiving, use `outer_aead_key()` which returns the key as soon as
/// it's derived (needed because the peer may start encrypting before we've
/// finished our send).
// AIDEV-NOTE: This fixes a bug where the initiator encrypted the first Noise
// message with outer AEAD, but the responder couldn't decrypt because it
// hadn't processed the PSQ yet to derive the same PSK.
pub fn outer_aead_key_for_sending(&self) -> Option<OuterAeadKey> {
let psq_state = self.psq_state.lock();
match &*psq_state {
PSQState::Completed { .. } => self.outer_aead_key.lock().clone(),
_ => None,
}
}
/// Creates a new session and initializes the Noise protocol state.
///
/// PSQ always runs during the handshake to derive the real PSK from X25519 DHKEM.
@@ -735,10 +769,9 @@ impl LpSession {
combined.extend_from_slice(&psq_payload);
combined.extend_from_slice(&noise_msg);
// Update PSQ state to InitiatorWaiting
*psq_state = PSQState::InitiatorWaiting {
ciphertext: psq_payload,
};
// PSK is derived but we stay in InitiatorWaiting until we receive msg 2.
// This ensures we send msg 1 in cleartext (responder can't decrypt yet).
*psq_state = PSQState::InitiatorWaiting { psk };
return Some(Ok(LpMessage::Handshake(HandshakeData(combined))));
}
@@ -905,36 +938,39 @@ impl LpSession {
}
// Check if initiator should extract PSK handle from message 2
if self.is_initiator && matches!(*psq_state, PSQState::InitiatorWaiting { .. }) {
// Extract PSK handle: [u16 handle_len][handle_bytes][noise_msg]
if payload.len() >= 2 {
let handle_len = u16::from_le_bytes([payload[0], payload[1]]) as usize;
if let PSQState::InitiatorWaiting { psk } = *psq_state {
if self.is_initiator {
// Extract PSK handle: [u16 handle_len][handle_bytes][noise_msg]
if payload.len() >= 2 {
let handle_len = u16::from_le_bytes([payload[0], payload[1]]) as usize;
if handle_len > 0 && payload.len() >= 2 + handle_len {
// Extract and store the PSK handle
let handle_bytes = &payload[2..2 + handle_len];
let noise_payload = &payload[2 + handle_len..];
if handle_len > 0 && payload.len() >= 2 + handle_len {
// Extract and store the PSK handle
let handle_bytes = &payload[2..2 + handle_len];
let noise_payload = &payload[2 + handle_len..];
tracing::debug!(
"Extracted PSK handle ({} bytes) from message 2",
handle_len
);
tracing::debug!(
"Extracted PSK handle ({} bytes) from message 2",
handle_len
);
{
let mut psk_handle = self.psk_handle.lock();
*psk_handle = Some(handle_bytes.to_vec());
{
let mut psk_handle = self.psk_handle.lock();
*psk_handle = Some(handle_bytes.to_vec());
}
// Transition to Completed - we've received confirmation from responder
*psq_state = PSQState::Completed { psk };
drop(psq_state);
// Process only the Noise message part
return noise_state
.read_message(noise_payload)
.map_err(LpError::NoiseError);
}
// Release psq_state lock before processing
drop(psq_state);
// Process only the Noise message part
return noise_state
.read_message(noise_payload)
.map_err(LpError::NoiseError);
}
// If no valid handle found, fall through to normal processing
}
// If no valid handle found, fall through to normal processing
}
// The sans-io NoiseProtocol::read_message expects only the payload.
+17 -5
View File
@@ -6,8 +6,8 @@ use super::registration::process_registration;
use super::LpHandlerState;
use crate::error::GatewayError;
use nym_lp::{
codec::OuterAeadKey, keypair::PublicKey, message::ForwardPacketData, OuterHeader,
LpMessage, LpPacket,
codec::OuterAeadKey, keypair::PublicKey, message::ForwardPacketData, packet::LpHeader,
LpMessage, LpPacket, OuterHeader,
};
use nym_metrics::{add_histogram_obs, inc};
use std::net::SocketAddr;
@@ -332,6 +332,14 @@ impl LpConnectionHandler {
self.remote_addr, receiver_idx
);
// Get outer key for Ack encryption before releasing borrow
let outer_key = state_entry
.value()
.state
.session()
.ok()
.and_then(|s| s.outer_aead_key());
// Move state machine to session_states (already in Transport state)
// We keep the state machine (not just session) to enable
// subsession/rekeying support during transport phase
@@ -344,9 +352,13 @@ impl LpConnectionHandler {
inc!("lp_handshakes_success");
// No response packet to send - HandshakeComplete means we're done
trace!("Moved session {} to transport mode", receiver_idx);
None
// Send Ack to confirm handshake completion to the client
let ack_packet = LpPacket::new(
LpHeader::new(receiver_idx, 0),
LpMessage::Ack,
);
trace!("Moved session {} to transport mode, sending Ack", receiver_idx);
Some((ack_packet, outer_key))
}
other => {
debug!("Received action during handshake: {:?}", other);
+81 -29
View File
@@ -771,6 +771,10 @@ impl Probe {
);
// Determine entry and exit gateways
// AIDEV-NOTE: Three modes for gateway resolution:
// 1. direct_gateway_node/exit_gateway_node - from --gateway-ip (HTTP API query)
// 2. localnet_entry/localnet_exit - from --entry-gateway-identity (CLI-only)
// 3. directory lookup - original behavior for production
let (entry_gateway, exit_gateway) = if let Some(exit_node) = &self.exit_gateway_node {
// Both entry and exit gateways were pre-queried (direct IP mode)
info!("Using pre-queried entry and exit gateways for LP forwarding test");
@@ -783,6 +787,15 @@ impl Probe {
let exit_gateway = exit_node.to_testable_node()?;
(entry_gateway, exit_gateway)
} else if let Some(exit_localnet) = &self.localnet_exit {
// Localnet mode: use CLI-provided identities and LP addresses
info!("Using localnet entry and exit gateways for LP forwarding test");
let entry_localnet = self
.localnet_entry
.as_ref()
.ok_or_else(|| anyhow::anyhow!("Entry gateway not available in localnet mode"))?;
(entry_localnet.clone(), exit_localnet.clone())
} else {
// Original behavior: query from directory
// The tested node is the exit
@@ -1230,24 +1243,45 @@ where
.map_err(|e| anyhow::anyhow!("Invalid exit gateway identity: {}", e))?;
// Perform handshake and registration with exit gateway via forwarding
if use_mock_ecash {
info!("Note: Using mock ecash mode - gateways must be started with --lp-use-mock-ecash");
}
let exit_gateway_data = match nested_session
.handshake_and_register(
&mut entry_client,
&exit_wg_keypair,
&exit_gateway_pubkey,
bandwidth_controller,
let exit_gateway_data = if use_mock_ecash {
info!("Using mock ecash credential for exit gateway registration");
let credential = crate::bandwidth_helpers::create_dummy_credential(
&exit_gateway_pubkey.to_bytes(),
TicketType::V1WireguardExit,
exit_ip,
)
.await
{
Ok(data) => data,
Err(e) => {
error!("Failed to register with exit gateway: {}", e);
return Ok(wg_outcome);
);
match nested_session
.handshake_and_register_with_credential(
&mut entry_client,
&exit_wg_keypair,
credential,
TicketType::V1WireguardExit,
exit_ip,
)
.await
{
Ok(data) => data,
Err(e) => {
error!("Failed to register with exit gateway (mock ecash): {}", e);
return Ok(wg_outcome);
}
}
} else {
match nested_session
.handshake_and_register(
&mut entry_client,
&exit_wg_keypair,
&exit_gateway_pubkey,
bandwidth_controller,
TicketType::V1WireguardExit,
exit_ip,
)
.await
{
Ok(data) => data,
Err(e) => {
error!("Failed to register with exit gateway: {}", e);
return Ok(wg_outcome);
}
}
};
info!("Exit gateway registration successful via forwarding");
@@ -1259,19 +1293,37 @@ where
.map_err(|e| anyhow::anyhow!("Invalid entry gateway identity: {}", e))?;
// Use packet-per-connection register() which returns GatewayData directly
let _entry_gateway_data = match entry_client
.register(
&entry_wg_keypair,
&entry_gateway_pubkey,
bandwidth_controller,
let _entry_gateway_data = if use_mock_ecash {
info!("Using mock ecash credential for entry gateway registration");
let credential = crate::bandwidth_helpers::create_dummy_credential(
&entry_gateway_pubkey.to_bytes(),
TicketType::V1WireguardEntry,
)
.await
{
Ok(data) => data,
Err(e) => {
error!("Failed to register with entry gateway: {}", e);
return Ok(wg_outcome);
);
match entry_client
.register_with_credential(&entry_wg_keypair, credential, TicketType::V1WireguardEntry)
.await
{
Ok(data) => data,
Err(e) => {
error!("Failed to register with entry gateway (mock ecash): {}", e);
return Ok(wg_outcome);
}
}
} else {
match entry_client
.register(
&entry_wg_keypair,
&entry_gateway_pubkey,
bandwidth_controller,
TicketType::V1WireguardEntry,
)
.await
{
Ok(data) => data,
Err(e) => {
error!("Failed to register with entry gateway: {}", e);
return Ok(wg_outcome);
}
}
};
info!("Entry gateway registration successful");
+54 -18
View File
@@ -207,7 +207,8 @@ impl LpRegistrationClient {
let ack_response = Self::connect_send_receive(
self.gateway_lp_address,
&client_hello_packet,
None, // No outer key before handshake
None, // No outer key for ClientHello (before PSK)
None, // No outer key for Ack response (before PSK)
&self.config,
)
.await?;
@@ -258,17 +259,25 @@ impl LpRegistrationClient {
loop {
// Send pending packet if we have one
if let Some(packet) = pending_packet.take() {
// Get outer key from session (None before PSK, Some after)
let outer_key = state_machine
// Get outer keys from session:
// - send_key: outer_aead_key_for_sending() returns None until PSQ complete
// - recv_key: outer_aead_key() returns key as soon as PSK is derived
let send_key = state_machine
.session()
.ok()
.and_then(|s| s.outer_aead_key_for_sending());
let recv_key = state_machine
.session()
.ok()
.and_then(|s| s.outer_aead_key());
tracing::trace!("Sending handshake packet (outer_key={})", outer_key.is_some());
tracing::trace!("Sending handshake packet (send_key={}, recv_key={})",
send_key.is_some(), recv_key.is_some());
let response = Self::connect_send_receive(
self.gateway_lp_address,
&packet,
outer_key.as_ref(),
send_key.as_ref(),
recv_key.as_ref(),
&self.config,
)
.await?;
@@ -287,7 +296,11 @@ impl LpRegistrationClient {
if state_machine.session()?.is_handshake_complete() {
// Send the final packet before breaking
if let Some(final_packet) = pending_packet.take() {
let outer_key = state_machine
let send_key = state_machine
.session()
.ok()
.and_then(|s| s.outer_aead_key_for_sending());
let recv_key = state_machine
.session()
.ok()
.and_then(|s| s.outer_aead_key());
@@ -295,7 +308,8 @@ impl LpRegistrationClient {
let ack_response = Self::connect_send_receive(
self.gateway_lp_address,
&final_packet,
outer_key.as_ref(),
send_key.as_ref(),
recv_key.as_ref(),
&self.config,
)
.await?;
@@ -369,10 +383,22 @@ impl LpRegistrationClient {
///
/// # Errors
/// Returns an error if connection, send, or receive fails.
///
/// # Outer AEAD Keys
///
/// Send and receive use separate keys because during the PSQ handshake:
/// - Initiator derives PSK when preparing msg 1, but must send it cleartext
/// (responder hasn't derived PSK yet)
/// - Responder sends msg 2 encrypted (both have PSK now)
/// - Initiator can decrypt msg 2 (has had PSK since preparing msg 1)
///
/// Use `outer_aead_key_for_sending()` for `send_key` (gates on PSQ completion)
/// and `outer_aead_key()` for `recv_key` (available as soon as PSK derived).
async fn connect_send_receive(
address: SocketAddr,
packet: &LpPacket,
outer_key: Option<&OuterAeadKey>,
send_key: Option<&OuterAeadKey>,
recv_key: Option<&OuterAeadKey>,
config: &LpConfig,
) -> Result<LpPacket> {
// 1. Connect with timeout
@@ -398,11 +424,11 @@ impl LpRegistrationClient {
source,
})?;
// 3. Send packet with optional outer AEAD
Self::send_packet_with_key(&mut stream, packet, outer_key).await?;
// 3. Send packet with send_key
Self::send_packet_with_key(&mut stream, packet, send_key).await?;
// 4. Receive response with optional outer AEAD
let response = Self::receive_packet_with_key(&mut stream, outer_key).await?;
// 4. Receive response with recv_key
let response = Self::receive_packet_with_key(&mut stream, recv_key).await?;
// Connection drops when stream goes out of scope
Ok(response)
@@ -617,8 +643,12 @@ impl LpRegistrationClient {
}
};
// 4. Get outer key from session
let outer_key = state_machine
// 4. Get outer keys from session
let send_key = state_machine
.session()
.ok()
.and_then(|s| s.outer_aead_key_for_sending());
let recv_key = state_machine
.session()
.ok()
.and_then(|s| s.outer_aead_key());
@@ -629,7 +659,8 @@ impl LpRegistrationClient {
Self::connect_send_receive(
self.gateway_lp_address,
&request_packet,
outer_key.as_ref(),
send_key.as_ref(),
recv_key.as_ref(),
&self.config,
),
)
@@ -797,8 +828,12 @@ impl LpRegistrationClient {
}
};
// 4. Get outer key from session
let outer_key = state_machine
// 4. Get outer keys from session
let send_key = state_machine
.session()
.ok()
.and_then(|s| s.outer_aead_key_for_sending());
let recv_key = state_machine
.session()
.ok()
.and_then(|s| s.outer_aead_key());
@@ -807,7 +842,8 @@ impl LpRegistrationClient {
let response_packet = Self::connect_send_receive(
self.gateway_lp_address,
&forward_packet,
outer_key.as_ref(),
send_key.as_ref(),
recv_key.as_ref(),
&self.config,
)
.await?;
@@ -280,6 +280,158 @@ impl NestedLpSession {
Ok(())
}
/// Performs handshake and registration with the exit gateway via forwarding,
/// using a pre-made credential.
///
/// This variant is useful for mock ecash testing where the credential is provided
/// directly instead of being acquired from a bandwidth controller.
///
/// # Arguments
/// * `outer_client` - Connected LP client with established outer session to entry gateway
/// * `wg_keypair` - Client's WireGuard x25519 keypair
/// * `credential` - Pre-made bandwidth credential (e.g., mock ecash)
/// * `ticket_type` - Type of bandwidth ticket to use
/// * `client_ip` - Client IP address for registration metadata
///
/// # Returns
/// * `Ok(GatewayData)` - Exit gateway configuration data on successful registration
pub async fn handshake_and_register_with_credential(
&mut self,
outer_client: &mut LpRegistrationClient,
wg_keypair: &x25519::KeyPair,
credential: nym_credentials_interface::CredentialSpendingData,
ticket_type: TicketType,
client_ip: IpAddr,
) -> Result<GatewayData> {
// Step 1: Perform handshake with exit gateway via forwarding
self.perform_handshake(outer_client).await?;
// Step 2: Get the state machine (must exist after successful handshake)
let state_machine = self.state_machine.as_mut().ok_or_else(|| {
LpClientError::Transport("State machine missing after handshake".to_string())
})?;
tracing::debug!("Building registration request for exit gateway (with pre-made credential)");
// Step 3: Build registration request (credential already provided)
let wg_public_key = PeerPublicKey::new(wg_keypair.public_key().to_bytes().into());
let request = LpRegistrationRequest::new_dvpn(wg_public_key, credential, ticket_type, client_ip);
tracing::trace!("Built registration request: {:?}", request);
// Step 4: Serialize the request
let request_bytes = bincode::serialize(&request).map_err(|e| {
LpClientError::Transport(format!("Failed to serialize registration request: {}", e))
})?;
tracing::debug!(
"Sending registration request to exit gateway via forwarding ({} bytes)",
request_bytes.len()
);
// Step 5: Encrypt and prepare packet via state machine
let action = state_machine
.process_input(LpInput::SendData(request_bytes))
.ok_or_else(|| {
LpClientError::Transport("State machine returned no action".to_string())
})?
.map_err(|e| {
LpClientError::Transport(format!(
"Failed to encrypt registration request: {}",
e
))
})?;
// Step 6: Send the encrypted packet via forwarding
// Get outer key for AEAD encryption (PSK is available after handshake)
let outer_key = state_machine.session().ok().and_then(|s| s.outer_aead_key_for_sending());
let response_bytes = match action {
LpAction::SendPacket(packet) => {
let packet_bytes = Self::serialize_packet(&packet, outer_key.as_ref())?;
outer_client
.send_forward_packet(
self.exit_identity,
self.exit_address.clone(),
packet_bytes,
)
.await?
}
other => {
return Err(LpClientError::Transport(format!(
"Unexpected action when sending registration data: {:?}",
other
)));
}
};
tracing::trace!("Received registration response from exit gateway");
// Step 7: Parse response bytes to LP packet
let outer_key = state_machine.session().ok().and_then(|s| s.outer_aead_key());
let response_packet = Self::parse_packet(&response_bytes, outer_key.as_ref())?;
// Step 8: Decrypt via state machine
let action = state_machine
.process_input(LpInput::ReceivePacket(response_packet))
.ok_or_else(|| {
LpClientError::Transport("State machine returned no action".to_string())
})?
.map_err(|e| {
LpClientError::Transport(format!(
"Failed to decrypt registration response: {}",
e
))
})?;
// Step 9: Extract decrypted data
let response_data = match action {
LpAction::DeliverData(data) => data,
other => {
return Err(LpClientError::Transport(format!(
"Unexpected action when receiving registration response: {:?}",
other
)));
}
};
// Step 10: Deserialize the response
let response: LpRegistrationResponse =
bincode::deserialize(&response_data).map_err(|e| {
LpClientError::Transport(format!(
"Failed to deserialize registration response: {}",
e
))
})?;
tracing::debug!(
"Received registration response from exit: success={}",
response.success,
);
// Step 11: Validate and extract GatewayData
if !response.success {
let error_msg = response
.error
.unwrap_or_else(|| "Unknown error".to_string());
tracing::warn!("Exit gateway rejected registration: {}", error_msg);
return Err(LpClientError::RegistrationRejected { reason: error_msg });
}
// Extract gateway_data
let gateway_data = response.gateway_data.ok_or_else(|| {
LpClientError::Transport(
"Gateway response missing gateway_data despite success=true".to_string(),
)
})?;
tracing::info!(
"Exit gateway registration successful! Allocated bandwidth: {} bytes",
response.allocated_bandwidth
);
Ok(gateway_data)
}
/// Performs handshake and registration with the exit gateway via forwarding.
///
/// This is the main entry point for nested LP registration. It:
@@ -369,7 +521,7 @@ impl NestedLpSession {
// Step 7: Send the encrypted packet via forwarding
// Get outer key for AEAD encryption (PSK is available after handshake)
let outer_key = state_machine.session().ok().and_then(|s| s.outer_aead_key());
let outer_key = state_machine.session().ok().and_then(|s| s.outer_aead_key_for_sending());
let response_bytes = match action {
LpAction::SendPacket(packet) => {
let packet_bytes = Self::serialize_packet(&packet, outer_key.as_ref())?;
@@ -470,8 +622,9 @@ impl NestedLpSession {
state_machine: &LpStateMachine,
packet: &LpPacket,
) -> Result<LpPacket> {
let outer_key = state_machine.session().ok().and_then(|s| s.outer_aead_key());
let packet_bytes = Self::serialize_packet(packet, outer_key.as_ref())?;
// Use outer_aead_key_for_sending() for send, outer_aead_key() for receive
let send_key = state_machine.session().ok().and_then(|s| s.outer_aead_key_for_sending());
let packet_bytes = Self::serialize_packet(packet, send_key.as_ref())?;
let response_bytes = outer_client
.send_forward_packet(
self.exit_identity,
@@ -479,7 +632,8 @@ impl NestedLpSession {
packet_bytes,
)
.await?;
Self::parse_packet(&response_bytes, outer_key.as_ref())
let recv_key = state_machine.session().ok().and_then(|s| s.outer_aead_key());
Self::parse_packet(&response_bytes, recv_key.as_ref())
}
/// Serializes an LP packet to bytes.
+35
View File
@@ -0,0 +1,35 @@
#!/bin/bash
# Probe localnet gateways for LP two-hop testing
# Usage: ./scripts/probe-localnet.sh [mode]
# Modes: two-hop (default), single-hop, lp-only
set -e
MODE="${1:-two-hop}"
# Gateway API (localhost mapped ports)
ENTRY_API="127.0.0.1:30004"
EXIT_API="127.0.0.1:30005"
# Get gateway identities from API
ENTRY_ID=$(curl -s "http://${ENTRY_API}/api/v1/host-information" | jq -r '.data.keys.ed25519_identity')
EXIT_ID=$(curl -s "http://${EXIT_API}/api/v1/host-information" | jq -r '.data.keys.ed25519_identity')
if [ -z "$ENTRY_ID" ] || [ "$ENTRY_ID" = "null" ] || [ -z "$EXIT_ID" ] || [ "$EXIT_ID" = "null" ]; then
echo "Error: Could not get gateway identities from API"
echo "Make sure localnet is running: container list"
exit 1
fi
echo "Entry gateway: $ENTRY_ID"
echo "Exit gateway: $EXIT_ID"
echo "Mode: $MODE"
echo "---"
cargo run -p nym-gateway-probe -- run-local \
--entry-gateway-identity "$ENTRY_ID" \
--entry-lp-address '127.0.0.1:41264' \
--exit-gateway-identity "$EXIT_ID" \
--exit-lp-address '192.168.65.6:41264' \
--mode "$MODE" \
--use-mock-ecash