Optimize database queries by eliminating N+1 patterns in simulation system
This commit addresses critical N+1 query performance issues identified in the reward simulation system. The optimizations significantly reduce database round trips and improve performance when processing large datasets. **Key Optimizations:** 1. **Batch Identity Key Lookups** - Added `get_mixnode_identity_keys_batch()` and `get_gateway_identity_keys_batch()` - Updated simulation performance conversion to use batch operations - Reduced from N individual queries to 2 batch queries 2. **Batch Node Classification** - Added `classify_nodes_batch()` method for mixnode/gateway determination - Updated reliability calculation methods to use batch classification - Reduced from N individual lookups to 2 batch queries 3. **Batch Epoch Metadata Enhancement** - Added `count_simulated_node_performance_for_epochs_batch()` - Added `get_available_calculation_methods_for_epochs_batch()` - Updated API handlers to use batch operations for metadata enhancement - Reduced from 2N queries to 2 batch queries for epoch data 4. **Bulk Insert Optimizations** - Converted individual INSERT operations to use `sqlx::QueryBuilder::push_values()` - Optimized simulation data insertion methods - Eliminated transaction overhead from individual inserts **Performance Impact:** - Before: N+2N database queries for N nodes/epochs - After: 2+2 batch queries regardless of dataset size - Significant performance improvement for large simulation datasets All changes maintain backward compatibility while providing substantial performance benefits for the reward simulation system.
This commit is contained in:
@@ -82,19 +82,27 @@ async fn list_simulation_epochs(
|
||||
.await
|
||||
.map_err(to_axum_error)?;
|
||||
|
||||
// Enhance epochs with additional metadata
|
||||
// Enhance epochs with additional metadata using batch operations
|
||||
let epoch_db_ids: Vec<i64> = epochs.iter().map(|e| e.id).collect();
|
||||
let epoch_ids: Vec<u32> = epochs.iter().map(|e| e.epoch_id).collect();
|
||||
|
||||
// Batch fetch node counts and available methods for all epochs
|
||||
let node_counts = storage
|
||||
.manager
|
||||
.count_simulated_node_performance_for_epochs_batch(&epoch_db_ids)
|
||||
.await
|
||||
.map_err(|e| to_axum_error(SimulationApiError::with_details("Database error", &e.to_string())))?;
|
||||
let available_methods = storage
|
||||
.manager
|
||||
.get_available_calculation_methods_for_epochs_batch(&epoch_ids)
|
||||
.await
|
||||
.map_err(|e| to_axum_error(SimulationApiError::with_details("Database error", &e.to_string())))?;
|
||||
|
||||
let mut enhanced_epochs = Vec::new();
|
||||
for mut epoch in epochs {
|
||||
// Count nodes analyzed for this epoch
|
||||
epoch.nodes_analyzed = count_nodes_for_epoch(storage, epoch.id)
|
||||
.await
|
||||
.map_err(to_axum_error)?;
|
||||
|
||||
// Get available calculation methods for this epoch_id
|
||||
epoch.available_methods = get_available_methods_for_epoch(storage, epoch.epoch_id)
|
||||
.await
|
||||
.map_err(to_axum_error)?;
|
||||
|
||||
// Get metadata from our batch results
|
||||
epoch.nodes_analyzed = node_counts.get(&epoch.id).copied().unwrap_or(0);
|
||||
epoch.available_methods = available_methods.get(&epoch.epoch_id).cloned().unwrap_or_default();
|
||||
enhanced_epochs.push(epoch);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user