From 3995e4a4ccad334d04107dd8dabc362d34bfbfba Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 14 Oct 2024 13:49:16 +0800 Subject: [PATCH 01/52] test: validation behaviour establish_connection --- .../dispersal/validator/behaviour.rs | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index a227ca32a..42c243efc 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -147,3 +147,118 @@ impl + 'static> Netw Poll::Pending } } + +#[cfg(test)] +mod tests { + use super::*; + use futures::task::ArcWake; + use libp2p::{identity, PeerId}; + use std::collections::{HashMap, HashSet}; + use std::sync::Arc; + + #[derive(Clone, Debug)] + struct MockMembershipHandler { + membership: HashMap>, + } + + impl MembershipHandler for MockMembershipHandler { + type NetworkId = SubnetworkId; + type Id = PeerId; + + fn membership(&self, peer_id: &PeerId) -> HashSet { + self.membership.get(peer_id).cloned().unwrap_or_default() + } + + fn members_of(&self, subnetwork: &Self::NetworkId) -> HashSet { + self.membership + .iter() + .filter_map(|(id, nets)| { + if nets.contains(subnetwork) { + Some(*id) + } else { + None + } + }) + .collect() + } + + fn is_allowed(&self, _id: &Self::Id) -> bool { + unimplemented!() + } + + fn members(&self) -> HashSet { + HashSet::from(self.membership.keys().cloned().collect()) + } + } + + struct TestWaker; + + impl ArcWake for TestWaker { + fn wake_by_ref(_arc_self: &Arc) {} + } + + fn get_peer_from_membership(ids: &HashSet, idx: usize) -> PeerId { + let mut peer_id_vec: Vec<_> = ids.into_iter().collect(); + peer_id_vec.sort(); + **peer_id_vec.iter().nth(idx).unwrap() + } + fn create_validation_behaviours( + num_instances: usize, + subnet_id: u32, + membership: &mut HashMap>, + ) -> Vec> { + let mut behaviours = Vec::new(); + + let mut peer_ids = Vec::new(); + for _ in 0..num_instances { + let keypair = identity::Keypair::generate_ed25519(); + let peer_id = PeerId::from(keypair.public()); + peer_ids.push(peer_id); + } + + for peer_id in &peer_ids { + membership.insert(*peer_id, HashSet::from([subnet_id])); + } + + let membership_handler = MockMembershipHandler { + membership: HashMap::default(), // This will be updated after all behaviours are added. + }; + + for peer_id in peer_ids { + let behaviour = DispersalValidatorBehaviour::new(membership_handler.clone()); + behaviours.push(behaviour); + } + + behaviours + } + + fn establish_connection( + behaviours: &mut [DispersalValidatorBehaviour], + i: usize, + j: usize, + connection_id: ConnectionId, + ) { + let mut members: Vec<_> = behaviours[i].membership.members().into_iter().collect(); + members.sort(); + let peer_id_i = members[i]; + let peer_id_j = members[j]; + + behaviours[i] + .handle_established_outbound_connection( + connection_id, + peer_id_j, + &Multiaddr::empty(), + Endpoint::Dialer, + ) + .unwrap(); + + behaviours[j] + .handle_established_inbound_connection( + connection_id, + peer_id_i, + &Multiaddr::empty(), + &Multiaddr::empty(), + ) + .unwrap(); + } +} From a2c54231b35a52557f37f0e0e7a3e159fecb12d7 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 14 Oct 2024 14:57:29 +0800 Subject: [PATCH 02/52] fix: members for MockMembershipHandler --- .../protocols/dispersal/validator/behaviour.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 42c243efc..bb9896c09 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -187,7 +187,11 @@ mod tests { } fn members(&self) -> HashSet { - HashSet::from(self.membership.keys().cloned().collect()) + let mut members = HashSet::new(); + for id in self.membership.keys() { + members.insert(*id); + } + members } } @@ -197,11 +201,6 @@ mod tests { fn wake_by_ref(_arc_self: &Arc) {} } - fn get_peer_from_membership(ids: &HashSet, idx: usize) -> PeerId { - let mut peer_id_vec: Vec<_> = ids.into_iter().collect(); - peer_id_vec.sort(); - **peer_id_vec.iter().nth(idx).unwrap() - } fn create_validation_behaviours( num_instances: usize, subnet_id: u32, @@ -224,7 +223,7 @@ mod tests { membership: HashMap::default(), // This will be updated after all behaviours are added. }; - for peer_id in peer_ids { + for _ in peer_ids { let behaviour = DispersalValidatorBehaviour::new(membership_handler.clone()); behaviours.push(behaviour); } @@ -238,7 +237,7 @@ mod tests { j: usize, connection_id: ConnectionId, ) { - let mut members: Vec<_> = behaviours[i].membership.members().into_iter().collect(); + let mut members: Vec = behaviours[i].membership.members().into_iter().collect(); members.sort(); let peer_id_i = members[i]; let peer_id_j = members[j]; From 7d21571e557c1885a0bee84cbfe91508b1162ef2 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 16 Oct 2024 09:28:34 +0800 Subject: [PATCH 03/52] test: simple basics to understand --- .../dispersal/validator/behaviour.rs | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index bb9896c09..91373acd9 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -183,7 +183,10 @@ mod tests { } fn is_allowed(&self, _id: &Self::Id) -> bool { - unimplemented!() + if self.membership.contains_key(_id) { + return true; + } + false } fn members(&self) -> HashSet { @@ -260,4 +263,48 @@ mod tests { ) .unwrap(); } + + #[test] + fn test_handle_established_inbound_connection() { + let mut allowed_peers = HashMap::new(); + allowed_peers.insert(PeerId::random(), HashSet::from([0, 1])); + let membership = MockMembershipHandler { + membership: allowed_peers, + }; + let mut behaviour = DispersalValidatorBehaviour::new(membership); + + let allowed_peer = *behaviour.membership.members().iter().next().unwrap(); + let disallowed_peer = PeerId::random(); + let local_addr = "/ip4/127.0.0.1/tcp/8080".parse().unwrap(); + let remote_addr = "/ip4/127.0.0.1/tcp/8081".parse().unwrap(); + + let result = behaviour.handle_established_inbound_connection( + ConnectionId::new_unchecked(0), + allowed_peer, + &local_addr, + &remote_addr, + ); + assert!(matches!(result, Ok(Either::Left(_)))); + + let result = behaviour.handle_established_inbound_connection( + ConnectionId::new_unchecked(1), + disallowed_peer, + &local_addr, + &remote_addr, + ); + assert!(matches!(result, Ok(Either::Right(_)))); + } + + #[test] + fn test_poll() { + let membership = MockMembershipHandler { + membership: HashMap::new(), + }; + let mut behaviour = DispersalValidatorBehaviour::new(membership); + + let mut cx = std::task::Context::from_waker(futures::task::noop_waker_ref()); + let poll_result = behaviour.poll(&mut cx); + + assert!(matches!(poll_result, Poll::Pending)); + } } From c0b3cb1b95b04060644bd6dd2b5b7a3ee0a97e87 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 17 Oct 2024 15:36:43 +0800 Subject: [PATCH 04/52] test: attempt to open stream between two peers without swarm --- .../dispersal/validator/behaviour.rs | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 91373acd9..8d3c8829b 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -151,8 +151,11 @@ impl + 'static> Netw #[cfg(test)] mod tests { use super::*; + use crate::protocols::replication::handler::DaMessage; use futures::task::ArcWake; use libp2p::{identity, PeerId}; + use libp2p_stream::OpenStreamError; + use nomos_da_messages::common::Blob; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -234,12 +237,12 @@ mod tests { behaviours } - fn establish_connection( + async fn establish_connection( behaviours: &mut [DispersalValidatorBehaviour], i: usize, j: usize, connection_id: ConnectionId, - ) { + ) -> Result { let mut members: Vec = behaviours[i].membership.members().into_iter().collect(); members.sort(); let peer_id_i = members[i]; @@ -262,6 +265,14 @@ mod tests { &Multiaddr::empty(), ) .unwrap(); + + let mut stream_control = behaviours[i].stream_behaviour.new_control(); + let stream = stream_control + .open_stream(peer_id_j, DISPERSAL_PROTOCOL) + .await + .expect("open stream"); + + Ok(stream) } #[test] @@ -307,4 +318,42 @@ mod tests { assert!(matches!(poll_result, Poll::Pending)); } + + #[tokio::test] + async fn test_validation_behaviour() { + let num_instances = 20; + let mut membership = HashMap::default(); + + let subnet_0_behaviours = + create_validation_behaviours(num_instances / 2, 0, &mut membership); + let subnet_1_behaviours = + create_validation_behaviours(num_instances / 2, 1, &mut membership); + + let mut all_behaviours = subnet_0_behaviours; + all_behaviours.extend(subnet_1_behaviours); + + for behaviour in all_behaviours.iter_mut() { + let membership_handler = MockMembershipHandler { + membership: membership.clone(), + }; + behaviour.update_membership(membership_handler); + } + + // Simulate peer connections. + for (i, j) in (0..num_instances).flat_map(|i| (i + 1..num_instances).map(move |j| (i, j))) { + let connection_id = ConnectionId::new_unchecked(i); + let stream = establish_connection(&mut all_behaviours, i, j, connection_id) + .await + .unwrap(); + } + + // Simulate sending a message from the first behavior. + let message = DaMessage { + blob: Some(Blob { + blob_id: vec![1, 2, 3], + data: vec![4, 5, 6], + }), + subnetwork_id: 0, + }; + } } From 81e571d43434e310a29fe9ebc4c0224a84906139 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 17 Oct 2024 20:05:54 +0800 Subject: [PATCH 05/52] test: extend behavior with channels --- .../dispersal/validator/behaviour.rs | 45 ++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 8d3c8829b..e6b009054 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -151,13 +151,17 @@ impl + 'static> Netw #[cfg(test)] mod tests { use super::*; + use crate::protocols::dispersal::executor::behaviour::DispersalError; use crate::protocols::replication::handler::DaMessage; + use futures::stream::BoxStream; use futures::task::ArcWake; use libp2p::{identity, PeerId}; - use libp2p_stream::OpenStreamError; + use libp2p_stream::Control; use nomos_da_messages::common::Blob; use std::collections::{HashMap, HashSet}; use std::sync::Arc; + use tokio::sync::mpsc; + use tokio_stream::wrappers::UnboundedReceiverStream; #[derive(Clone, Debug)] struct MockMembershipHandler { @@ -237,12 +241,12 @@ mod tests { behaviours } - async fn establish_connection( + fn establish_connection( behaviours: &mut [DispersalValidatorBehaviour], i: usize, j: usize, connection_id: ConnectionId, - ) -> Result { + ) -> BoxStream { let mut members: Vec = behaviours[i].membership.members().into_iter().collect(); members.sort(); let peer_id_i = members[i]; @@ -267,11 +271,23 @@ mod tests { .unwrap(); let mut stream_control = behaviours[i].stream_behaviour.new_control(); - let stream = stream_control - .open_stream(peer_id_j, DISPERSAL_PROTOCOL) - .await - .expect("open stream"); + let (pending_out_streams_sender, receiver) = mpsc::unbounded_channel::(); + let pending_out_streams = UnboundedReceiverStream::new(receiver) + .zip(futures::stream::repeat(stream_control)) + .then(|(peer_id, control)| open_stream(peer_id_j, control)) + .boxed(); + + let (pending_blobs_sender, receiver) = mpsc::unbounded_channel::(); + let pending_blobs_stream = UnboundedReceiverStream::new(receiver).boxed(); + + pending_blobs_stream + } + async fn open_stream(peer_id: PeerId, mut control: Control) -> Result { + let stream = control + .open_stream(peer_id, DISPERSAL_PROTOCOL) + .await + .map_err(|error| DispersalError::OpenStreamError { peer_id, error })?; Ok(stream) } @@ -339,15 +355,6 @@ mod tests { behaviour.update_membership(membership_handler); } - // Simulate peer connections. - for (i, j) in (0..num_instances).flat_map(|i| (i + 1..num_instances).map(move |j| (i, j))) { - let connection_id = ConnectionId::new_unchecked(i); - let stream = establish_connection(&mut all_behaviours, i, j, connection_id) - .await - .unwrap(); - } - - // Simulate sending a message from the first behavior. let message = DaMessage { blob: Some(Blob { blob_id: vec![1, 2, 3], @@ -355,5 +362,11 @@ mod tests { }), subnetwork_id: 0, }; + + // Simulate peer connections. + for (i, j) in (0..num_instances).flat_map(|i| (i + 1..num_instances).map(move |j| (i, j))) { + let connection_id = ConnectionId::new_unchecked(i); + let stream_sender = establish_connection(&mut all_behaviours, i, j, connection_id); + } } } From cb4681de420aabac0802fcb62bc5b4e313e38c2b Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 18 Oct 2024 14:03:44 +0800 Subject: [PATCH 06/52] test: plug in behaviours into swarms --- .../dispersal/validator/behaviour.rs | 155 ++++++++---------- 1 file changed, 68 insertions(+), 87 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index e6b009054..899e4b47b 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -151,24 +151,26 @@ impl + 'static> Netw #[cfg(test)] mod tests { use super::*; - use crate::protocols::dispersal::executor::behaviour::DispersalError; + use crate::protocols::dispersal::executor::behaviour::{ + DispersalError, DispersalExecutorBehaviour, + }; use crate::protocols::replication::handler::DaMessage; - use futures::stream::BoxStream; use futures::task::ArcWake; - use libp2p::{identity, PeerId}; + use libp2p::identity::Keypair; + use libp2p::{identity, quic, PeerId}; use libp2p_stream::Control; use nomos_da_messages::common::Blob; use std::collections::{HashMap, HashSet}; use std::sync::Arc; - use tokio::sync::mpsc; - use tokio_stream::wrappers::UnboundedReceiverStream; + use tracing_subscriber::fmt::TestWriter; + use tracing_subscriber::EnvFilter; #[derive(Clone, Debug)] - struct MockMembershipHandler { - membership: HashMap>, + struct Neighbourhood { + pub membership: HashMap>, } - impl MembershipHandler for MockMembershipHandler { + impl MembershipHandler for Neighbourhood { type NetworkId = SubnetworkId; type Id = PeerId; @@ -211,12 +213,44 @@ mod tests { fn wake_by_ref(_arc_self: &Arc) {} } - fn create_validation_behaviours( - num_instances: usize, - subnet_id: u32, - membership: &mut HashMap>, - ) -> Vec> { - let mut behaviours = Vec::new(); + pub fn executor_swarm( + key: Keypair, + membership: impl MembershipHandler + 'static, + ) -> libp2p::Swarm< + DispersalExecutorBehaviour>, + > { + libp2p::SwarmBuilder::with_existing_identity(key) + .with_tokio() + .with_other_transport(|keypair| quic::tokio::Transport::new(quic::Config::new(keypair))) + .unwrap() + .with_behaviour(|_key| DispersalExecutorBehaviour::new(membership)) + .unwrap() + .with_swarm_config(|cfg| { + cfg.with_idle_connection_timeout(std::time::Duration::from_secs(u64::MAX)) + }) + .build() + } + + pub fn validator_swarm( + key: Keypair, + membership: impl MembershipHandler + 'static, + ) -> libp2p::Swarm< + DispersalValidatorBehaviour>, + > { + libp2p::SwarmBuilder::with_existing_identity(key) + .with_tokio() + .with_other_transport(|keypair| quic::tokio::Transport::new(quic::Config::new(keypair))) + .unwrap() + .with_behaviour(|_key| DispersalValidatorBehaviour::new(membership)) + .unwrap() + .with_swarm_config(|cfg| { + cfg.with_idle_connection_timeout(std::time::Duration::from_secs(u64::MAX)) + }) + .build() + } + + fn create_validation_membership(num_instances: usize, subnet_id: u32) -> Neighbourhood { + let mut membership = HashMap::default(); let mut peer_ids = Vec::new(); for _ in 0..num_instances { @@ -229,58 +263,7 @@ mod tests { membership.insert(*peer_id, HashSet::from([subnet_id])); } - let membership_handler = MockMembershipHandler { - membership: HashMap::default(), // This will be updated after all behaviours are added. - }; - - for _ in peer_ids { - let behaviour = DispersalValidatorBehaviour::new(membership_handler.clone()); - behaviours.push(behaviour); - } - - behaviours - } - - fn establish_connection( - behaviours: &mut [DispersalValidatorBehaviour], - i: usize, - j: usize, - connection_id: ConnectionId, - ) -> BoxStream { - let mut members: Vec = behaviours[i].membership.members().into_iter().collect(); - members.sort(); - let peer_id_i = members[i]; - let peer_id_j = members[j]; - - behaviours[i] - .handle_established_outbound_connection( - connection_id, - peer_id_j, - &Multiaddr::empty(), - Endpoint::Dialer, - ) - .unwrap(); - - behaviours[j] - .handle_established_inbound_connection( - connection_id, - peer_id_i, - &Multiaddr::empty(), - &Multiaddr::empty(), - ) - .unwrap(); - - let mut stream_control = behaviours[i].stream_behaviour.new_control(); - let (pending_out_streams_sender, receiver) = mpsc::unbounded_channel::(); - let pending_out_streams = UnboundedReceiverStream::new(receiver) - .zip(futures::stream::repeat(stream_control)) - .then(|(peer_id, control)| open_stream(peer_id_j, control)) - .boxed(); - - let (pending_blobs_sender, receiver) = mpsc::unbounded_channel::(); - let pending_blobs_stream = UnboundedReceiverStream::new(receiver).boxed(); - - pending_blobs_stream + Neighbourhood { membership } } async fn open_stream(peer_id: PeerId, mut control: Control) -> Result { @@ -295,7 +278,7 @@ mod tests { fn test_handle_established_inbound_connection() { let mut allowed_peers = HashMap::new(); allowed_peers.insert(PeerId::random(), HashSet::from([0, 1])); - let membership = MockMembershipHandler { + let membership = Neighbourhood { membership: allowed_peers, }; let mut behaviour = DispersalValidatorBehaviour::new(membership); @@ -324,7 +307,7 @@ mod tests { #[test] fn test_poll() { - let membership = MockMembershipHandler { + let membership = Neighbourhood { membership: HashMap::new(), }; let mut behaviour = DispersalValidatorBehaviour::new(membership); @@ -337,23 +320,27 @@ mod tests { #[tokio::test] async fn test_validation_behaviour() { + let _ = tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .compact() + .with_writer(TestWriter::default()) + .try_init(); + let k1 = libp2p::identity::Keypair::generate_ed25519(); + let k2 = libp2p::identity::Keypair::generate_ed25519(); + let validator_peer = PeerId::from_public_key(&k2.public()); + let num_instances = 20; - let mut membership = HashMap::default(); - let subnet_0_behaviours = - create_validation_behaviours(num_instances / 2, 0, &mut membership); - let subnet_1_behaviours = - create_validation_behaviours(num_instances / 2, 1, &mut membership); + let subnet_0_membership = create_validation_membership(num_instances / 2, 0); + let subnet_1_membership = create_validation_membership(num_instances / 2, 1); - let mut all_behaviours = subnet_0_behaviours; - all_behaviours.extend(subnet_1_behaviours); + let mut all_neighbours = subnet_0_membership; + all_neighbours + .membership + .extend(subnet_1_membership.membership); - for behaviour in all_behaviours.iter_mut() { - let membership_handler = MockMembershipHandler { - membership: membership.clone(), - }; - behaviour.update_membership(membership_handler); - } + let mut executor = executor_swarm(k1, all_neighbours.clone()); + let mut validator = validator_swarm(k2, all_neighbours); let message = DaMessage { blob: Some(Blob { @@ -362,11 +349,5 @@ mod tests { }), subnetwork_id: 0, }; - - // Simulate peer connections. - for (i, j) in (0..num_instances).flat_map(|i| (i + 1..num_instances).map(move |j| (i, j))) { - let connection_id = ConnectionId::new_unchecked(i); - let stream_sender = establish_connection(&mut all_behaviours, i, j, connection_id); - } } } From ac627a3885dc6d49a052b967bd17def658405c9d Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 21 Oct 2024 11:23:06 +0800 Subject: [PATCH 07/52] test: create swarms --- .../dispersal/validator/behaviour.rs | 75 +++++++++++++------ 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 899e4b47b..722083722 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -151,14 +151,14 @@ impl + 'static> Netw #[cfg(test)] mod tests { use super::*; + use crate::address_book::AddressBook; use crate::protocols::dispersal::executor::behaviour::{ - DispersalError, DispersalExecutorBehaviour, + DispersalExecutorBehaviour, }; use crate::protocols::replication::handler::DaMessage; use futures::task::ArcWake; use libp2p::identity::Keypair; use libp2p::{identity, quic, PeerId}; - use libp2p_stream::Control; use nomos_da_messages::common::Blob; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -214,7 +214,9 @@ mod tests { } pub fn executor_swarm( + addressbook: AddressBook, key: Keypair, + peer_id: PeerId, membership: impl MembershipHandler + 'static, ) -> libp2p::Swarm< DispersalExecutorBehaviour>, @@ -223,7 +225,9 @@ mod tests { .with_tokio() .with_other_transport(|keypair| quic::tokio::Transport::new(quic::Config::new(keypair))) .unwrap() - .with_behaviour(|_key| DispersalExecutorBehaviour::new(membership)) + .with_behaviour(|_key| { + DispersalExecutorBehaviour::new(peer_id, membership, addressbook) + }) .unwrap() .with_swarm_config(|cfg| { cfg.with_idle_connection_timeout(std::time::Duration::from_secs(u64::MAX)) @@ -249,31 +253,37 @@ mod tests { .build() } - fn create_validation_membership(num_instances: usize, subnet_id: u32) -> Neighbourhood { - let mut membership = HashMap::default(); + fn prepare_swarm_config(num_instances: usize) -> Vec<(Keypair, PeerId, AddressBook)> { + let mut configs = Vec::with_capacity(num_instances); - let mut peer_ids = Vec::new(); for _ in 0..num_instances { let keypair = identity::Keypair::generate_ed25519(); let peer_id = PeerId::from(keypair.public()); - peer_ids.push(peer_id); + + let addr: Multiaddr = "/ip4/127.0.0.1/udp/5063/quic-v1".parse().unwrap(); + let addressbook = AddressBook::from_iter([( + PeerId::from_public_key(&keypair.public()), + addr.clone(), + )]); + configs.push((keypair, peer_id, addressbook)); } + configs + } - for peer_id in &peer_ids { - membership.insert(*peer_id, HashSet::from([subnet_id])); + fn create_membership( + num_instances: usize, + subnet_id: u32, + peer_ids: &[PeerId], + ) -> Neighbourhood { + let mut membership = HashMap::default(); + + for i in 0..num_instances { + membership.insert(peer_ids[i], HashSet::from([subnet_id])); } Neighbourhood { membership } } - async fn open_stream(peer_id: PeerId, mut control: Control) -> Result { - let stream = control - .open_stream(peer_id, DISPERSAL_PROTOCOL) - .await - .map_err(|error| DispersalError::OpenStreamError { peer_id, error })?; - Ok(stream) - } - #[test] fn test_handle_established_inbound_connection() { let mut allowed_peers = HashMap::new(); @@ -325,22 +335,39 @@ mod tests { .compact() .with_writer(TestWriter::default()) .try_init(); - let k1 = libp2p::identity::Keypair::generate_ed25519(); - let k2 = libp2p::identity::Keypair::generate_ed25519(); - let validator_peer = PeerId::from_public_key(&k2.public()); let num_instances = 20; - let subnet_0_membership = create_validation_membership(num_instances / 2, 0); - let subnet_1_membership = create_validation_membership(num_instances / 2, 1); + let subnet_0_config = prepare_swarm_config(num_instances / 2); + let subnet_0_ids = subnet_0_config + .iter() + .map(|(_, peer_id, _)| peer_id.clone()) + .collect::>(); + let subnet_1_config = prepare_swarm_config(num_instances / 2); + let subnet_1_ids = subnet_1_config + .iter() + .map(|(_, peer_id, _)| peer_id.clone()) + .collect::>(); + + let subnet_0_membership = create_membership(num_instances / 2, 1, &subnet_0_ids); + let subnet_1_membership = create_membership(num_instances / 2, 1, &subnet_1_ids); let mut all_neighbours = subnet_0_membership; all_neighbours .membership .extend(subnet_1_membership.membership); - let mut executor = executor_swarm(k1, all_neighbours.clone()); - let mut validator = validator_swarm(k2, all_neighbours); + // create swarms + let mut executors_swarms: Vec<_> = vec![]; + let mut validator_swarms: Vec<_> = vec![]; + for i in 0..num_instances / 2 { + let cfg_0 = subnet_0_config[i].clone(); + let cfg_1 = subnet_1_config[i].clone(); + let executor = executor_swarm(cfg_0.2, cfg_0.0, cfg_0.1, all_neighbours.clone()); + let validator = validator_swarm(cfg_1.0, all_neighbours.clone()); + executors_swarms.push(executor); + validator_swarms.push(validator); + } let message = DaMessage { blob: Some(Blob { From d18144d4af3a766d0a9fc5bb5e9e416fd4d4009c Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 21 Oct 2024 12:33:44 +0800 Subject: [PATCH 08/52] fix: formatting --- .../core/src/protocols/dispersal/validator/behaviour.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 722083722..a9ecbafec 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -152,9 +152,7 @@ impl + 'static> Netw mod tests { use super::*; use crate::address_book::AddressBook; - use crate::protocols::dispersal::executor::behaviour::{ - DispersalExecutorBehaviour, - }; + use crate::protocols::dispersal::executor::behaviour::DispersalExecutorBehaviour; use crate::protocols::replication::handler::DaMessage; use futures::task::ArcWake; use libp2p::identity::Keypair; From 1a7d01f032977093b52aa72b60e0ba8f77c79561 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 22 Oct 2024 14:25:11 +0800 Subject: [PATCH 09/52] test: send messages using swarms --- .../dispersal/validator/behaviour.rs | 125 +++++++++++++++--- 1 file changed, 103 insertions(+), 22 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index c544f5222..a5d41e103 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -150,16 +150,24 @@ impl + 'static> Netw mod tests { use super::*; use crate::address_book::AddressBook; - use crate::protocols::dispersal::executor::behaviour::DispersalExecutorBehaviour; + use crate::protocols::dispersal::executor::behaviour::{DispersalExecutorBehaviour, DispersalExecutorEvent}; use crate::protocols::replication::handler::DaMessage; use futures::task::ArcWake; use libp2p::identity::Keypair; - use libp2p::{identity, quic, PeerId}; + use libp2p::{identity, quic, PeerId, Swarm}; use nomos_da_messages::common::Blob; use std::collections::{HashMap, HashSet}; use std::sync::Arc; + use std::time::Duration; + use libp2p::swarm::SwarmEvent; + use tokio::time; + use tracing::{error, info}; use tracing_subscriber::fmt::TestWriter; use tracing_subscriber::EnvFilter; + use kzgrs_backend::common::blob::DaBlob; + use kzgrs_backend::common::Column; + use crate::behaviour::validator::ValidatorBehaviourEvent; + use crate::protocols::sampling::behaviour::{SamplingBehaviour, SamplingEvent}; #[derive(Clone, Debug)] struct Neighbourhood { @@ -210,13 +218,15 @@ mod tests { } pub fn executor_swarm( - addressbook: AddressBook, + addr: Multiaddr, key: Keypair, peer_id: PeerId, membership: impl MembershipHandler + 'static, ) -> libp2p::Swarm< DispersalExecutorBehaviour>, > { + let addressbook = + AddressBook::from_iter([(peer_id, addr.clone())]); libp2p::SwarmBuilder::with_existing_identity(key) .with_tokio() .with_other_transport(|keypair| quic::tokio::Transport::new(quic::Config::new(keypair))) @@ -249,19 +259,16 @@ mod tests { .build() } - fn prepare_swarm_config(num_instances: usize) -> Vec<(Keypair, PeerId, AddressBook)> { + fn prepare_swarm_config(num_instances: usize) -> Vec<(Keypair, PeerId, Multiaddr)> { let mut configs = Vec::with_capacity(num_instances); - for _ in 0..num_instances { + for i in 0..num_instances { let keypair = identity::Keypair::generate_ed25519(); let peer_id = PeerId::from(keypair.public()); + let port = 5100 + i; + let addr: Multiaddr = format!("/ip4/127.0.0.1/udp/{port}/quic-v1").parse().unwrap(); - let addr: Multiaddr = "/ip4/127.0.0.1/udp/5063/quic-v1".parse().unwrap(); - let addressbook = AddressBook::from_iter([( - PeerId::from_public_key(&keypair.public()), - addr.clone(), - )]); - configs.push((keypair, peer_id, addressbook)); + configs.push((keypair, peer_id, addr)); } configs } @@ -332,7 +339,7 @@ mod tests { .with_writer(TestWriter::default()) .try_init(); - let num_instances = 20; + let num_instances = 2; let subnet_0_config = prepare_swarm_config(num_instances / 2); let subnet_0_ids = subnet_0_config @@ -357,20 +364,94 @@ mod tests { let mut executors_swarms: Vec<_> = vec![]; let mut validator_swarms: Vec<_> = vec![]; for i in 0..num_instances / 2 { - let cfg_0 = subnet_0_config[i].clone(); - let cfg_1 = subnet_1_config[i].clone(); - let executor = executor_swarm(cfg_0.2, cfg_0.0, cfg_0.1, all_neighbours.clone()); - let validator = validator_swarm(cfg_1.0, all_neighbours.clone()); + let (keypair, peer_id, addr) = subnet_0_config[i].clone(); + let (keypair2, peer_id2, addr2) = subnet_1_config[i].clone(); + let executor = executor_swarm(addr, keypair, peer_id, all_neighbours.clone()); + let validator = validator_swarm(keypair2, all_neighbours.clone()); executors_swarms.push(executor); validator_swarms.push(validator); } - let message = DaMessage { - blob: Some(Blob { - blob_id: vec![1, 2, 3], - data: vec![4, 5, 6], - }), - subnetwork_id: 0, + let (validator_key, validator_id, validator_addr) = subnet_1_config[0].clone(); + validator_swarms[0].listen_on(validator_addr.clone()).unwrap(); + let validator_addr_p2p = validator_addr.with_p2p(validator_id).unwrap(); + + let blobs_sender = executors_swarms[0].behaviour().blobs_sender(); + + let msg_count = 10usize; + let validator_task = async move { + validator_swarms[0].listen_on(validator_addr_p2p).unwrap(); + + let mut res = vec![]; + loop { + match validator_swarms[0].next().await { + Some(SwarmEvent::Behaviour(ValidatorBehaviourEvent::Dispersal(event))) => { + res.push(event); + } + event => { + info!("Validator event: {event:?}"); + } + } + if res.len() == msg_count { + tokio::time::sleep(Duration::from_secs(1)).await; + break; + } + } + res + }; + let join_validator = tokio::spawn(validator_task); + + executors_swarms[0].dial(validator_addr_p2p.clone()).unwrap(); + + let executor_open_stream_sender = executors_swarms[0].behaviour().open_stream_sender(); + let executor_disperse_blob_sender = executors_swarms[0].behaviour().blobs_sender(); + + let executor_poll = async move { + let mut res = vec![]; + loop { + tokio::select! { + Some(event) = executors_swarms[0].next() => { + info!("Executor event: {event:?}"); + if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess{blob_id, ..}) = event { + res.push(blob_id); + } + } + + _ = time::sleep(Duration::from_secs(2)) => { + if res.len() < msg_count {error!("Executor timeout reached");} + break; + } + } + } + res }; + + let executor_task = tokio::spawn(executor_poll); + + executor_open_stream_sender.send(validator_id).unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + + for i in 0..msg_count { + info!("Sending blob {i}..."); + executor_disperse_blob_sender + .send(( + 0, + DaBlob { + column_idx: 0, + column: Column(vec![]), + column_commitment: Default::default(), + aggregated_column_commitment: Default::default(), + aggregated_column_proof: Default::default(), + rows_commitments: vec![], + rows_proofs: vec![], + }, + )) + .unwrap(); + } + + assert_eq!( + executor_task.await.unwrap().len(), + join_validator.await.unwrap().len() + ); } } From 6861dc18a636022076f01f945a9431730ad8df60 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 22 Oct 2024 20:08:36 +0800 Subject: [PATCH 10/52] test: executor > validator auto connect --- .../dispersal/validator/behaviour.rs | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index a5d41e103..41cab887b 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -218,15 +218,13 @@ mod tests { } pub fn executor_swarm( - addr: Multiaddr, + addressbook: AddressBook, key: Keypair, peer_id: PeerId, membership: impl MembershipHandler + 'static, ) -> libp2p::Swarm< DispersalExecutorBehaviour>, > { - let addressbook = - AddressBook::from_iter([(peer_id, addr.clone())]); libp2p::SwarmBuilder::with_existing_identity(key) .with_tokio() .with_other_transport(|keypair| quic::tokio::Transport::new(quic::Config::new(keypair))) @@ -361,39 +359,41 @@ mod tests { .extend(subnet_1_membership.membership); // create swarms - let mut executors_swarms: Vec<_> = vec![]; + let mut executor_swarms: Vec<_> = vec![]; let mut validator_swarms: Vec<_> = vec![]; + for i in 0..num_instances / 2 { - let (keypair, peer_id, addr) = subnet_0_config[i].clone(); - let (keypair2, peer_id2, addr2) = subnet_1_config[i].clone(); - let executor = executor_swarm(addr, keypair, peer_id, all_neighbours.clone()); - let validator = validator_swarm(keypair2, all_neighbours.clone()); - executors_swarms.push(executor); + let (k, executor_peer, addr) = subnet_0_config[i].clone(); + let (k2, validator_peer, addr2) = subnet_1_config[i].clone(); + let addressbook = + AddressBook::from_iter([(validator_peer, addr2.clone())]); + let executor = executor_swarm(addressbook, k, executor_peer, all_neighbours.clone()); + let validator = validator_swarm(k2, all_neighbours.clone()); + executor_swarms.push(executor); validator_swarms.push(validator); } let (validator_key, validator_id, validator_addr) = subnet_1_config[0].clone(); - validator_swarms[0].listen_on(validator_addr.clone()).unwrap(); - let validator_addr_p2p = validator_addr.with_p2p(validator_id).unwrap(); + let validator_addr_p2p = validator_addr.clone().with_p2p(validator_id).unwrap(); - let blobs_sender = executors_swarms[0].behaviour().blobs_sender(); + let blobs_sender = executor_swarms[0].behaviour().blobs_sender(); let msg_count = 10usize; let validator_task = async move { - validator_swarms[0].listen_on(validator_addr_p2p).unwrap(); + validator_swarms[0].listen_on(validator_addr).unwrap(); let mut res = vec![]; loop { match validator_swarms[0].next().await { - Some(SwarmEvent::Behaviour(ValidatorBehaviourEvent::Dispersal(event))) => { - res.push(event); + Some(SwarmEvent::Behaviour(DispersalEvent::IncomingMessage {message})) => { + res.push(message); } event => { info!("Validator event: {event:?}"); } } if res.len() == msg_count { - tokio::time::sleep(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_secs(2)).await; break; } } @@ -401,23 +401,21 @@ mod tests { }; let join_validator = tokio::spawn(validator_task); - executors_swarms[0].dial(validator_addr_p2p.clone()).unwrap(); - - let executor_open_stream_sender = executors_swarms[0].behaviour().open_stream_sender(); - let executor_disperse_blob_sender = executors_swarms[0].behaviour().blobs_sender(); + let executor_open_stream_sender = executor_swarms[0].behaviour().open_stream_sender(); + let executor_disperse_blob_sender = executor_swarms[0].behaviour().blobs_sender(); let executor_poll = async move { let mut res = vec![]; loop { tokio::select! { - Some(event) = executors_swarms[0].next() => { + Some(event) = executor_swarms[0].next() => { info!("Executor event: {event:?}"); if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess{blob_id, ..}) = event { res.push(blob_id); } } - _ = time::sleep(Duration::from_secs(2)) => { + _ = time::sleep(Duration::from_secs(20)) => { if res.len() < msg_count {error!("Executor timeout reached");} break; } @@ -428,8 +426,7 @@ mod tests { let executor_task = tokio::spawn(executor_poll); - executor_open_stream_sender.send(validator_id).unwrap(); - tokio::time::sleep(Duration::from_secs(1)).await; + //executor_open_stream_sender.send(validator_id).unwrap(); for i in 0..msg_count { info!("Sending blob {i}..."); From 3eeafc819ce53f591033c98a8694d7697d7c228d Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 23 Oct 2024 15:44:11 +0800 Subject: [PATCH 11/52] fix: address to contain p2p address --- .../dispersal/validator/behaviour.rs | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 41cab887b..875899136 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -150,24 +150,26 @@ impl + 'static> Netw mod tests { use super::*; use crate::address_book::AddressBook; - use crate::protocols::dispersal::executor::behaviour::{DispersalExecutorBehaviour, DispersalExecutorEvent}; + use crate::behaviour::validator::ValidatorBehaviourEvent; + use crate::protocols::dispersal::executor::behaviour::{ + DispersalExecutorBehaviour, DispersalExecutorEvent, + }; use crate::protocols::replication::handler::DaMessage; + use crate::protocols::sampling::behaviour::{SamplingBehaviour, SamplingEvent}; use futures::task::ArcWake; + use kzgrs_backend::common::blob::DaBlob; + use kzgrs_backend::common::Column; use libp2p::identity::Keypair; + use libp2p::swarm::SwarmEvent; use libp2p::{identity, quic, PeerId, Swarm}; use nomos_da_messages::common::Blob; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; - use libp2p::swarm::SwarmEvent; use tokio::time; use tracing::{error, info}; use tracing_subscriber::fmt::TestWriter; use tracing_subscriber::EnvFilter; - use kzgrs_backend::common::blob::DaBlob; - use kzgrs_backend::common::Column; - use crate::behaviour::validator::ValidatorBehaviourEvent; - use crate::protocols::sampling::behaviour::{SamplingBehaviour, SamplingEvent}; #[derive(Clone, Debug)] struct Neighbourhood { @@ -225,6 +227,8 @@ mod tests { ) -> libp2p::Swarm< DispersalExecutorBehaviour>, > { + info!("Creating executor_swarm with peerID {}", peer_id); + libp2p::SwarmBuilder::with_existing_identity(key) .with_tokio() .with_other_transport(|keypair| quic::tokio::Transport::new(quic::Config::new(keypair))) @@ -264,7 +268,9 @@ mod tests { let keypair = identity::Keypair::generate_ed25519(); let peer_id = PeerId::from(keypair.public()); let port = 5100 + i; - let addr: Multiaddr = format!("/ip4/127.0.0.1/udp/{port}/quic-v1").parse().unwrap(); + let addr: Multiaddr = format!("/ip4/127.0.0.1/udp/{port}/quic-v1") + .parse() + .unwrap(); configs.push((keypair, peer_id, addr)); } @@ -344,12 +350,19 @@ mod tests { .iter() .map(|(_, peer_id, _)| peer_id.clone()) .collect::>(); + let subnet_1_config = prepare_swarm_config(num_instances / 2); let subnet_1_ids = subnet_1_config .iter() .map(|(_, peer_id, _)| peer_id.clone()) .collect::>(); + let addressbook = AddressBook::from_iter( + subnet_1_config + .iter() + .map(|(_, peer_id, addr)| (peer_id.clone(), addr.clone().with_p2p(peer_id.clone()).unwrap())), + ); + let subnet_0_membership = create_membership(num_instances / 2, 1, &subnet_0_ids); let subnet_1_membership = create_membership(num_instances / 2, 1, &subnet_1_ids); @@ -365,18 +378,14 @@ mod tests { for i in 0..num_instances / 2 { let (k, executor_peer, addr) = subnet_0_config[i].clone(); let (k2, validator_peer, addr2) = subnet_1_config[i].clone(); - let addressbook = - AddressBook::from_iter([(validator_peer, addr2.clone())]); - let executor = executor_swarm(addressbook, k, executor_peer, all_neighbours.clone()); + let executor = executor_swarm(addressbook.clone(), k, executor_peer, all_neighbours.clone()); let validator = validator_swarm(k2, all_neighbours.clone()); executor_swarms.push(executor); validator_swarms.push(validator); } let (validator_key, validator_id, validator_addr) = subnet_1_config[0].clone(); - let validator_addr_p2p = validator_addr.clone().with_p2p(validator_id).unwrap(); - - let blobs_sender = executor_swarms[0].behaviour().blobs_sender(); + let validator_addr_p2p = validator_addr.clone().with_p2p(validator_id.clone()).unwrap(); let msg_count = 10usize; let validator_task = async move { @@ -385,7 +394,7 @@ mod tests { let mut res = vec![]; loop { match validator_swarms[0].next().await { - Some(SwarmEvent::Behaviour(DispersalEvent::IncomingMessage {message})) => { + Some(SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message })) => { res.push(message); } event => { @@ -401,8 +410,10 @@ mod tests { }; let join_validator = tokio::spawn(validator_task); - let executor_open_stream_sender = executor_swarms[0].behaviour().open_stream_sender(); + //executor_swarms[0].dial(validator_addr_p2p).unwrap(); + let executor_disperse_blob_sender = executor_swarms[0].behaviour().blobs_sender(); + let executor_open_stream_sender = executor_swarms[0].behaviour().open_stream_sender(); let executor_poll = async move { let mut res = vec![]; From 54e693914141fbde186203bc0c39124d06dbda14 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 23 Oct 2024 18:26:28 +0800 Subject: [PATCH 12/52] fix: have executors and validators in the same subnet_id = 0 --- .../src/protocols/dispersal/validator/behaviour.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 875899136..d2ef45546 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -363,8 +363,8 @@ mod tests { .map(|(_, peer_id, addr)| (peer_id.clone(), addr.clone().with_p2p(peer_id.clone()).unwrap())), ); - let subnet_0_membership = create_membership(num_instances / 2, 1, &subnet_0_ids); - let subnet_1_membership = create_membership(num_instances / 2, 1, &subnet_1_ids); + let subnet_0_membership = create_membership(num_instances / 2, 0, &subnet_0_ids); + let subnet_1_membership = create_membership(num_instances / 2, 0, &subnet_1_ids); let mut all_neighbours = subnet_0_membership; all_neighbours @@ -410,10 +410,7 @@ mod tests { }; let join_validator = tokio::spawn(validator_task); - //executor_swarms[0].dial(validator_addr_p2p).unwrap(); - let executor_disperse_blob_sender = executor_swarms[0].behaviour().blobs_sender(); - let executor_open_stream_sender = executor_swarms[0].behaviour().open_stream_sender(); let executor_poll = async move { let mut res = vec![]; @@ -426,7 +423,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(20)) => { + _ = time::sleep(Duration::from_secs(1)) => { if res.len() < msg_count {error!("Executor timeout reached");} break; } @@ -437,8 +434,6 @@ mod tests { let executor_task = tokio::spawn(executor_poll); - //executor_open_stream_sender.send(validator_id).unwrap(); - for i in 0..msg_count { info!("Sending blob {i}..."); executor_disperse_blob_sender From ecd45c12e4725291018d1794cc990f60aa8eaa44 Mon Sep 17 00:00:00 2001 From: Roman Date: Fri, 25 Oct 2024 16:27:35 +0800 Subject: [PATCH 13/52] test: calling poll on validator_behaviour --- .../dispersal/validator/behaviour.rs | 95 ++++++++++++------- 1 file changed, 60 insertions(+), 35 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index d2ef45546..746e8116a 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -156,7 +156,7 @@ mod tests { }; use crate::protocols::replication::handler::DaMessage; use crate::protocols::sampling::behaviour::{SamplingBehaviour, SamplingEvent}; - use futures::task::ArcWake; + use futures::task::{waker_ref, ArcWake}; use kzgrs_backend::common::blob::DaBlob; use kzgrs_backend::common::Column; use libp2p::identity::Keypair; @@ -357,11 +357,13 @@ mod tests { .map(|(_, peer_id, _)| peer_id.clone()) .collect::>(); - let addressbook = AddressBook::from_iter( - subnet_1_config - .iter() - .map(|(_, peer_id, addr)| (peer_id.clone(), addr.clone().with_p2p(peer_id.clone()).unwrap())), - ); + let addressbook = + AddressBook::from_iter(subnet_1_config.iter().map(|(_, peer_id, addr)| { + ( + peer_id.clone(), + addr.clone().with_p2p(peer_id.clone()).unwrap(), + ) + })); let subnet_0_membership = create_membership(num_instances / 2, 0, &subnet_0_ids); let subnet_1_membership = create_membership(num_instances / 2, 0, &subnet_1_ids); @@ -378,40 +380,29 @@ mod tests { for i in 0..num_instances / 2 { let (k, executor_peer, addr) = subnet_0_config[i].clone(); let (k2, validator_peer, addr2) = subnet_1_config[i].clone(); - let executor = executor_swarm(addressbook.clone(), k, executor_peer, all_neighbours.clone()); + let executor = executor_swarm( + addressbook.clone(), + k, + executor_peer, + all_neighbours.clone(), + ); let validator = validator_swarm(k2, all_neighbours.clone()); executor_swarms.push(executor); validator_swarms.push(validator); } let (validator_key, validator_id, validator_addr) = subnet_1_config[0].clone(); - let validator_addr_p2p = validator_addr.clone().with_p2p(validator_id.clone()).unwrap(); + let validator_addr_p2p = validator_addr + .clone() + .with_p2p(validator_id.clone()) + .unwrap(); - let msg_count = 10usize; - let validator_task = async move { - validator_swarms[0].listen_on(validator_addr).unwrap(); - - let mut res = vec![]; - loop { - match validator_swarms[0].next().await { - Some(SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message })) => { - res.push(message); - } - event => { - info!("Validator event: {event:?}"); - } - } - if res.len() == msg_count { - tokio::time::sleep(Duration::from_secs(2)).await; - break; - } - } - res - }; - let join_validator = tokio::spawn(validator_task); + validator_swarms[0].listen_on(validator_addr).unwrap(); + let validator_behaviour = validator_swarms[0].behaviour_mut(); let executor_disperse_blob_sender = executor_swarms[0].behaviour().blobs_sender(); + let msg_count = 10usize; let executor_poll = async move { let mut res = vec![]; loop { @@ -423,7 +414,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(1)) => { + _ = time::sleep(Duration::from_secs(10)) => { if res.len() < msg_count {error!("Executor timeout reached");} break; } @@ -452,9 +443,43 @@ mod tests { .unwrap(); } - assert_eq!( - executor_task.await.unwrap().len(), - join_validator.await.unwrap().len() - ); + tokio::time::sleep(Duration::from_secs(1)).await; + + // let mut res = vec![]; + // loop { + // match validator_swarms[0].next().await { + // Some(SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message })) => { + // res.push(message); + // } + // event => { + // info!("Validator event: {event:?}"); + // } + // } + // if res.len() == msg_count { + // tokio::time::sleep(Duration::from_secs(2)).await; + // break; + // } + // } + + let waker = Arc::new(TestWaker); + let waker_ref = waker_ref(&waker); + let mut cx = Context::from_waker(&waker_ref); + + let mut res = vec![]; + loop { + match validator_behaviour.poll(&mut cx) { + event => { + info!("Validator collected event: {event:?}"); + res.push(event); + } + } + + if res.len() == msg_count { + tokio::time::sleep(Duration::from_secs(2)).await; + break; + } + } + + executor_task.await.unwrap(); } } From 91577aaeac60d3aa473449c8c4306ce82dbd626d Mon Sep 17 00:00:00 2001 From: Roman Date: Sat, 26 Oct 2024 15:53:44 +0800 Subject: [PATCH 14/52] test: check incoming data structure and content --- .../dispersal/validator/behaviour.rs | 61 +++++++++---------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 746e8116a..def16f069 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -150,18 +150,15 @@ impl + 'static> Netw mod tests { use super::*; use crate::address_book::AddressBook; - use crate::behaviour::validator::ValidatorBehaviourEvent; use crate::protocols::dispersal::executor::behaviour::{ DispersalExecutorBehaviour, DispersalExecutorEvent, }; - use crate::protocols::replication::handler::DaMessage; - use crate::protocols::sampling::behaviour::{SamplingBehaviour, SamplingEvent}; - use futures::task::{waker_ref, ArcWake}; + use futures::task::ArcWake; use kzgrs_backend::common::blob::DaBlob; use kzgrs_backend::common::Column; use libp2p::identity::Keypair; use libp2p::swarm::SwarmEvent; - use libp2p::{identity, quic, PeerId, Swarm}; + use libp2p::{identity, quic, PeerId}; use nomos_da_messages::common::Blob; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -414,7 +411,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(10)) => { + _ = time::sleep(Duration::from_secs(2)) => { if res.len() < msg_count {error!("Executor timeout reached");} break; } @@ -445,36 +442,34 @@ mod tests { tokio::time::sleep(Duration::from_secs(1)).await; - // let mut res = vec![]; - // loop { - // match validator_swarms[0].next().await { - // Some(SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message })) => { - // res.push(message); - // } - // event => { - // info!("Validator event: {event:?}"); - // } - // } - // if res.len() == msg_count { - // tokio::time::sleep(Duration::from_secs(2)).await; - // break; - // } - // } - - let waker = Arc::new(TestWaker); - let waker_ref = waker_ref(&waker); - let mut cx = Context::from_waker(&waker_ref); - - let mut res = vec![]; + let mut msg_counter = 0; loop { - match validator_behaviour.poll(&mut cx) { - event => { - info!("Validator collected event: {event:?}"); - res.push(event); + match validator_swarms[0].next().await { + Some(SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message })) => { + info!("Validator received blob: {message:?}"); + + // Check data has structure and content as expected + match message.blob { + Some(Blob { blob_id, data }) => { + let deserialized_blob: DaBlob = bincode::deserialize(&data).unwrap(); + assert_eq!(blob_id, deserialized_blob.id()); + msg_counter += 1; + } + None => {} + } + match message.subnetwork_id { + subnet_id => { + assert_eq!(subnet_id, 0); + } + } + } + Some(event) => { + info!("Validator received event: {event:?}"); } - } - if res.len() == msg_count { + _ => {} + } + if msg_counter == msg_count { tokio::time::sleep(Duration::from_secs(2)).await; break; } From 7400f6329a1654b3e7ecf3ffbd29efe1a8fa6138 Mon Sep 17 00:00:00 2001 From: Roman Date: Sat, 26 Oct 2024 19:41:52 +0800 Subject: [PATCH 15/52] test: generate 2 groups of executors and validators --- .../dispersal/validator/behaviour.rs | 65 +++++++++++++------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index def16f069..f2c99f982 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -258,13 +258,16 @@ mod tests { .build() } - fn prepare_swarm_config(num_instances: usize) -> Vec<(Keypair, PeerId, Multiaddr)> { + fn prepare_swarm_config( + num_instances: usize, + group_id: usize, + ) -> Vec<(Keypair, PeerId, Multiaddr)> { let mut configs = Vec::with_capacity(num_instances); for i in 0..num_instances { let keypair = identity::Keypair::generate_ed25519(); let peer_id = PeerId::from(keypair.public()); - let port = 5100 + i; + let port = 5100 + group_id * num_instances + i; let addr: Multiaddr = format!("/ip4/127.0.0.1/udp/{port}/quic-v1") .parse() .unwrap(); @@ -340,30 +343,50 @@ mod tests { .with_writer(TestWriter::default()) .try_init(); - let num_instances = 2; + let all_instances = 20; + + let executor_0_config = prepare_swarm_config(all_instances / 4, 0); + let validator_0_config = prepare_swarm_config(all_instances / 4, 1); - let subnet_0_config = prepare_swarm_config(num_instances / 2); - let subnet_0_ids = subnet_0_config + let executor_1_config = prepare_swarm_config(all_instances / 4, 2); + let validator_1_config = prepare_swarm_config(all_instances / 4, 3); + + let subnet_0_ids = executor_0_config .iter() .map(|(_, peer_id, _)| peer_id.clone()) + .chain( + validator_0_config + .iter() + .map(|(_, peer_id, _)| peer_id.clone()), + ) .collect::>(); - let subnet_1_config = prepare_swarm_config(num_instances / 2); - let subnet_1_ids = subnet_1_config + let subnet_1_ids = executor_1_config .iter() .map(|(_, peer_id, _)| peer_id.clone()) + .chain( + validator_1_config + .iter() + .map(|(_, peer_id, _)| peer_id.clone()), + ) .collect::>(); - let addressbook = - AddressBook::from_iter(subnet_1_config.iter().map(|(_, peer_id, addr)| { - ( - peer_id.clone(), - addr.clone().with_p2p(peer_id.clone()).unwrap(), - ) - })); + let to_p2p_address = |(_, peer_id, addr): &(_, PeerId, Multiaddr)| { + ( + peer_id.clone(), + addr.clone().with_p2p(peer_id.clone()).unwrap(), + ) + }; + + let validator_addressbook = AddressBook::from_iter( + validator_0_config + .iter() + .map(to_p2p_address) + .chain(validator_1_config.iter().map(to_p2p_address)), + ); - let subnet_0_membership = create_membership(num_instances / 2, 0, &subnet_0_ids); - let subnet_1_membership = create_membership(num_instances / 2, 0, &subnet_1_ids); + let subnet_0_membership = create_membership(all_instances / 2, 0, &subnet_0_ids); + let subnet_1_membership = create_membership(all_instances / 2, 0, &subnet_1_ids); let mut all_neighbours = subnet_0_membership; all_neighbours @@ -374,11 +397,11 @@ mod tests { let mut executor_swarms: Vec<_> = vec![]; let mut validator_swarms: Vec<_> = vec![]; - for i in 0..num_instances / 2 { - let (k, executor_peer, addr) = subnet_0_config[i].clone(); - let (k2, validator_peer, addr2) = subnet_1_config[i].clone(); + for i in 0..all_instances / 2 { + let (k, executor_peer, addr) = executor_0_config[i].clone(); + let (k2, validator_peer, addr2) = validator_0_config[i].clone(); let executor = executor_swarm( - addressbook.clone(), + validator_addressbook.clone(), k, executor_peer, all_neighbours.clone(), @@ -388,7 +411,7 @@ mod tests { validator_swarms.push(validator); } - let (validator_key, validator_id, validator_addr) = subnet_1_config[0].clone(); + let (validator_key, validator_id, validator_addr) = validator_0_config[0].clone(); let validator_addr_p2p = validator_addr .clone() .with_p2p(validator_id.clone()) From 4bb706c6cc178890c1c9a8010bc4078a5e774c73 Mon Sep 17 00:00:00 2001 From: Roman Date: Sun, 27 Oct 2024 19:58:01 +0800 Subject: [PATCH 16/52] test: get everything into motion --- .../dispersal/validator/behaviour.rs | 225 ++++++++++++------ 1 file changed, 154 insertions(+), 71 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index f2c99f982..0fa9cc787 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -150,19 +150,24 @@ impl + 'static> Netw mod tests { use super::*; use crate::address_book::AddressBook; + use crate::behaviour::executor::ExecutorBehaviour; use crate::protocols::dispersal::executor::behaviour::{ DispersalExecutorBehaviour, DispersalExecutorEvent, }; + use crate::protocols::sampling::behaviour::SamplingBehaviour; use futures::task::ArcWake; use kzgrs_backend::common::blob::DaBlob; use kzgrs_backend::common::Column; use libp2p::identity::Keypair; use libp2p::swarm::SwarmEvent; - use libp2p::{identity, quic, PeerId}; + use libp2p::{identity, quic, PeerId, Swarm}; use nomos_da_messages::common::Blob; use std::collections::{HashMap, HashSet}; + use std::mem; use std::sync::Arc; use std::time::Duration; + use futures::future::join_all; + use tokio::sync::mpsc::UnboundedSender; use tokio::time; use tracing::{error, info}; use tracing_subscriber::fmt::TestWriter; @@ -394,110 +399,188 @@ mod tests { .extend(subnet_1_membership.membership); // create swarms - let mut executor_swarms: Vec<_> = vec![]; - let mut validator_swarms: Vec<_> = vec![]; + let mut executor_0_swarms: Vec<_> = vec![]; + let mut validator_0_swarms: Vec<_> = vec![]; + let mut executor_1_swarms: Vec<_> = vec![]; + let mut validator_1_swarms: Vec<_> = vec![]; - for i in 0..all_instances / 2 { + for i in 0..all_instances / 4 { let (k, executor_peer, addr) = executor_0_config[i].clone(); - let (k2, validator_peer, addr2) = validator_0_config[i].clone(); - let executor = executor_swarm( + let (k2, _, addr2) = validator_0_config[i].clone(); + let (k3, executor_peer2, addr3) = executor_1_config[i].clone(); + let (k4, _, addr4) = validator_1_config[i].clone(); + + let executor_0 = executor_swarm( validator_addressbook.clone(), k, executor_peer, all_neighbours.clone(), ); - let validator = validator_swarm(k2, all_neighbours.clone()); - executor_swarms.push(executor); - validator_swarms.push(validator); + let validator_0 = validator_swarm(k2, all_neighbours.clone()); + executor_0_swarms.push(executor_0); + validator_0_swarms.push(validator_0); + + let executor_1 = executor_swarm( + validator_addressbook.clone(), + k3, + executor_peer2, + all_neighbours.clone(), + ); + let validator_1 = validator_swarm(k4, all_neighbours.clone()); + executor_1_swarms.push(executor_1); + validator_1_swarms.push(validator_1); } - let (validator_key, validator_id, validator_addr) = validator_0_config[0].clone(); - let validator_addr_p2p = validator_addr - .clone() - .with_p2p(validator_id.clone()) - .unwrap(); + // Let validator swarms to listen + for i in 0..all_instances / 4 { + let (_, peer_id, addr) = validator_0_config[i].clone(); + let validator_0_addr = addr.clone().with_p2p(peer_id).unwrap(); + validator_0_swarms[i].listen_on(validator_0_addr).unwrap(); - validator_swarms[0].listen_on(validator_addr).unwrap(); + let (_, peer_id, addr) = validator_1_config[i].clone(); + let validator_1_addr = addr.clone().with_p2p(peer_id).unwrap(); + validator_1_swarms[i].listen_on(validator_1_addr).unwrap(); + } + + tokio::time::sleep(Duration::from_secs(5)).await; - let validator_behaviour = validator_swarms[0].behaviour_mut(); - let executor_disperse_blob_sender = executor_swarms[0].behaviour().blobs_sender(); + let mut message_senders: Vec> = Vec::new(); - let msg_count = 10usize; - let executor_poll = async move { - let mut res = vec![]; + for i in 0..all_instances / 4 { + let blob_sender_0 = executor_0_swarms[i].behaviour().blobs_sender(); + let blob_sender_1 = executor_1_swarms[i].behaviour().blobs_sender(); + message_senders.extend(vec![blob_sender_0, blob_sender_1]); + } + + async fn run_executor_swarm( + mut swarm: Swarm + Sized + 'static>> + ) { + let mut msg_counter = 0; loop { tokio::select! { - Some(event) = executor_swarms[0].next() => { + Some(event) = swarm.next() => { info!("Executor event: {event:?}"); if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess{blob_id, ..}) = event { - res.push(blob_id); + msg_counter += 1; } } _ = time::sleep(Duration::from_secs(2)) => { - if res.len() < msg_count {error!("Executor timeout reached");} + if msg_counter < 10 {error!("Executor timeout reached");} break; } } } - res - }; - - let executor_task = tokio::spawn(executor_poll); - - for i in 0..msg_count { - info!("Sending blob {i}..."); - executor_disperse_blob_sender - .send(( - 0, - DaBlob { - column_idx: 0, - column: Column(vec![]), - column_commitment: Default::default(), - aggregated_column_commitment: Default::default(), - aggregated_column_proof: Default::default(), - rows_commitments: vec![], - rows_proofs: vec![], - }, - )) - .unwrap(); } - tokio::time::sleep(Duration::from_secs(1)).await; - - let mut msg_counter = 0; - loop { - match validator_swarms[0].next().await { - Some(SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message })) => { - info!("Validator received blob: {message:?}"); - - // Check data has structure and content as expected - match message.blob { - Some(Blob { blob_id, data }) => { - let deserialized_blob: DaBlob = bincode::deserialize(&data).unwrap(); - assert_eq!(blob_id, deserialized_blob.id()); - msg_counter += 1; + async fn run_validator_swarm( + mut swarm: Swarm + Sized + 'static>> + ) { + let mut msg_counter = 0; + loop { + match swarm.next().await { + Some(SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message })) => { + info!("Validator received blob: {message:?}"); + + // Check data has structure and content as expected + match message.blob { + Some(Blob { blob_id, data }) => { + let deserialized_blob: DaBlob = bincode::deserialize(&data).unwrap(); + assert_eq!(blob_id, deserialized_blob.id()); + msg_counter += 1; + } + None => {} } - None => {} - } - match message.subnetwork_id { - subnet_id => { - assert_eq!(subnet_id, 0); + match message.subnetwork_id { + subnet_id => { + assert_eq!(subnet_id, 0); + } } } + Some(event) => { + info!("Validator received event: {event:?}"); + } + + _ => {} } - Some(event) => { - info!("Validator received event: {event:?}"); + if msg_counter == 10 { + tokio::time::sleep(Duration::from_secs(2)).await; + break; } - - _ => {} } - if msg_counter == msg_count { - tokio::time::sleep(Duration::from_secs(2)).await; - break; + + } + + async fn send_dispersal_messages( + disperse_blob_sender: UnboundedSender<(u32, DaBlob)> + ) { + for i in 0..10 { + info!("Sending blob {i}..."); + disperse_blob_sender + .send(( + 0, + DaBlob { + column_idx: 0, + column: Column(vec![]), + column_commitment: Default::default(), + aggregated_column_commitment: Default::default(), + aggregated_column_proof: Default::default(), + rows_commitments: vec![], + rows_proofs: vec![], + }, + )) + .unwrap(); } } - executor_task.await.unwrap(); + let mut executor_tasks = vec![]; + + // Spawn executors + for i in (0..all_instances / 4).rev() { + let swarm = executor_0_swarms.remove(i); + let executor_poll = async { + run_executor_swarm(swarm); + }; + + let executor_task = tokio::spawn(executor_poll); + executor_tasks.push(executor_task); + + let swarm = executor_1_swarms.remove(i); + let executor_poll = async { + run_executor_swarm(swarm); + }; + + let executor_task = tokio::spawn(executor_poll); + executor_tasks.push(executor_task); + } + + // Send messages + for i in (0..all_instances / 2).rev() { + let sender = message_senders.remove(i); + let send_messages_task = async { + send_dispersal_messages(sender); + }; + tokio::spawn(send_messages_task); + } + + // Spawn validators + for i in (0..all_instances / 4).rev() { + let swarm = validator_0_swarms.remove(i); + let validator_poll = async { + run_validator_swarm(swarm); + }; + tokio::spawn(validator_poll); + + let swarm = validator_1_swarms.remove(i); + let validator_poll = async { + run_validator_swarm(swarm); + }; + tokio::spawn(validator_poll); + } + + + + + join_all(executor_tasks).await; } } From 5be2adec5c2dc7cf0f343e6ea0d1589af80bb1f2 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 28 Oct 2024 11:01:01 +0800 Subject: [PATCH 17/52] fix: add forgotten await and cleanup --- .../dispersal/validator/behaviour.rs | 67 +++++++++---------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 0fa9cc787..c519e112e 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -150,11 +150,10 @@ impl + 'static> Netw mod tests { use super::*; use crate::address_book::AddressBook; - use crate::behaviour::executor::ExecutorBehaviour; use crate::protocols::dispersal::executor::behaviour::{ DispersalExecutorBehaviour, DispersalExecutorEvent, }; - use crate::protocols::sampling::behaviour::SamplingBehaviour; + use futures::future::join_all; use futures::task::ArcWake; use kzgrs_backend::common::blob::DaBlob; use kzgrs_backend::common::Column; @@ -163,10 +162,8 @@ mod tests { use libp2p::{identity, quic, PeerId, Swarm}; use nomos_da_messages::common::Blob; use std::collections::{HashMap, HashSet}; - use std::mem; use std::sync::Arc; use std::time::Duration; - use futures::future::join_all; use tokio::sync::mpsc::UnboundedSender; use tokio::time; use tracing::{error, info}; @@ -229,8 +226,6 @@ mod tests { ) -> libp2p::Swarm< DispersalExecutorBehaviour>, > { - info!("Creating executor_swarm with peerID {}", peer_id); - libp2p::SwarmBuilder::with_existing_identity(key) .with_tokio() .with_other_transport(|keypair| quic::tokio::Transport::new(quic::Config::new(keypair))) @@ -405,10 +400,10 @@ mod tests { let mut validator_1_swarms: Vec<_> = vec![]; for i in 0..all_instances / 4 { - let (k, executor_peer, addr) = executor_0_config[i].clone(); - let (k2, _, addr2) = validator_0_config[i].clone(); - let (k3, executor_peer2, addr3) = executor_1_config[i].clone(); - let (k4, _, addr4) = validator_1_config[i].clone(); + let (k, executor_peer, _) = executor_0_config[i].clone(); + let (k2, _, _) = validator_0_config[i].clone(); + let (k3, executor_peer2, _) = executor_1_config[i].clone(); + let (k4, _, _) = validator_1_config[i].clone(); let executor_0 = executor_swarm( validator_addressbook.clone(), @@ -433,17 +428,16 @@ mod tests { // Let validator swarms to listen for i in 0..all_instances / 4 { - let (_, peer_id, addr) = validator_0_config[i].clone(); - let validator_0_addr = addr.clone().with_p2p(peer_id).unwrap(); - validator_0_swarms[i].listen_on(validator_0_addr).unwrap(); + let (_, _, mut addr) = validator_0_config[i].clone(); + validator_0_swarms[i].listen_on(addr).unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; - let (_, peer_id, addr) = validator_1_config[i].clone(); - let validator_1_addr = addr.clone().with_p2p(peer_id).unwrap(); - validator_1_swarms[i].listen_on(validator_1_addr).unwrap(); + (_, _, addr) = validator_1_config[i].clone(); + validator_1_swarms[i].listen_on(addr).unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; } - tokio::time::sleep(Duration::from_secs(5)).await; - + // Collect blob message senders from executors let mut message_senders: Vec> = Vec::new(); for i in 0..all_instances / 4 { @@ -453,14 +447,18 @@ mod tests { } async fn run_executor_swarm( - mut swarm: Swarm + Sized + 'static>> + mut swarm: Swarm< + DispersalExecutorBehaviour< + impl MembershipHandler + Sized + 'static, + >, + >, ) { let mut msg_counter = 0; loop { tokio::select! { Some(event) = swarm.next() => { info!("Executor event: {event:?}"); - if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess{blob_id, ..}) = event { + if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess{..}) = event { msg_counter += 1; } } @@ -474,7 +472,11 @@ mod tests { } async fn run_validator_swarm( - mut swarm: Swarm + Sized + 'static>> + mut swarm: Swarm< + DispersalValidatorBehaviour< + impl MembershipHandler + Sized + 'static, + >, + >, ) { let mut msg_counter = 0; loop { @@ -485,7 +487,8 @@ mod tests { // Check data has structure and content as expected match message.blob { Some(Blob { blob_id, data }) => { - let deserialized_blob: DaBlob = bincode::deserialize(&data).unwrap(); + let deserialized_blob: DaBlob = + bincode::deserialize(&data).unwrap(); assert_eq!(blob_id, deserialized_blob.id()); msg_counter += 1; } @@ -508,12 +511,9 @@ mod tests { break; } } - } - async fn send_dispersal_messages( - disperse_blob_sender: UnboundedSender<(u32, DaBlob)> - ) { + async fn send_dispersal_messages(disperse_blob_sender: UnboundedSender<(u32, DaBlob)>) { for i in 0..10 { info!("Sending blob {i}..."); disperse_blob_sender @@ -539,7 +539,7 @@ mod tests { for i in (0..all_instances / 4).rev() { let swarm = executor_0_swarms.remove(i); let executor_poll = async { - run_executor_swarm(swarm); + run_executor_swarm(swarm).await; }; let executor_task = tokio::spawn(executor_poll); @@ -547,18 +547,18 @@ mod tests { let swarm = executor_1_swarms.remove(i); let executor_poll = async { - run_executor_swarm(swarm); + run_executor_swarm(swarm).await; }; let executor_task = tokio::spawn(executor_poll); executor_tasks.push(executor_task); } - // Send messages + // Send messages in parallel from all executors for i in (0..all_instances / 2).rev() { let sender = message_senders.remove(i); let send_messages_task = async { - send_dispersal_messages(sender); + send_dispersal_messages(sender).await; }; tokio::spawn(send_messages_task); } @@ -567,20 +567,17 @@ mod tests { for i in (0..all_instances / 4).rev() { let swarm = validator_0_swarms.remove(i); let validator_poll = async { - run_validator_swarm(swarm); + run_validator_swarm(swarm).await; }; tokio::spawn(validator_poll); let swarm = validator_1_swarms.remove(i); let validator_poll = async { - run_validator_swarm(swarm); + run_validator_swarm(swarm).await; }; tokio::spawn(validator_poll); } - - - join_all(executor_tasks).await; } } From 47ed7641cb07e1226729a860a3f782990e7ec35c Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 28 Oct 2024 15:18:59 +0800 Subject: [PATCH 18/52] fix: add message counting to validator tasks --- .../dispersal/validator/behaviour.rs | 104 +++++++++--------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index c519e112e..e70456896 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -153,7 +153,6 @@ mod tests { use crate::protocols::dispersal::executor::behaviour::{ DispersalExecutorBehaviour, DispersalExecutorEvent, }; - use futures::future::join_all; use futures::task::ArcWake; use kzgrs_backend::common::blob::DaBlob; use kzgrs_backend::common::Column; @@ -343,13 +342,14 @@ mod tests { .with_writer(TestWriter::default()) .try_init(); - let all_instances = 20; + const ALL_INSTANCES: usize = 20; + const MESSAGES_TO_SEND: usize = 10; - let executor_0_config = prepare_swarm_config(all_instances / 4, 0); - let validator_0_config = prepare_swarm_config(all_instances / 4, 1); + let executor_0_config = prepare_swarm_config(ALL_INSTANCES / 4, 0); + let validator_0_config = prepare_swarm_config(ALL_INSTANCES / 4, 1); - let executor_1_config = prepare_swarm_config(all_instances / 4, 2); - let validator_1_config = prepare_swarm_config(all_instances / 4, 3); + let executor_1_config = prepare_swarm_config(ALL_INSTANCES / 4, 2); + let validator_1_config = prepare_swarm_config(ALL_INSTANCES / 4, 3); let subnet_0_ids = executor_0_config .iter() @@ -385,8 +385,8 @@ mod tests { .chain(validator_1_config.iter().map(to_p2p_address)), ); - let subnet_0_membership = create_membership(all_instances / 2, 0, &subnet_0_ids); - let subnet_1_membership = create_membership(all_instances / 2, 0, &subnet_1_ids); + let subnet_0_membership = create_membership(ALL_INSTANCES / 2, 0, &subnet_0_ids); + let subnet_1_membership = create_membership(ALL_INSTANCES / 2, 0, &subnet_1_ids); let mut all_neighbours = subnet_0_membership; all_neighbours @@ -399,7 +399,7 @@ mod tests { let mut executor_1_swarms: Vec<_> = vec![]; let mut validator_1_swarms: Vec<_> = vec![]; - for i in 0..all_instances / 4 { + for i in 0..ALL_INSTANCES / 4 { let (k, executor_peer, _) = executor_0_config[i].clone(); let (k2, _, _) = validator_0_config[i].clone(); let (k3, executor_peer2, _) = executor_1_config[i].clone(); @@ -427,20 +427,18 @@ mod tests { } // Let validator swarms to listen - for i in 0..all_instances / 4 { + for i in 0..ALL_INSTANCES / 4 { let (_, _, mut addr) = validator_0_config[i].clone(); validator_0_swarms[i].listen_on(addr).unwrap(); - tokio::time::sleep(Duration::from_secs(1)).await; (_, _, addr) = validator_1_config[i].clone(); validator_1_swarms[i].listen_on(addr).unwrap(); - tokio::time::sleep(Duration::from_secs(1)).await; } // Collect blob message senders from executors let mut message_senders: Vec> = Vec::new(); - for i in 0..all_instances / 4 { + for i in 0..ALL_INSTANCES / 4 { let blob_sender_0 = executor_0_swarms[i].behaviour().blobs_sender(); let blob_sender_1 = executor_1_swarms[i].behaviour().blobs_sender(); message_senders.extend(vec![blob_sender_0, blob_sender_1]); @@ -452,7 +450,7 @@ mod tests { impl MembershipHandler + Sized + 'static, >, >, - ) { + ) -> usize { let mut msg_counter = 0; loop { tokio::select! { @@ -464,11 +462,12 @@ mod tests { } _ = time::sleep(Duration::from_secs(2)) => { - if msg_counter < 10 {error!("Executor timeout reached");} + if msg_counter < MESSAGES_TO_SEND {error!("Executor timeout reached");} break; } } } + msg_counter } async fn run_validator_swarm( @@ -477,8 +476,8 @@ mod tests { impl MembershipHandler + Sized + 'static, >, >, - ) { - let mut msg_counter = 0; + ) -> (usize, usize) { + let (mut msg_0_counter, mut msg_1_counter) = (0, 0); loop { match swarm.next().await { Some(SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message })) => { @@ -490,15 +489,14 @@ mod tests { let deserialized_blob: DaBlob = bincode::deserialize(&data).unwrap(); assert_eq!(blob_id, deserialized_blob.id()); - msg_counter += 1; + if message.subnetwork_id == 0 { + msg_0_counter += 1; + } else { + msg_1_counter += 1; + } } None => {} } - match message.subnetwork_id { - subnet_id => { - assert_eq!(subnet_id, 0); - } - } } Some(event) => { info!("Validator received event: {event:?}"); @@ -506,19 +504,22 @@ mod tests { _ => {} } - if msg_counter == 10 { - tokio::time::sleep(Duration::from_secs(2)).await; - break; + if msg_0_counter == MESSAGES_TO_SEND || msg_1_counter == MESSAGES_TO_SEND { + tokio::time::sleep(Duration::from_secs(5)).await; + return (msg_0_counter, msg_1_counter) } } } - async fn send_dispersal_messages(disperse_blob_sender: UnboundedSender<(u32, DaBlob)>) { - for i in 0..10 { - info!("Sending blob {i}..."); + async fn send_dispersal_messages( + disperse_blob_sender: UnboundedSender<(u32, DaBlob)>, + subnet_id: u32, + ) { + for i in 0..MESSAGES_TO_SEND { + info!("Sending blob {i} to subnet {subnet_id} ..."); disperse_blob_sender .send(( - 0, + subnet_id, DaBlob { column_idx: 0, column: Column(vec![]), @@ -536,48 +537,47 @@ mod tests { let mut executor_tasks = vec![]; // Spawn executors - for i in (0..all_instances / 4).rev() { + for i in (0..ALL_INSTANCES / 4).rev() { let swarm = executor_0_swarms.remove(i); - let executor_poll = async { - run_executor_swarm(swarm).await; - }; - - let executor_task = tokio::spawn(executor_poll); - executor_tasks.push(executor_task); + let executor_0_poll = async { run_executor_swarm(swarm).await }; let swarm = executor_1_swarms.remove(i); - let executor_poll = async { - run_executor_swarm(swarm).await; - }; + let executor_1_poll = async { run_executor_swarm(swarm).await }; - let executor_task = tokio::spawn(executor_poll); - executor_tasks.push(executor_task); + executor_tasks.extend(vec![tokio::spawn(executor_0_poll), tokio::spawn(executor_1_poll)]); } - // Send messages in parallel from all executors - for i in (0..all_instances / 2).rev() { + // Send messages from all executors + for i in (0..ALL_INSTANCES / 2).rev() { let sender = message_senders.remove(i); - let send_messages_task = async { - send_dispersal_messages(sender).await; + let send_messages_task = async move { + send_dispersal_messages(sender, if i % 2 == 0 { 0 } else { 1 }).await; }; tokio::spawn(send_messages_task); } + let mut validator_tasks = vec![]; + // Spawn validators - for i in (0..all_instances / 4).rev() { + for i in (0..ALL_INSTANCES / 4).rev() { let swarm = validator_0_swarms.remove(i); - let validator_poll = async { + let validator_0_poll = async { run_validator_swarm(swarm).await; }; - tokio::spawn(validator_poll); let swarm = validator_1_swarms.remove(i); - let validator_poll = async { + let validator_1_poll = async { run_validator_swarm(swarm).await; }; - tokio::spawn(validator_poll); + + validator_tasks.extend(vec![tokio::spawn(validator_0_poll), tokio::spawn(validator_1_poll)]); } - join_all(executor_tasks).await; + tokio::time::sleep(Duration::from_secs(10)).await; + + for task in executor_tasks { + info!("Executor task received: {:?} message dispersal success", task.await.unwrap()); + + } } } From 8975923e97b4d70250d0c1a1685955492ff53d3d Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 28 Oct 2024 19:02:14 +0800 Subject: [PATCH 19/52] fix: rewrite run_validator_swarm to count messages --- .../dispersal/validator/behaviour.rs | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index e70456896..b07369c8c 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -479,8 +479,9 @@ mod tests { ) -> (usize, usize) { let (mut msg_0_counter, mut msg_1_counter) = (0, 0); loop { - match swarm.next().await { - Some(SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message })) => { + tokio::select! { + Some(event) = swarm.next() => { + if let SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message }) = event { info!("Validator received blob: {message:?}"); // Check data has structure and content as expected @@ -498,17 +499,17 @@ mod tests { None => {} } } - Some(event) => { - info!("Validator received event: {event:?}"); } - _ => {} - } - if msg_0_counter == MESSAGES_TO_SEND || msg_1_counter == MESSAGES_TO_SEND { - tokio::time::sleep(Duration::from_secs(5)).await; - return (msg_0_counter, msg_1_counter) + _ = time::sleep(Duration::from_secs(2)) => { + if msg_0_counter <= MESSAGES_TO_SEND && msg_1_counter <= MESSAGES_TO_SEND { + error!("Validator timeout reached"); + } + break; + } } } + (msg_0_counter, msg_1_counter) } async fn send_dispersal_messages( @@ -544,7 +545,10 @@ mod tests { let swarm = executor_1_swarms.remove(i); let executor_1_poll = async { run_executor_swarm(swarm).await }; - executor_tasks.extend(vec![tokio::spawn(executor_0_poll), tokio::spawn(executor_1_poll)]); + executor_tasks.extend(vec![ + tokio::spawn(executor_0_poll), + tokio::spawn(executor_1_poll), + ]); } // Send messages from all executors @@ -561,23 +565,30 @@ mod tests { // Spawn validators for i in (0..ALL_INSTANCES / 4).rev() { let swarm = validator_0_swarms.remove(i); - let validator_0_poll = async { - run_validator_swarm(swarm).await; - }; + let validator_0_poll = async { run_validator_swarm(swarm).await }; let swarm = validator_1_swarms.remove(i); - let validator_1_poll = async { - run_validator_swarm(swarm).await; - }; + let validator_1_poll = async { run_validator_swarm(swarm).await }; - validator_tasks.extend(vec![tokio::spawn(validator_0_poll), tokio::spawn(validator_1_poll)]); + executor_tasks.extend(vec![ + tokio::spawn(validator_0_poll), + tokio::spawn(validator_1_poll), + ]); } tokio::time::sleep(Duration::from_secs(10)).await; for task in executor_tasks { - info!("Executor task received: {:?} message dispersal success", task.await.unwrap()); + info!( + "Executor task received: {:?} messages dispersal success", + task.await.unwrap() + ); + } + for task in validator_tasks { + let result = task.await.unwrap(); + info!("Validator received {:?} messages from subnet 0", result.0); + info!("Validator received {:?} messages from subnet 1\n", result.1); } } } From a9880f5f7f6c3968709d2a604ee656f105078d65 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 28 Oct 2024 19:04:00 +0800 Subject: [PATCH 20/52] fix: assignment to wrong collection --- .../network/core/src/protocols/dispersal/validator/behaviour.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index b07369c8c..117e21d79 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -570,7 +570,7 @@ mod tests { let swarm = validator_1_swarms.remove(i); let validator_1_poll = async { run_validator_swarm(swarm).await }; - executor_tasks.extend(vec![ + validator_tasks.extend(vec![ tokio::spawn(validator_0_poll), tokio::spawn(validator_1_poll), ]); From dd4525fc5ee3f6caf38d4c597491fb1d6892b431 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 28 Oct 2024 19:18:02 +0800 Subject: [PATCH 21/52] fix: subnet 1 membership --- .../dispersal/validator/behaviour.rs | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 117e21d79..8e372cb65 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -386,14 +386,14 @@ mod tests { ); let subnet_0_membership = create_membership(ALL_INSTANCES / 2, 0, &subnet_0_ids); - let subnet_1_membership = create_membership(ALL_INSTANCES / 2, 0, &subnet_1_ids); + let subnet_1_membership = create_membership(ALL_INSTANCES / 2, 1, &subnet_1_ids); let mut all_neighbours = subnet_0_membership; all_neighbours .membership .extend(subnet_1_membership.membership); - // create swarms + // Create swarms let mut executor_0_swarms: Vec<_> = vec![]; let mut validator_0_swarms: Vec<_> = vec![]; let mut executor_1_swarms: Vec<_> = vec![]; @@ -481,25 +481,25 @@ mod tests { loop { tokio::select! { Some(event) = swarm.next() => { - if let SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message }) = event { - info!("Validator received blob: {message:?}"); - - // Check data has structure and content as expected - match message.blob { - Some(Blob { blob_id, data }) => { - let deserialized_blob: DaBlob = - bincode::deserialize(&data).unwrap(); - assert_eq!(blob_id, deserialized_blob.id()); - if message.subnetwork_id == 0 { - msg_0_counter += 1; - } else { - msg_1_counter += 1; + if let SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message }) = event { + info!("Validator received blob: {message:?}"); + + // Check data has structure and content as expected + match message.blob { + Some(Blob { blob_id, data }) => { + let deserialized_blob: DaBlob = + bincode::deserialize(&data).unwrap(); + assert_eq!(blob_id, deserialized_blob.id()); + if message.subnetwork_id == 0 { + msg_0_counter += 1; + } else { + msg_1_counter += 1; + } } + None => {} } - None => {} } } - } _ = time::sleep(Duration::from_secs(2)) => { if msg_0_counter <= MESSAGES_TO_SEND && msg_1_counter <= MESSAGES_TO_SEND { From eac478d56d85c1ca6dcd86d920c4f4fe836f6544 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 28 Oct 2024 19:47:32 +0800 Subject: [PATCH 22/52] fix: add assertions for counts --- .../dispersal/validator/behaviour.rs | 74 +++++++++++++------ 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 8e372cb65..d963b8d56 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -165,7 +165,7 @@ mod tests { use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tokio::time; - use tracing::{error, info}; + use tracing::{error, warn}; use tracing_subscriber::fmt::TestWriter; use tracing_subscriber::EnvFilter; @@ -344,12 +344,13 @@ mod tests { const ALL_INSTANCES: usize = 20; const MESSAGES_TO_SEND: usize = 10; + const GROUPS: usize = 4; - let executor_0_config = prepare_swarm_config(ALL_INSTANCES / 4, 0); - let validator_0_config = prepare_swarm_config(ALL_INSTANCES / 4, 1); + let executor_0_config = prepare_swarm_config(ALL_INSTANCES / GROUPS, 0); + let validator_0_config = prepare_swarm_config(ALL_INSTANCES / GROUPS, 1); - let executor_1_config = prepare_swarm_config(ALL_INSTANCES / 4, 2); - let validator_1_config = prepare_swarm_config(ALL_INSTANCES / 4, 3); + let executor_1_config = prepare_swarm_config(ALL_INSTANCES / GROUPS, 2); + let validator_1_config = prepare_swarm_config(ALL_INSTANCES / GROUPS, 3); let subnet_0_ids = executor_0_config .iter() @@ -399,7 +400,7 @@ mod tests { let mut executor_1_swarms: Vec<_> = vec![]; let mut validator_1_swarms: Vec<_> = vec![]; - for i in 0..ALL_INSTANCES / 4 { + for i in 0..ALL_INSTANCES / GROUPS { let (k, executor_peer, _) = executor_0_config[i].clone(); let (k2, _, _) = validator_0_config[i].clone(); let (k3, executor_peer2, _) = executor_1_config[i].clone(); @@ -427,7 +428,7 @@ mod tests { } // Let validator swarms to listen - for i in 0..ALL_INSTANCES / 4 { + for i in 0..ALL_INSTANCES / GROUPS { let (_, _, mut addr) = validator_0_config[i].clone(); validator_0_swarms[i].listen_on(addr).unwrap(); @@ -438,7 +439,7 @@ mod tests { // Collect blob message senders from executors let mut message_senders: Vec> = Vec::new(); - for i in 0..ALL_INSTANCES / 4 { + for i in 0..ALL_INSTANCES / GROUPS { let blob_sender_0 = executor_0_swarms[i].behaviour().blobs_sender(); let blob_sender_1 = executor_1_swarms[i].behaviour().blobs_sender(); message_senders.extend(vec![blob_sender_0, blob_sender_1]); @@ -455,14 +456,14 @@ mod tests { loop { tokio::select! { Some(event) = swarm.next() => { - info!("Executor event: {event:?}"); + debug!("Executor event: {event:?}"); if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess{..}) = event { msg_counter += 1; } } - _ = time::sleep(Duration::from_secs(2)) => { - if msg_counter < MESSAGES_TO_SEND {error!("Executor timeout reached");} + _ = time::sleep(Duration::from_secs(3)) => { + if msg_counter < MESSAGES_TO_SEND { error!("Executor timeout reached"); } break; } } @@ -482,7 +483,7 @@ mod tests { tokio::select! { Some(event) = swarm.next() => { if let SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message }) = event { - info!("Validator received blob: {message:?}"); + debug!("Validator received blob: {message:?}"); // Check data has structure and content as expected match message.blob { @@ -503,7 +504,7 @@ mod tests { _ = time::sleep(Duration::from_secs(2)) => { if msg_0_counter <= MESSAGES_TO_SEND && msg_1_counter <= MESSAGES_TO_SEND { - error!("Validator timeout reached"); + warn!("Validator timeout reached"); } break; } @@ -517,7 +518,7 @@ mod tests { subnet_id: u32, ) { for i in 0..MESSAGES_TO_SEND { - info!("Sending blob {i} to subnet {subnet_id} ..."); + debug!("Sending blob {i} to subnet {subnet_id} ..."); disperse_blob_sender .send(( subnet_id, @@ -538,7 +539,7 @@ mod tests { let mut executor_tasks = vec![]; // Spawn executors - for i in (0..ALL_INSTANCES / 4).rev() { + for i in (0..ALL_INSTANCES / GROUPS).rev() { let swarm = executor_0_swarms.remove(i); let executor_0_poll = async { run_executor_swarm(swarm).await }; @@ -563,7 +564,7 @@ mod tests { let mut validator_tasks = vec![]; // Spawn validators - for i in (0..ALL_INSTANCES / 4).rev() { + for i in (0..ALL_INSTANCES / GROUPS).rev() { let swarm = validator_0_swarms.remove(i); let validator_0_poll = async { run_validator_swarm(swarm).await }; @@ -576,19 +577,48 @@ mod tests { ]); } - tokio::time::sleep(Duration::from_secs(10)).await; + let mut dispersal_success_counter = 0usize; + let mut dispersal_request_counter = (0usize, 0usize); for task in executor_tasks { - info!( + let dispersed = task.await.unwrap(); + debug!( "Executor task received: {:?} messages dispersal success", - task.await.unwrap() + dispersed ); + dispersal_success_counter += dispersed; } for task in validator_tasks { - let result = task.await.unwrap(); - info!("Validator received {:?} messages from subnet 0", result.0); - info!("Validator received {:?} messages from subnet 1\n", result.1); + let requested = task.await.unwrap(); + debug!( + "Validator received {:?} messages from subnet 0", + requested.0 + ); + debug!( + "Validator received {:?} messages from subnet 1\n", + requested.1 + ); + dispersal_request_counter = ( + dispersal_request_counter.0 + requested.0, + dispersal_request_counter.1 + requested.1, + ); } + + // Count all dispersed and confirmed messages + assert_eq!( + dispersal_success_counter, + MESSAGES_TO_SEND * (ALL_INSTANCES / GROUPS) * 2 + ); + + // Count all received per subnet + assert_eq!( + dispersal_request_counter.0, + MESSAGES_TO_SEND * (ALL_INSTANCES / GROUPS) + ); + assert_eq!( + dispersal_request_counter.1, + MESSAGES_TO_SEND * (ALL_INSTANCES / GROUPS) + ); } } From c4abaf479d4e93c70346491ce16cf4d6bbd16737 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 28 Oct 2024 20:11:07 +0800 Subject: [PATCH 23/52] fix: move futures to shared function section - remove simple tests --- .../dispersal/validator/behaviour.rs | 226 +++++++----------- 1 file changed, 93 insertions(+), 133 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index d963b8d56..18001623d 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -290,48 +290,98 @@ mod tests { Neighbourhood { membership } } - #[test] - fn test_handle_established_inbound_connection() { - let mut allowed_peers = HashMap::new(); - allowed_peers.insert(PeerId::random(), HashSet::from([0, 1])); - let membership = Neighbourhood { - membership: allowed_peers, - }; - let mut behaviour = DispersalValidatorBehaviour::new(membership); - - let allowed_peer = *behaviour.membership.members().iter().next().unwrap(); - let disallowed_peer = PeerId::random(); - let local_addr = "/ip4/127.0.0.1/tcp/8080".parse().unwrap(); - let remote_addr = "/ip4/127.0.0.1/tcp/8081".parse().unwrap(); - - let result = behaviour.handle_established_inbound_connection( - ConnectionId::new_unchecked(0), - allowed_peer, - &local_addr, - &remote_addr, - ); - assert!(matches!(result, Ok(Either::Left(_)))); + async fn run_executor_swarm( + mut swarm: Swarm< + DispersalExecutorBehaviour< + impl MembershipHandler + Sized + 'static, + >, + >, + messages_to_expect: usize, + ) -> usize { + let mut msg_counter = 0; + loop { + tokio::select! { + Some(event) = swarm.next() => { + debug!("Executor event: {event:?}"); + if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess{..}) = event { + msg_counter += 1; + } + } - let result = behaviour.handle_established_inbound_connection( - ConnectionId::new_unchecked(1), - disallowed_peer, - &local_addr, - &remote_addr, - ); - assert!(matches!(result, Ok(Either::Right(_)))); + _ = time::sleep(Duration::from_secs(3)) => { + if msg_counter < messages_to_expect { error!("Executor timeout reached"); } + break; + } + } + } + msg_counter } - #[test] - fn test_poll() { - let membership = Neighbourhood { - membership: HashMap::new(), - }; - let mut behaviour = DispersalValidatorBehaviour::new(membership); + async fn run_validator_swarm( + mut swarm: Swarm< + DispersalValidatorBehaviour< + impl MembershipHandler + Sized + 'static, + >, + >, + messages_to_expect: usize, + ) -> (usize, usize) { + let (mut msg_0_counter, mut msg_1_counter) = (0, 0); + loop { + tokio::select! { + Some(event) = swarm.next() => { + if let SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message }) = event { + debug!("Validator received blob: {message:?}"); + + // Check data has structure and content as expected + match message.blob { + Some(Blob { blob_id, data }) => { + let deserialized_blob: DaBlob = + bincode::deserialize(&data).unwrap(); + assert_eq!(blob_id, deserialized_blob.id()); + if message.subnetwork_id == 0 { + msg_0_counter += 1; + } else { + msg_1_counter += 1; + } + } + None => {} + } + } + } - let mut cx = std::task::Context::from_waker(futures::task::noop_waker_ref()); - let poll_result = behaviour.poll(&mut cx); + _ = time::sleep(Duration::from_secs(2)) => { + if msg_0_counter < messages_to_expect && msg_1_counter < messages_to_expect { + warn!("Validator timeout reached"); + } + break; + } + } + } + (msg_0_counter, msg_1_counter) + } - assert!(matches!(poll_result, Poll::Pending)); + async fn send_dispersal_messages( + disperse_blob_sender: UnboundedSender<(u32, DaBlob)>, + subnet_id: u32, + messages_to_send: usize, + ) { + for i in 0..messages_to_send { + debug!("Sending blob {i} to subnet {subnet_id} ..."); + disperse_blob_sender + .send(( + subnet_id, + DaBlob { + column_idx: 0, + column: Column(vec![]), + column_commitment: Default::default(), + aggregated_column_commitment: Default::default(), + aggregated_column_proof: Default::default(), + rows_commitments: vec![], + rows_proofs: vec![], + }, + )) + .unwrap(); + } } #[tokio::test] @@ -445,106 +495,15 @@ mod tests { message_senders.extend(vec![blob_sender_0, blob_sender_1]); } - async fn run_executor_swarm( - mut swarm: Swarm< - DispersalExecutorBehaviour< - impl MembershipHandler + Sized + 'static, - >, - >, - ) -> usize { - let mut msg_counter = 0; - loop { - tokio::select! { - Some(event) = swarm.next() => { - debug!("Executor event: {event:?}"); - if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess{..}) = event { - msg_counter += 1; - } - } - - _ = time::sleep(Duration::from_secs(3)) => { - if msg_counter < MESSAGES_TO_SEND { error!("Executor timeout reached"); } - break; - } - } - } - msg_counter - } - - async fn run_validator_swarm( - mut swarm: Swarm< - DispersalValidatorBehaviour< - impl MembershipHandler + Sized + 'static, - >, - >, - ) -> (usize, usize) { - let (mut msg_0_counter, mut msg_1_counter) = (0, 0); - loop { - tokio::select! { - Some(event) = swarm.next() => { - if let SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message }) = event { - debug!("Validator received blob: {message:?}"); - - // Check data has structure and content as expected - match message.blob { - Some(Blob { blob_id, data }) => { - let deserialized_blob: DaBlob = - bincode::deserialize(&data).unwrap(); - assert_eq!(blob_id, deserialized_blob.id()); - if message.subnetwork_id == 0 { - msg_0_counter += 1; - } else { - msg_1_counter += 1; - } - } - None => {} - } - } - } - - _ = time::sleep(Duration::from_secs(2)) => { - if msg_0_counter <= MESSAGES_TO_SEND && msg_1_counter <= MESSAGES_TO_SEND { - warn!("Validator timeout reached"); - } - break; - } - } - } - (msg_0_counter, msg_1_counter) - } - - async fn send_dispersal_messages( - disperse_blob_sender: UnboundedSender<(u32, DaBlob)>, - subnet_id: u32, - ) { - for i in 0..MESSAGES_TO_SEND { - debug!("Sending blob {i} to subnet {subnet_id} ..."); - disperse_blob_sender - .send(( - subnet_id, - DaBlob { - column_idx: 0, - column: Column(vec![]), - column_commitment: Default::default(), - aggregated_column_commitment: Default::default(), - aggregated_column_proof: Default::default(), - rows_commitments: vec![], - rows_proofs: vec![], - }, - )) - .unwrap(); - } - } - let mut executor_tasks = vec![]; // Spawn executors for i in (0..ALL_INSTANCES / GROUPS).rev() { let swarm = executor_0_swarms.remove(i); - let executor_0_poll = async { run_executor_swarm(swarm).await }; + let executor_0_poll = async { run_executor_swarm(swarm, MESSAGES_TO_SEND).await }; let swarm = executor_1_swarms.remove(i); - let executor_1_poll = async { run_executor_swarm(swarm).await }; + let executor_1_poll = async { run_executor_swarm(swarm, MESSAGES_TO_SEND).await }; executor_tasks.extend(vec![ tokio::spawn(executor_0_poll), @@ -556,7 +515,8 @@ mod tests { for i in (0..ALL_INSTANCES / 2).rev() { let sender = message_senders.remove(i); let send_messages_task = async move { - send_dispersal_messages(sender, if i % 2 == 0 { 0 } else { 1 }).await; + send_dispersal_messages(sender, if i % 2 == 0 { 0 } else { 1 }, MESSAGES_TO_SEND) + .await; }; tokio::spawn(send_messages_task); } @@ -566,10 +526,10 @@ mod tests { // Spawn validators for i in (0..ALL_INSTANCES / GROUPS).rev() { let swarm = validator_0_swarms.remove(i); - let validator_0_poll = async { run_validator_swarm(swarm).await }; + let validator_0_poll = async { run_validator_swarm(swarm, MESSAGES_TO_SEND).await }; let swarm = validator_1_swarms.remove(i); - let validator_1_poll = async { run_validator_swarm(swarm).await }; + let validator_1_poll = async { run_validator_swarm(swarm, MESSAGES_TO_SEND).await }; validator_tasks.extend(vec![ tokio::spawn(validator_0_poll), From 4f0231ff260a8bedf69fa8d2ee8a8c7f1211608d Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 28 Oct 2024 20:32:54 +0800 Subject: [PATCH 24/52] fix: extend timeout to accommodate slow runner --- .../network/core/src/protocols/dispersal/validator/behaviour.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 18001623d..ef076516d 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -308,7 +308,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(3)) => { + _ = time::sleep(Duration::from_secs(5)) => { if msg_counter < messages_to_expect { error!("Executor timeout reached"); } break; } From 2c0c00d2a032f0e4a337fae6466d7d7015295238 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 10:10:02 +0800 Subject: [PATCH 25/52] fix: extend timeout more --- .../dispersal/validator/behaviour.rs | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index ef076516d..8189b6560 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -308,7 +308,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(5)) => { + _ = time::sleep(Duration::from_secs(6)) => { if msg_counter < messages_to_expect { error!("Executor timeout reached"); } break; } @@ -404,22 +404,14 @@ mod tests { let subnet_0_ids = executor_0_config .iter() + .chain(validator_0_config.iter()) .map(|(_, peer_id, _)| peer_id.clone()) - .chain( - validator_0_config - .iter() - .map(|(_, peer_id, _)| peer_id.clone()), - ) .collect::>(); let subnet_1_ids = executor_1_config .iter() + .chain(validator_1_config.iter()) .map(|(_, peer_id, _)| peer_id.clone()) - .chain( - validator_1_config - .iter() - .map(|(_, peer_id, _)| peer_id.clone()), - ) .collect::>(); let to_p2p_address = |(_, peer_id, addr): &(_, PeerId, Multiaddr)| { @@ -538,7 +530,6 @@ mod tests { } let mut dispersal_success_counter = 0usize; - let mut dispersal_request_counter = (0usize, 0usize); for task in executor_tasks { let dispersed = task.await.unwrap(); @@ -549,6 +540,8 @@ mod tests { dispersal_success_counter += dispersed; } + let mut dispersal_request_counter = (0usize, 0usize); + for task in validator_tasks { let requested = task.await.unwrap(); debug!( @@ -565,13 +558,13 @@ mod tests { ); } - // Count all dispersed and confirmed messages + // Check dispersed and confirmed equal to sent messages assert_eq!( dispersal_success_counter, MESSAGES_TO_SEND * (ALL_INSTANCES / GROUPS) * 2 ); - // Count all received per subnet + // Check received per subnet are half of all messages sent assert_eq!( dispersal_request_counter.0, MESSAGES_TO_SEND * (ALL_INSTANCES / GROUPS) From 044de28153f49fa3f35aced313bd361a3d2b35fe Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 10:26:37 +0800 Subject: [PATCH 26/52] fix: re-balance timeouts --- .../core/src/protocols/dispersal/validator/behaviour.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 8189b6560..2d0757c4d 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -308,7 +308,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(6)) => { + _ = time::sleep(Duration::from_secs(3)) => { if msg_counter < messages_to_expect { error!("Executor timeout reached"); } break; } @@ -349,7 +349,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(2)) => { + _ = time::sleep(Duration::from_secs(3)) => { if msg_0_counter < messages_to_expect && msg_1_counter < messages_to_expect { warn!("Validator timeout reached"); } From 8c2041cface0f15c87a65b3e4d2726cdd5e0f0b1 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 10:44:57 +0800 Subject: [PATCH 27/52] fix: increase timeouts --- .../core/src/protocols/dispersal/validator/behaviour.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 2d0757c4d..be84eadb0 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -308,7 +308,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(3)) => { + _ = time::sleep(Duration::from_secs(4)) => { if msg_counter < messages_to_expect { error!("Executor timeout reached"); } break; } @@ -349,7 +349,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(3)) => { + _ = time::sleep(Duration::from_secs(4)) => { if msg_0_counter < messages_to_expect && msg_1_counter < messages_to_expect { warn!("Validator timeout reached"); } From fdb9998958917ddadf0c23c7efe1707c84014f84 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 10:59:05 +0800 Subject: [PATCH 28/52] fix: let executor wait for all messages --- .../src/protocols/dispersal/validator/behaviour.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index be84eadb0..d9852d46d 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -165,7 +165,7 @@ mod tests { use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tokio::time; - use tracing::{error, warn}; + use tracing::warn; use tracing_subscriber::fmt::TestWriter; use tracing_subscriber::EnvFilter; @@ -308,11 +308,15 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(4)) => { - if msg_counter < messages_to_expect { error!("Executor timeout reached"); } + _ = time::sleep(Duration::from_secs(2)) => { + if msg_counter < messages_to_expect { + warn!("Executor timeout reached"); + continue; + } break; } } + } msg_counter } From 2c390144683fc32a0484b52e8b7a856343efac40 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 11:05:35 +0800 Subject: [PATCH 29/52] fix: formatting --- .../network/core/src/protocols/dispersal/validator/behaviour.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index d9852d46d..fc152cdd7 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -316,7 +316,6 @@ mod tests { break; } } - } msg_counter } From 7d2e0c8570e31aeb086770f72cf969821b1be807 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 11:09:50 +0800 Subject: [PATCH 30/52] fix: add retry counter for executor --- .../core/src/protocols/dispersal/validator/behaviour.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index fc152cdd7..a02931b7d 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -299,6 +299,7 @@ mod tests { messages_to_expect: usize, ) -> usize { let mut msg_counter = 0; + let mut retry_counter = 0; loop { tokio::select! { Some(event) = swarm.next() => { @@ -309,7 +310,7 @@ mod tests { } _ = time::sleep(Duration::from_secs(2)) => { - if msg_counter < messages_to_expect { + if msg_counter < messages_to_expect && retry_counter < 3 { warn!("Executor timeout reached"); continue; } From 8216868f765b82c7f24c316bcdbe872d79b54549 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 12:37:43 +0800 Subject: [PATCH 31/52] fix: increment retry counter --- .../network/core/src/protocols/dispersal/validator/behaviour.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index a02931b7d..5088e34fa 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -312,6 +312,7 @@ mod tests { _ = time::sleep(Duration::from_secs(2)) => { if msg_counter < messages_to_expect && retry_counter < 3 { warn!("Executor timeout reached"); + retry_counter += 1; continue; } break; From 1fddd4ee81b3c792f1e10d199aca234594e74d78 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 16:29:07 +0800 Subject: [PATCH 32/52] fix: make validator swarm polling more deterministic --- .../core/src/protocols/dispersal/validator/behaviour.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 5088e34fa..151265315 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -331,6 +331,7 @@ mod tests { messages_to_expect: usize, ) -> (usize, usize) { let (mut msg_0_counter, mut msg_1_counter) = (0, 0); + loop { tokio::select! { Some(event) = swarm.next() => { @@ -357,6 +358,9 @@ mod tests { _ = time::sleep(Duration::from_secs(4)) => { if msg_0_counter < messages_to_expect && msg_1_counter < messages_to_expect { warn!("Validator timeout reached"); + if !swarm.behaviour_mut().tasks.is_empty() { + continue; + } } break; } From eae257615a020ab85256605c093676003de279f6 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 16:30:09 +0800 Subject: [PATCH 33/52] fix: add one more retry for executor swarm --- .../network/core/src/protocols/dispersal/validator/behaviour.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 151265315..4edcae175 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -310,7 +310,7 @@ mod tests { } _ = time::sleep(Duration::from_secs(2)) => { - if msg_counter < messages_to_expect && retry_counter < 3 { + if msg_counter < messages_to_expect && retry_counter < 4 { warn!("Executor timeout reached"); retry_counter += 1; continue; From 8b114e6d423322e48d557c41adaeb531ff7c22a6 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 18:16:37 +0800 Subject: [PATCH 34/52] test: increase executor timeout for self-hosted --- .../network/core/src/protocols/dispersal/validator/behaviour.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 4edcae175..c15ee143c 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -309,7 +309,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(2)) => { + _ = time::sleep(Duration::from_secs(3)) => { if msg_counter < messages_to_expect && retry_counter < 4 { warn!("Executor timeout reached"); retry_counter += 1; From 868614eb8a11f4c4366b341a9c942b37f1cb8e24 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 21:38:08 +0800 Subject: [PATCH 35/52] fix: remove select for executor swarm --- .../dispersal/validator/behaviour.rs | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index c15ee143c..cea11f37d 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -299,26 +299,17 @@ mod tests { messages_to_expect: usize, ) -> usize { let mut msg_counter = 0; - let mut retry_counter = 0; - loop { - tokio::select! { - Some(event) = swarm.next() => { - debug!("Executor event: {event:?}"); - if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess{..}) = event { - msg_counter += 1; - } - } + while let Some(event) = swarm.next().await { + debug!("Executor event: {event:?}"); + if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess { .. }) = event { + msg_counter += 1; + } - _ = time::sleep(Duration::from_secs(3)) => { - if msg_counter < messages_to_expect && retry_counter < 4 { - warn!("Executor timeout reached"); - retry_counter += 1; - continue; - } - break; - } + if msg_counter >= messages_to_expect { + break; } } + msg_counter } From 57ef85f1918c023d45f01a106422386ed8468e57 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 21:58:05 +0800 Subject: [PATCH 36/52] fix: implement clippy suggestions --- .../dispersal/validator/behaviour.rs | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index cea11f37d..e0ebcacdc 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -283,8 +283,8 @@ mod tests { ) -> Neighbourhood { let mut membership = HashMap::default(); - for i in 0..num_instances { - membership.insert(peer_ids[i], HashSet::from([subnet_id])); + for peer_id in peer_ids.iter().take(num_instances) { + membership.insert(*peer_id, HashSet::from([subnet_id])); } Neighbourhood { membership } @@ -330,19 +330,16 @@ mod tests { debug!("Validator received blob: {message:?}"); // Check data has structure and content as expected - match message.blob { - Some(Blob { blob_id, data }) => { - let deserialized_blob: DaBlob = - bincode::deserialize(&data).unwrap(); - assert_eq!(blob_id, deserialized_blob.id()); - if message.subnetwork_id == 0 { - msg_0_counter += 1; - } else { - msg_1_counter += 1; - } - } - None => {} - } + if let Some(Blob { blob_id, data }) = message.blob { + let deserialized_blob: DaBlob = + bincode::deserialize(&data).unwrap(); + assert_eq!(blob_id, deserialized_blob.id()); + if message.subnetwork_id == 0 { + msg_0_counter += 1; + } else { + msg_1_counter += 1; + } + } } } @@ -405,20 +402,17 @@ mod tests { let subnet_0_ids = executor_0_config .iter() .chain(validator_0_config.iter()) - .map(|(_, peer_id, _)| peer_id.clone()) + .map(|(_, peer_id, _)| *peer_id) .collect::>(); let subnet_1_ids = executor_1_config .iter() .chain(validator_1_config.iter()) - .map(|(_, peer_id, _)| peer_id.clone()) + .map(|(_, peer_id, _)| *peer_id) .collect::>(); let to_p2p_address = |(_, peer_id, addr): &(_, PeerId, Multiaddr)| { - ( - peer_id.clone(), - addr.clone().with_p2p(peer_id.clone()).unwrap(), - ) + (*peer_id, addr.clone().with_p2p(*peer_id).unwrap()) }; let validator_addressbook = AddressBook::from_iter( From 336ee493e680edf6e0278f4b6282ef74a99e4357 Mon Sep 17 00:00:00 2001 From: Roman Date: Tue, 29 Oct 2024 22:20:44 +0800 Subject: [PATCH 37/52] fix: remove unnecessary check for validator swarm --- .../src/protocols/dispersal/validator/behaviour.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index e0ebcacdc..1b629f35b 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -319,7 +319,6 @@ mod tests { impl MembershipHandler + Sized + 'static, >, >, - messages_to_expect: usize, ) -> (usize, usize) { let (mut msg_0_counter, mut msg_1_counter) = (0, 0); @@ -344,11 +343,9 @@ mod tests { } _ = time::sleep(Duration::from_secs(4)) => { - if msg_0_counter < messages_to_expect && msg_1_counter < messages_to_expect { - warn!("Validator timeout reached"); - if !swarm.behaviour_mut().tasks.is_empty() { - continue; - } + warn!("Validator timeout reached"); + if !swarm.behaviour_mut().tasks.is_empty() { + continue; } break; } @@ -512,10 +509,10 @@ mod tests { // Spawn validators for i in (0..ALL_INSTANCES / GROUPS).rev() { let swarm = validator_0_swarms.remove(i); - let validator_0_poll = async { run_validator_swarm(swarm, MESSAGES_TO_SEND).await }; + let validator_0_poll = async { run_validator_swarm(swarm).await }; let swarm = validator_1_swarms.remove(i); - let validator_1_poll = async { run_validator_swarm(swarm, MESSAGES_TO_SEND).await }; + let validator_1_poll = async { run_validator_swarm(swarm).await }; validator_tasks.extend(vec![ tokio::spawn(validator_0_poll), From 504968973cf5f63b4e5d3b6112d1e3023f78918d Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 30 Oct 2024 14:26:32 +0800 Subject: [PATCH 38/52] fix: use watch channel to terminate executors - bump port numbers for quick to workaround stuck processes --- .../dispersal/validator/behaviour.rs | 68 ++++++++++++------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 1b629f35b..69d583f6e 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -164,6 +164,7 @@ mod tests { use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; + use tokio::sync::watch; use tokio::time; use tracing::warn; use tracing_subscriber::fmt::TestWriter; @@ -266,7 +267,7 @@ mod tests { for i in 0..num_instances { let keypair = identity::Keypair::generate_ed25519(); let peer_id = PeerId::from(keypair.public()); - let port = 5100 + group_id * num_instances + i; + let port = 5200 + group_id * num_instances + i; let addr: Multiaddr = format!("/ip4/127.0.0.1/udp/{port}/quic-v1") .parse() .unwrap(); @@ -297,19 +298,29 @@ mod tests { >, >, messages_to_expect: usize, + mut terminator_rx: watch::Receiver, ) -> usize { let mut msg_counter = 0; - while let Some(event) = swarm.next().await { - debug!("Executor event: {event:?}"); - if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess { .. }) = event { - msg_counter += 1; - } + loop { + tokio::select! { + Some(event) = swarm.next() => { + debug!("Executor event: {event:?}"); + if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess{..}) = event { + msg_counter += 1; + } + if msg_counter >= messages_to_expect { + break; + } + } - if msg_counter >= messages_to_expect { - break; + _ = terminator_rx.changed() => { + if *terminator_rx.borrow() { + warn!("Executor terminated"); + break; + } + } } } - msg_counter } @@ -342,11 +353,8 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(4)) => { + _ = time::sleep(Duration::from_secs(2)) => { warn!("Validator timeout reached"); - if !swarm.behaviour_mut().tasks.is_empty() { - continue; - } break; } } @@ -479,14 +487,19 @@ mod tests { } let mut executor_tasks = vec![]; + let (terminator_tx, terminator_rx) = watch::channel::(false); // Spawn executors for i in (0..ALL_INSTANCES / GROUPS).rev() { + let (terminator_0, terminator_1) = (terminator_rx.clone(), terminator_rx.clone()); + let swarm = executor_0_swarms.remove(i); - let executor_0_poll = async { run_executor_swarm(swarm, MESSAGES_TO_SEND).await }; + let executor_0_poll = + async move { run_executor_swarm(swarm, MESSAGES_TO_SEND, terminator_0).await }; let swarm = executor_1_swarms.remove(i); - let executor_1_poll = async { run_executor_swarm(swarm, MESSAGES_TO_SEND).await }; + let executor_1_poll = + async move { run_executor_swarm(swarm, MESSAGES_TO_SEND, terminator_1).await }; executor_tasks.extend(vec![ tokio::spawn(executor_0_poll), @@ -520,17 +533,6 @@ mod tests { ]); } - let mut dispersal_success_counter = 0usize; - - for task in executor_tasks { - let dispersed = task.await.unwrap(); - debug!( - "Executor task received: {:?} messages dispersal success", - dispersed - ); - dispersal_success_counter += dispersed; - } - let mut dispersal_request_counter = (0usize, 0usize); for task in validator_tasks { @@ -549,6 +551,20 @@ mod tests { ); } + // Terminate any remaining executors + terminator_tx.send(true).unwrap(); + + let mut dispersal_success_counter = 0usize; + + for task in executor_tasks { + let dispersed = task.await.unwrap(); + debug!( + "Executor task received: {:?} messages dispersal success", + dispersed + ); + dispersal_success_counter += dispersed; + } + // Check dispersed and confirmed equal to sent messages assert_eq!( dispersal_success_counter, From 30f38d462ca318da651a2a7737d46a0365a8ab4f Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 30 Oct 2024 14:35:46 +0800 Subject: [PATCH 39/52] fix: remove TestWaker --- .../core/src/protocols/dispersal/validator/behaviour.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 69d583f6e..2178d82b1 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -212,12 +212,6 @@ mod tests { } } - struct TestWaker; - - impl ArcWake for TestWaker { - fn wake_by_ref(_arc_self: &Arc) {} - } - pub fn executor_swarm( addressbook: AddressBook, key: Keypair, From 560456349c56fd88b86a7a1a5884ffbcec4a6d00 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 30 Oct 2024 15:14:07 +0800 Subject: [PATCH 40/52] fix: simplify is_allowed --- .../core/src/protocols/dispersal/validator/behaviour.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 2178d82b1..3b0b290bf 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -197,10 +197,7 @@ mod tests { } fn is_allowed(&self, _id: &Self::Id) -> bool { - if self.membership.contains_key(_id) { - return true; - } - false + self.membership.contains_key(_id) } fn members(&self) -> HashSet { From e7f90a6bc64c6f0edb1a6fc250df98394ee19364 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 30 Oct 2024 20:18:33 +0800 Subject: [PATCH 41/52] fix: changes for core review 1 --- .../dispersal/validator/behaviour.rs | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 3b0b290bf..701db780f 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -153,7 +153,6 @@ mod tests { use crate::protocols::dispersal::executor::behaviour::{ DispersalExecutorBehaviour, DispersalExecutorEvent, }; - use futures::task::ArcWake; use kzgrs_backend::common::blob::DaBlob; use kzgrs_backend::common::Column; use libp2p::identity::Keypair; @@ -161,7 +160,6 @@ mod tests { use libp2p::{identity, quic, PeerId, Swarm}; use nomos_da_messages::common::Blob; use std::collections::{HashMap, HashSet}; - use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tokio::sync::watch; @@ -209,7 +207,7 @@ mod tests { } } - pub fn executor_swarm( + fn executor_swarm( addressbook: AddressBook, key: Keypair, peer_id: PeerId, @@ -231,7 +229,7 @@ mod tests { .build() } - pub fn validator_swarm( + fn validator_swarm( key: Keypair, membership: impl MembershipHandler + 'static, ) -> libp2p::Swarm< @@ -289,26 +287,22 @@ mod tests { >, >, messages_to_expect: usize, - mut terminator_rx: watch::Receiver, + mut terminator_rx: watch::Receiver<()>, ) -> usize { let mut msg_counter = 0; - loop { + + while msg_counter < messages_to_expect { tokio::select! { - Some(event) = swarm.next() => { + event = swarm.select_next_some() => { debug!("Executor event: {event:?}"); if let SwarmEvent::Behaviour(DispersalExecutorEvent::DispersalSuccess{..}) = event { msg_counter += 1; } - if msg_counter >= messages_to_expect { - break; - } } _ = terminator_rx.changed() => { - if *terminator_rx.borrow() { - warn!("Executor terminated"); - break; - } + warn!("Executor terminated"); + break; } } } @@ -326,7 +320,7 @@ mod tests { loop { tokio::select! { - Some(event) = swarm.next() => { + event = swarm.select_next_some() => { if let SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message }) = event { debug!("Validator received blob: {message:?}"); @@ -478,7 +472,7 @@ mod tests { } let mut executor_tasks = vec![]; - let (terminator_tx, terminator_rx) = watch::channel::(false); + let (terminator_tx, terminator_rx) = watch::channel::<()>(()); // Spawn executors for i in (0..ALL_INSTANCES / GROUPS).rev() { @@ -543,7 +537,7 @@ mod tests { } // Terminate any remaining executors - terminator_tx.send(true).unwrap(); + terminator_tx.send(()).unwrap(); let mut dispersal_success_counter = 0usize; From 8397a1f043ce64968363950c087cb593d12e1f31 Mon Sep 17 00:00:00 2001 From: Roman Date: Wed, 30 Oct 2024 21:36:33 +0800 Subject: [PATCH 42/52] fix: create single membership for all swarms --- .../dispersal/validator/behaviour.rs | 40 ++++++++----------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 701db780f..eefcc0893 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -268,14 +268,18 @@ mod tests { fn create_membership( num_instances: usize, - subnet_id: u32, + subnets: usize, peer_ids: &[PeerId], ) -> Neighbourhood { - let mut membership = HashMap::default(); - - for peer_id in peer_ids.iter().take(num_instances) { - membership.insert(*peer_id, HashSet::from([subnet_id])); - } + let membership = peer_ids + .iter() + .take(num_instances) + .enumerate() + .map(|(i, &peer_id)| { + let subnet_id = (i * subnets / num_instances) as SubnetworkId; + (peer_id, HashSet::from([subnet_id])) + }) + .collect(); Neighbourhood { membership } } @@ -389,14 +393,10 @@ mod tests { let executor_1_config = prepare_swarm_config(ALL_INSTANCES / GROUPS, 2); let validator_1_config = prepare_swarm_config(ALL_INSTANCES / GROUPS, 3); - let subnet_0_ids = executor_0_config + let all_ids = executor_0_config .iter() .chain(validator_0_config.iter()) - .map(|(_, peer_id, _)| *peer_id) - .collect::>(); - - let subnet_1_ids = executor_1_config - .iter() + .chain(executor_1_config.iter()) .chain(validator_1_config.iter()) .map(|(_, peer_id, _)| *peer_id) .collect::>(); @@ -412,13 +412,7 @@ mod tests { .chain(validator_1_config.iter().map(to_p2p_address)), ); - let subnet_0_membership = create_membership(ALL_INSTANCES / 2, 0, &subnet_0_ids); - let subnet_1_membership = create_membership(ALL_INSTANCES / 2, 1, &subnet_1_ids); - - let mut all_neighbours = subnet_0_membership; - all_neighbours - .membership - .extend(subnet_1_membership.membership); + let neighbours = create_membership(ALL_INSTANCES, 2, &all_ids); // Create swarms let mut executor_0_swarms: Vec<_> = vec![]; @@ -436,9 +430,9 @@ mod tests { validator_addressbook.clone(), k, executor_peer, - all_neighbours.clone(), + neighbours.clone(), ); - let validator_0 = validator_swarm(k2, all_neighbours.clone()); + let validator_0 = validator_swarm(k2, neighbours.clone()); executor_0_swarms.push(executor_0); validator_0_swarms.push(validator_0); @@ -446,9 +440,9 @@ mod tests { validator_addressbook.clone(), k3, executor_peer2, - all_neighbours.clone(), + neighbours.clone(), ); - let validator_1 = validator_swarm(k4, all_neighbours.clone()); + let validator_1 = validator_swarm(k4, neighbours.clone()); executor_1_swarms.push(executor_1); validator_1_swarms.push(validator_1); } From 1121ffabdafe1058c62acfb91529ad18601b742c Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 31 Oct 2024 10:32:30 +0800 Subject: [PATCH 43/52] test: add executor swarms to addressbook --- .../dispersal/validator/behaviour.rs | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index eefcc0893..db2db85d4 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -405,11 +405,13 @@ mod tests { (*peer_id, addr.clone().with_p2p(*peer_id).unwrap()) }; - let validator_addressbook = AddressBook::from_iter( - validator_0_config + let addressbook = AddressBook::from_iter( + executor_0_config .iter() - .map(to_p2p_address) - .chain(validator_1_config.iter().map(to_p2p_address)), + .chain(validator_0_config.iter()) + .chain(executor_1_config.iter()) + .chain(validator_1_config.iter()) + .map(to_p2p_address), ); let neighbours = create_membership(ALL_INSTANCES, 2, &all_ids); @@ -426,22 +428,14 @@ mod tests { let (k3, executor_peer2, _) = executor_1_config[i].clone(); let (k4, _, _) = validator_1_config[i].clone(); - let executor_0 = executor_swarm( - validator_addressbook.clone(), - k, - executor_peer, - neighbours.clone(), - ); + let executor_0 = + executor_swarm(addressbook.clone(), k, executor_peer, neighbours.clone()); let validator_0 = validator_swarm(k2, neighbours.clone()); executor_0_swarms.push(executor_0); validator_0_swarms.push(validator_0); - let executor_1 = executor_swarm( - validator_addressbook.clone(), - k3, - executor_peer2, - neighbours.clone(), - ); + let executor_1 = + executor_swarm(addressbook.clone(), k3, executor_peer2, neighbours.clone()); let validator_1 = validator_swarm(k4, neighbours.clone()); executor_1_swarms.push(executor_1); validator_1_swarms.push(validator_1); From a546af48695f5b87fc82818486f4ebb3ba9a59f8 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 31 Oct 2024 16:58:48 +0800 Subject: [PATCH 44/52] test: only executors in addressbook --- .../network/core/src/protocols/dispersal/validator/behaviour.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index db2db85d4..895da2529 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -408,9 +408,7 @@ mod tests { let addressbook = AddressBook::from_iter( executor_0_config .iter() - .chain(validator_0_config.iter()) .chain(executor_1_config.iter()) - .chain(validator_1_config.iter()) .map(to_p2p_address), ); From d4a20fbe3075955bedf92757602df2559543b160 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 31 Oct 2024 17:11:52 +0800 Subject: [PATCH 45/52] fix: revert back to validators only in addressbook - add comment --- .../core/src/protocols/dispersal/validator/behaviour.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 895da2529..a5a8e5d86 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -405,10 +405,12 @@ mod tests { (*peer_id, addr.clone().with_p2p(*peer_id).unwrap()) }; + // Including only validators in the addressbook is for testing purposes only. + // In real-world scenarios, the addressbook should include all types of swarms. let addressbook = AddressBook::from_iter( - executor_0_config + validator_0_config .iter() - .chain(executor_1_config.iter()) + .chain(validator_1_config.iter()) .map(to_p2p_address), ); From 46f3300fe7f2af8bfdf00324c7ad90073961e81b Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 31 Oct 2024 20:57:29 +0800 Subject: [PATCH 46/52] fix: add time for validator swarm --- .../network/core/src/protocols/dispersal/validator/behaviour.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index a5a8e5d86..736ec53a7 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -342,7 +342,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(2)) => { + _ = time::sleep(Duration::from_secs(4)) => { warn!("Validator timeout reached"); break; } From a93d082056ae3515b1a35bcd26aa83060103f327 Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 31 Oct 2024 21:14:22 +0800 Subject: [PATCH 47/52] test: double the timeout value --- .../network/core/src/protocols/dispersal/validator/behaviour.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 736ec53a7..480ea5fe1 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -342,7 +342,7 @@ mod tests { } } - _ = time::sleep(Duration::from_secs(4)) => { + _ = time::sleep(Duration::from_secs(8)) => { warn!("Validator timeout reached"); break; } From ca85c7dfbac15fa358d46a0d21a4adf6e6c66bdd Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 31 Oct 2024 21:50:06 +0800 Subject: [PATCH 48/52] test: built-in timeout instead of select --- .../dispersal/validator/behaviour.rs | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 480ea5fe1..82aae9287 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -163,7 +163,7 @@ mod tests { use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tokio::sync::watch; - use tokio::time; + use tokio::time::timeout; use tracing::warn; use tracing_subscriber::fmt::TestWriter; use tracing_subscriber::EnvFilter; @@ -321,33 +321,37 @@ mod tests { >, ) -> (usize, usize) { let (mut msg_0_counter, mut msg_1_counter) = (0, 0); + let timeout_duration = Duration::from_secs(8); loop { - tokio::select! { - event = swarm.select_next_some() => { - if let SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message }) = event { + let event = timeout(timeout_duration, swarm.select_next_some()).await; + + match event { + Ok(event) => { + if let SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message }) = + event + { debug!("Validator received blob: {message:?}"); // Check data has structure and content as expected if let Some(Blob { blob_id, data }) = message.blob { - let deserialized_blob: DaBlob = - bincode::deserialize(&data).unwrap(); - assert_eq!(blob_id, deserialized_blob.id()); - if message.subnetwork_id == 0 { - msg_0_counter += 1; - } else { - msg_1_counter += 1; - } - } + let deserialized_blob: DaBlob = bincode::deserialize(&data).unwrap(); + assert_eq!(blob_id, deserialized_blob.id()); + if message.subnetwork_id == 0 { + msg_0_counter += 1; + } else { + msg_1_counter += 1; + } + } } } - - _ = time::sleep(Duration::from_secs(8)) => { + Err(_) => { warn!("Validator timeout reached"); break; } } } + (msg_0_counter, msg_1_counter) } From dedf5d89fe34c07211785248a76570aaad7554bb Mon Sep 17 00:00:00 2001 From: Roman Date: Thu, 31 Oct 2024 22:01:45 +0800 Subject: [PATCH 49/52] test: give 15 seconds for validator before exit --- .../network/core/src/protocols/dispersal/validator/behaviour.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 82aae9287..368110c84 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -321,7 +321,7 @@ mod tests { >, ) -> (usize, usize) { let (mut msg_0_counter, mut msg_1_counter) = (0, 0); - let timeout_duration = Duration::from_secs(8); + let timeout_duration = Duration::from_secs(15); loop { let event = timeout(timeout_duration, swarm.select_next_some()).await; From ba703b8cef49b2305a8b853c4944a9ea1f4fa8d0 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 4 Nov 2024 13:50:41 +0800 Subject: [PATCH 50/52] test: disperse random data --- .../dispersal/validator/behaviour.rs | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 368110c84..99d27ffe6 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -154,11 +154,14 @@ mod tests { DispersalExecutorBehaviour, DispersalExecutorEvent, }; use kzgrs_backend::common::blob::DaBlob; - use kzgrs_backend::common::Column; + use kzgrs_backend::common::ColumnIndex; + use kzgrs_backend::encoder::{DaEncoder, DaEncoderParams, EncodedData}; use libp2p::identity::Keypair; use libp2p::swarm::SwarmEvent; use libp2p::{identity, quic, PeerId, Swarm}; + use nomos_core::da::DaEncoder as TraitDaEncoder; use nomos_da_messages::common::Blob; + use rand::RngCore; use std::collections::{HashMap, HashSet}; use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; @@ -360,21 +363,31 @@ mod tests { subnet_id: u32, messages_to_send: usize, ) { + // Generate 16 blobs + let mut buff = vec![0; 32 * DaEncoderParams::MAX_BLS12_381_ENCODING_CHUNK_SIZE]; + rand::thread_rng().fill_bytes(&mut buff); + let params = DaEncoderParams::default_with(32); + let encoder = DaEncoder::new(params); + let encoded_data: EncodedData = encoder.encode(&buff).unwrap(); + let blobs: Vec = encoded_data + .chunked_data + .columns() + .enumerate() + .map(|(idx, column)| DaBlob { + column, + column_idx: idx as ColumnIndex, + column_commitment: Default::default(), + aggregated_column_commitment: Default::default(), + aggregated_column_proof: Default::default(), + rows_commitments: vec![], + rows_proofs: vec![], + }) + .collect(); + for i in 0..messages_to_send { debug!("Sending blob {i} to subnet {subnet_id} ..."); disperse_blob_sender - .send(( - subnet_id, - DaBlob { - column_idx: 0, - column: Column(vec![]), - column_commitment: Default::default(), - aggregated_column_commitment: Default::default(), - aggregated_column_proof: Default::default(), - rows_commitments: vec![], - rows_proofs: vec![], - }, - )) + .send((subnet_id, blobs[i].clone())) .unwrap(); } } From b84774324e37e361acc6be1c061548b7ad6f29a9 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 4 Nov 2024 15:24:51 +0800 Subject: [PATCH 51/52] test: add workflow to debug on self-hosted --- .../build-test-validator-behaviour.yml | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 .github/workflows/build-test-validator-behaviour.yml diff --git a/.github/workflows/build-test-validator-behaviour.yml b/.github/workflows/build-test-validator-behaviour.yml new file mode 100644 index 000000000..c83ee3882 --- /dev/null +++ b/.github/workflows/build-test-validator-behaviour.yml @@ -0,0 +1,164 @@ +# copy of https://github.com/actions-rs/meta/blob/master/recipes/quickstart.md +# Steps for checking PRs. +on: + push: + branches: [ chore-da-behavior-membership-tests ] + +name: Test Validator Behaviour + +jobs: + check: + name: Check + runs-on: [ 'self-hosted' ] + strategy: + fail-fast: true + matrix: + feature: [ libp2p ] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: cargo-bins/cargo-binstall@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install risc0 + run: cargo binstall -y cargo-risczero && cargo risczero install + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: actions-rs/cargo@v1 + with: + command: check + args: --all --no-default-features --features ${{ matrix.feature }} + + test: + name: Test Suite + if: ${{ !startsWith(github.event.pull_request.title, '[WIP]') && !contains(github.event.label.name, 'DO NOT MERGE') }} + strategy: + fail-fast: false # all OSes should be tested even if one fails (default: true) + matrix: + feature: [ libp2p ] + os: [ self-hosted, macos-latest ] # drop windows for now as risc0 does not support it + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Remove unwanted software + if: matrix.os == 'ubuntu-latest' + uses: ./.github/actions/prune-vm + - uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + # Setup Rust toolchain with GNU for Windows + - name: Setup Rust with GNU toolchain (Windows) + if: matrix.os == 'windows-latest' + uses: actions-rs/toolchain@v1 + with: + toolchain: stable-msvc + target: x86_64-pc-windows-msvc + override: true + # Setup Rust toolchain for other OSes + - name: Setup Rust toolchain (Other OSes) + if: matrix.os != 'windows-latest' + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + # Setup build environment (Windows) + - name: Setup build environment (Windows) + if: matrix.os == 'windows-latest' + uses: ./.github/actions/mingw-env + - uses: cargo-bins/cargo-binstall@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install risc0 + run: cargo binstall -y cargo-risczero && cargo risczero install + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Cargo build + uses: actions-rs/cargo@v1 + with: + command: build + args: --all --no-default-features --features ${{ matrix.feature }} + - name: Cargo test (Other OSes) + if: matrix.os != 'windows-latest' + uses: actions-rs/cargo@v1 + env: + RUST_LOG: debug + RISC0_DEV_MODE: true + CONSENSUS_SLOT_TIME: 5 + with: + command: test + args: --all --no-default-features --features ${{ matrix.feature }} -p nomos-da-network-core test_validation_behaviour -- --nocapture + - name: Cargo test (Windows) + if: matrix.os == 'windows-latest' + uses: actions-rs/cargo@v1 + env: + # Because Windows runners in Github Actions tend to be slow. + SLOW_TEST_ENV: true + RISC0_DEV_MODE: true + CONSENSUS_SLOT_TIME: 5 + with: + command: test + args: --no-default-features --features ${{ matrix.feature }} + - name: Cargo bench (no-run) + if: matrix.os != 'windows-latest' + uses: actions-rs/cargo@v1 + env: + RISC0_DEV_MODE: true + CONSENSUS_SLOT_TIME: 5 + with: + command: bench + args: --no-run + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: integration-test-artifacts + path: tests/.tmp* + + lints: + name: Rust lints + runs-on: self-hosted + strategy: + matrix: + feature: [ libp2p ] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + components: rustfmt, clippy + - uses: cargo-bins/cargo-binstall@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install risc0 + run: cargo binstall -y cargo-risczero && cargo risczero install + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Run cargo fmt + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + - name: Run cargo clippy + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --all --no-default-features --features ${{ matrix.feature }} -- --deny warnings From c5aebd31d145d922c092eec3a5aea459dc1c937c Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 4 Nov 2024 15:48:14 +0800 Subject: [PATCH 52/52] test: double validators timeount and see success rate change --- .../network/core/src/protocols/dispersal/validator/behaviour.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs index 99d27ffe6..d30d19d6d 100644 --- a/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs +++ b/nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs @@ -324,7 +324,7 @@ mod tests { >, ) -> (usize, usize) { let (mut msg_0_counter, mut msg_1_counter) = (0, 0); - let timeout_duration = Duration::from_secs(15); + let timeout_duration = Duration::from_secs(30); loop { let event = timeout(timeout_duration, swarm.select_next_some()).await;