Perform the deduplication of the validators set (#1621)

* Add dedupe utils

* Implement validator deduplication based on account id and the first-in order

* Fix the doc comment on the DedupeKeyExtractor::Output

* Fixed doc comment on DedupeIter::dedupe_key_extractor

* Apply suggestion from @dmitrylavrenov

Co-authored-by: Dmitry Lavrenov <39522748+dmitrylavrenov@users.noreply.github.com>

* Add a test

* Remove the broken impl (doesn't work either way)

---------

Co-authored-by: Dmitry Lavrenov <39522748+dmitrylavrenov@users.noreply.github.com>

Author: MOZGIII

crates/pallet-humanode-session/src/lib.rs

@@ -2,6 +2,8 @@
 
 #![cfg_attr(not(feature = "std"), no_std)]
 
+extern crate alloc;
+
 #[cfg(feature = "try-runtime")]
 use frame_support::sp_runtime::TryRuntimeError;
 use frame_support::traits::{Get, StorageVersion};
@@ -21,6 +23,9 @@ pub mod benchmarking;
 mod mock;
 #[cfg(test)]
 mod tests;
+mod utils;
+
+use self::utils::DedupeIteratorExt as _;
 
 /// The type representing the session index in our chain.
 type SessionIndex = u32;
@@ -275,6 +280,7 @@ impl<T: Config> Pallet<T> {
         bootnodes
             .chain(bioauth_active_authentications)
             .chain(fixed_validators)
+            .dedupe(AccountIdDedupeKey::<T>::default())
     }
 
     /// Clears and re-populates the [`SessionIdentities`] for a given session with the entries.
@@ -352,3 +358,20 @@ impl<T: Config> sp_runtime::traits::Convert<T::AccountId, Option<IdentificationF
         <SessionIdentities<T>>::get(session_index, account_id)
     }
 }
+
+/// The dedupe key extractor that provides Account Ids from Identification Tuples.
+struct AccountIdDedupeKey<T>(core::marker::PhantomData<T>);
+
+impl<T: Config> utils::DedupeKeyExtractor<IdentificationTupleFor<T>> for AccountIdDedupeKey<T> {
+    type Output = <T as frame_system::Config>::AccountId;
+
+    fn extract_key(&self, value: &IdentificationTupleFor<T>) -> Self::Output {
+        value.0.clone()
+    }
+}
+
+impl<T> Default for AccountIdDedupeKey<T> {
+    fn default() -> Self {
+        Self(core::marker::PhantomData)
+    }
+}

crates/pallet-humanode-session/src/utils.rs

@@ -0,0 +1,5 @@
+//! Utilities.
+
+mod dedupe_iter;
+
+pub use self::dedupe_iter::*;

crates/pallet-humanode-session/src/utils/dedupe_iter.rs

@@ -0,0 +1,119 @@
+//! Deduplicating iterator wrapper.
+
+use alloc::collections::BTreeSet;
+
+/// Dedupe key extraction.
+///
+/// Provides a dedupe key for type `T`.
+pub trait DedupeKeyExtractor<T> {
+    /// The type this extractor extracts; the dedupe key.
+    type Output;
+
+    /// Perform the dedupe key extraction from the given value.
+    fn extract_key(&self, value: &T) -> Self::Output;
+}
+
+/// The dedupe iterator.
+pub struct DedupeIter<Source, DedupeKeyExtractor>
+where
+    Source: Iterator,
+    DedupeKeyExtractor: self::DedupeKeyExtractor<Source::Item>,
+{
+    /// The source iterator.
+    pub source: Source,
+
+    /// The dedupe key extractor.
+    pub dedupe_key_extractor: DedupeKeyExtractor,
+
+    /// The state for tracking the duplicates.
+    pub dedupe_state:
+        BTreeSet<<DedupeKeyExtractor as self::DedupeKeyExtractor<Source::Item>>::Output>,
+}
+
+impl<Source, DedupeKeyExtractor> DedupeIter<Source, DedupeKeyExtractor>
+where
+    Source: Iterator,
+    DedupeKeyExtractor: self::DedupeKeyExtractor<Source::Item>,
+{
+    /// Create a new dedupe iterator from the given iterator.
+    pub fn new(source: Source, dedupe_key_extractor: DedupeKeyExtractor) -> Self {
+        Self {
+            source,
+            dedupe_key_extractor,
+            dedupe_state: Default::default(),
+        }
+    }
+}
+
+impl<Source, DedupeKeyExtractor> Iterator for DedupeIter<Source, DedupeKeyExtractor>
+where
+    Source: Iterator,
+    DedupeKeyExtractor: self::DedupeKeyExtractor<Source::Item>,
+    DedupeKeyExtractor::Output: Ord,
+{
+    type Item = Source::Item;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            let item = self.source.next()?;
+            let dedupe_key = self.dedupe_key_extractor.extract_key(&item);
+            let was_new = self.dedupe_state.insert(dedupe_key);
+            if !was_new {
+                continue;
+            }
+            return Some(item);
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let (_source_lower, source_higher) = self.source.size_hint();
+
+        // Lower bound is always unpredictably `0`.
+        (0, source_higher)
+    }
+}
+
+/// [`Iterator`] extension trait for `dedupe` fn.
+pub trait DedupeIteratorExt: Iterator + Sized {
+    /// Deduplicate the iterator.
+    fn dedupe<DedupeKeyExtractor: self::DedupeKeyExtractor<Self::Item>>(
+        self,
+        dedupe_key_extractor: DedupeKeyExtractor,
+    ) -> DedupeIter<Self, DedupeKeyExtractor>;
+}
+
+impl<T> DedupeIteratorExt for T
+where
+    T: Iterator,
+{
+    fn dedupe<DedupeKeyExtractor: self::DedupeKeyExtractor<Self::Item>>(
+        self,
+        dedupe_key_extractor: DedupeKeyExtractor,
+    ) -> DedupeIter<Self, DedupeKeyExtractor> {
+        DedupeIter::new(self, dedupe_key_extractor)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    struct IdentityCopy;
+
+    impl<T: Copy> DedupeKeyExtractor<T> for IdentityCopy {
+        type Output = T;
+
+        fn extract_key(&self, value: &T) -> Self::Output {
+            *value
+        }
+    }
+
+    #[test]
+    fn dedupe() {
+        let iter = vec![1usize, 2, 1].into_iter();
+
+        let deduped_iter = iter.dedupe(IdentityCopy);
+
+        assert_eq!(deduped_iter.collect::<Vec<_>>(), vec![1, 2]);
+    }
+}