Optimize ImmutableSortedSet<T>.SetEquals to avoid unnecessary allocations

[aw0lid]

Part of #127279

Summary

ImmutableSortedSet<T>.SetEquals always creates a new intermediate SortedSet<T> for the other collection, leading to avoidable allocations and GC pressure, especially for large datasets

Optimization Logic

• Type-Specific Fast Paths: Uses pattern matching to detect if other is an ImmutableSortedSet or SortedSet, triggering optimized logic only if their Comparer matches.
• O(1) Early Exit: Performs an immediate ReferenceEquals check and leverages ICollection to return false early if other.Count is less than this.Count.
• Sequential Lock-Step Comparison: Replaces the $O(\log n)$ per-element .Contains() check with a dual-enumerator while loop. This leverages the sorted nature of both sets to achieve $O(n)$ linear complexity.
• Zero Allocation Path: For compatible sorted sets, the comparison is performed directly on existing instances, eliminating the memory overhead of temporary collections.
• Refined Fallback: Even when a new SortedSet<T> is required for general IEnumerable types, the final comparison now uses the same efficient $O(n)$ sequential scan instead of repeated lookups.

Click to expand Benchmark Source Code

using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Order;
using BenchmarkDotNet.Running;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;

namespace ImmutableSortedSetBenchmarks
{
    [MemoryDiagnoser]
    [Orderer(SummaryOrderPolicy.FastestToSlowest)]
    [RankColumn]
    public class ImmutableSortedSetSetEqualsBenchmark_Int
    {
        private ImmutableSortedSet<int> _sourceSet = null!;
        private ImmutableSortedSet<int> _immutableSortedSetEqual = null!;
        private SortedSet<int> _bclSortedSetEqual = null!;
        private List<int> _listEqual = null!;
        private IEnumerable<int> _linqSelectEqual = null!;
        private int[] _arrayEqual = null!;
        private List<int> _listLastDiff = null!;
        private List<int> _listSmaller = null!;
        private ImmutableSortedSet<int> _immutableLarger = null!;
        private int[] _smallerArray = null!;
        private SortedSet<int> _smallerSortedSetDiffComparer = null!;

        private ImmutableSortedSet<int> _immutableSortedSetLastDiff = null!;
        private SortedSet<int> _bclSortedSetLastDiff = null!;
        private List<int> _listWithDuplicates = null!;
        private List<int> _listWithDuplicatesMatch = null!;

        private SortedSet<int> _bclSortedSetDiffComparer = null!;

        private ImmutableSortedSet<int> _immutableSortedSetSmaller = null!;
        private SortedSet<int> _bclSortedSetSmaller = null!;

        private IEnumerable<int> _lazyEnumerableLastDiff = null!;
       

        [Params(100000)]
        public int Size { get; set; }

        [GlobalSetup]
        public void Setup()
        {
            var elements = Enumerable.Range(0, Size).ToList();
            var elementsWithLastDiff = Enumerable.Range(0, Size - 1).Concat(new[] { Size + 1000 }).ToList();
            var smallerElements = Enumerable.Range(0, Size / 2).ToList();
            var duplicates = Enumerable.Repeat(1, Size).ToList();
            var smallerList = new List<int>();

            for(int i = 0; i < Size - 1; i++) smallerList.Add(i);

            var reverseComparer = new ReverseComparer<int>();

            _sourceSet = ImmutableSortedSet.CreateRange(elements);
            _immutableSortedSetEqual = ImmutableSortedSet.CreateRange(elements);
            _bclSortedSetEqual = new SortedSet<int>(elements);
            _listEqual = elements;
            _linqSelectEqual = elements.Select(x => x); 
            _arrayEqual = elements.ToArray();

            _immutableSortedSetLastDiff = ImmutableSortedSet.CreateRange(elementsWithLastDiff);
            _bclSortedSetLastDiff = new SortedSet<int>(elementsWithLastDiff);
            _listLastDiff = elementsWithLastDiff;

            _bclSortedSetDiffComparer = new SortedSet<int>(elements, reverseComparer);

            _immutableSortedSetSmaller = ImmutableSortedSet.CreateRange(smallerElements);
            _bclSortedSetSmaller = new SortedSet<int>(smallerElements);

            _lazyEnumerableLastDiff = elementsWithLastDiff.Select(x => x);
            _immutableLarger = ImmutableSortedSet.CreateRange(elements.Concat(new[] { -1 }));
            _listWithDuplicates = duplicates;
            _listWithDuplicatesMatch = elements.Concat(elements).ToList(); 
            _listSmaller = smallerList;
            _smallerArray = Enumerable.Range(0, Size - 1).ToArray();
            _smallerSortedSetDiffComparer = new SortedSet<int>(_listSmaller, reverseComparer);
        }

        #region Fast Path: Same Type and Comparer

        [Benchmark(Description = "ImmutableSortedSet (Match - Same Comparer)")]
        public bool Case_ImmutableSortedSet_Match() => _sourceSet.SetEquals(_immutableSortedSetEqual);

        [Benchmark(Description = "BCL SortedSet (Match - Same Comparer)")]
        public bool Case_BclSortedSet_Match() => _sourceSet.SetEquals(_bclSortedSetEqual);

        [Benchmark(Description = "ImmutableSortedSet (Mismatch - Same Count)")]
        public bool Case_ImmutableSortedSet_LastDiff() => _sourceSet.SetEquals(_immutableSortedSetLastDiff);

        [Benchmark(Description = "BCL SortedSet (Mismatch - Same Count)")]
        public bool Case_BclSortedSet_LastDiff() => _sourceSet.SetEquals(_bclSortedSetLastDiff);

        #endregion

        #region Early Exit: Count Mismatch

        [Benchmark(Description = "ImmutableSortedSet (Smaller Count)")]
        public bool Case_ImmutableSortedSet_SmallerCount() => _sourceSet.SetEquals(_immutableSortedSetSmaller);

        [Benchmark(Description = "BCL SortedSet (Smaller Count)")]
        public bool Case_BclSortedSet_SmallerCount() => _sourceSet.SetEquals(_bclSortedSetSmaller);

        [Benchmark(Description = "Array (Smaller Count)")]
        public bool Case_SmallerCollection_EarlyExit() => _sourceSet.SetEquals(_smallerArray);

        #endregion

        #region Fallback Path: Different Comparer

        [Benchmark(Description = "SortedSet (Different Comparer)")]
        public bool Case_SortedSet_DifferentComparer() => _sourceSet.SetEquals(_bclSortedSetDiffComparer);

        [Benchmark(Description = "SortedSet (Smaller Count - Different Comparer)")]
        public bool Case_SortedSet_SmallerCount_DiffComparer() => _sourceSet.SetEquals(_smallerSortedSetDiffComparer);

        #endregion

        #region Fallback Path: Non-Set Collections

        [Benchmark(Description = "List (Match - Fallback)")]
        public bool Case_List_Match() => _sourceSet.SetEquals(_listEqual);

        [Benchmark(Description = "LINQ (Mismatch - Lazy IEnumerable)")]
        public bool Case_LazyEnumerable_LastDiff() => _sourceSet.SetEquals(_lazyEnumerableLastDiff);

        [Benchmark(Description = "LINQ (Match - Lazy IEnumerable)")]
        public bool Case_LazyEnumerable_Match() => _sourceSet.SetEquals(_linqSelectEqual);

        [Benchmark(Description = "List (Last Diff - Fallback)")]
        public bool Case_List_LastDiff() => _sourceSet.SetEquals(_listLastDiff);

        [Benchmark(Description = "Array (Match - Fallback)")]
        public bool Case_Array_Match() => _sourceSet.SetEquals(_arrayEqual);

        [Benchmark(Description = "ImmutableSortedSet (Larger Count)")]
        public bool Case_LargerCount() => _sourceSet.SetEquals(_immutableLarger);

        #endregion

        #region Handling Duplicates

        [Benchmark(Description = "List with Duplicates (Mismatch)")]
        public bool Case_List_Duplicates_Mismatch() => _sourceSet.SetEquals(_listWithDuplicates);

        [Benchmark(Description = "List with Duplicates (Match)")]
        public bool Case_List_Duplicates_Match() => _sourceSet.SetEquals(_listWithDuplicatesMatch);

        #endregion
    }

    public class ReverseComparer<T> : IComparer<T> where T : IComparable<T>
    {
        public int Compare(T? x, T? y)
        {
            if (x == null && y == null) return 0;
            if (x == null) return 1;
            if (y == null) return -1;
            return y.CompareTo(x);
        }
    }

    public class Program
    {
        public static void Main(string[] args)
        {
            BenchmarkRunner.Run<ImmutableSortedSetSetEqualsBenchmark_Int>();
        }
    }
}

Click to expand Benchmark Results

Benchmark Results (Before Optimization)

Method	Size	Mean	Error	StdDev	Rank	Gen0	Gen1	Gen2	Allocated
'List with Duplicates (Mismatch)'	100000	1.931 ms	0.0321 ms	0.0268 ms	1	9.7656	9.7656	9.7656	390.8 KB
'BCL SortedSet (Smaller Count)'	100000	1.956 ms	0.0249 ms	0.0233 ms	1	371.0938	285.1563	-	1953.73 KB
'ImmutableSortedSet (Smaller Count)'	100000	3.516 ms	0.0413 ms	0.0387 ms	2	367.1875	304.6875	3.9063	2148.52 KB
'Array (Smaller Count)'	100000	4.910 ms	0.0860 ms	0.0718 ms	3	671.8750	609.3750	7.8125	4296.91 KB
'SortedSet (Smaller Count - Different Comparer)'	100000	6.799 ms	0.0873 ms	0.0816 ms	4	679.6875	609.3750	7.8125	4297.31 KB
'ImmutableSortedSet (Larger Count)'	100000	7.513 ms	0.1106 ms	0.0981 ms	5	671.8750	625.0000	7.8125	4297 KB
'List (Match - Fallback)'	100000	13.665 ms	0.1204 ms	0.1005 ms	6	671.8750	625.0000	-	4297.26 KB
'Array (Match - Fallback)'	100000	13.797 ms	0.0807 ms	0.0716 ms	6	656.2500	625.0000	-	4297.25 KB
'List (Last Diff - Fallback)'	100000	13.888 ms	0.1573 ms	0.1394 ms	6	671.8750	609.3750	-	4297.25 KB
'BCL SortedSet (Match - Same Comparer)'	100000	14.849 ms	0.2830 ms	0.2647 ms	7	671.8750	609.3750	-	3907.19 KB
'BCL SortedSet (Mismatch - Same Count)'	100000	15.137 ms	0.2700 ms	0.2526 ms	7	671.8750	625.0000	-	3907.19 KB
'LINQ (Mismatch - Lazy IEnumerable)'	100000	15.202 ms	0.1243 ms	0.1162 ms	7	750.0000	625.0000	15.6250	4931.04 KB
'SortedSet (Different Comparer)'	100000	15.214 ms	0.1632 ms	0.1447 ms	7	640.6250	625.0000	-	4297.65 KB
'LINQ (Match - Lazy IEnumerable)'	100000	15.360 ms	0.1323 ms	0.1238 ms	7	734.3750	640.6250	15.6250	4931.13 KB
'ImmutableSortedSet (Mismatch - Same Count)'	100000	17.122 ms	0.2080 ms	0.1946 ms	8	656.2500	593.7500	-	4297.26 KB
'ImmutableSortedSet (Match - Same Comparer)'	100000	17.256 ms	0.2191 ms	0.1829 ms	8	750.0000	593.7500	-	4297.25 KB
'List with Duplicates (Match)'	100000	31.781 ms	0.2065 ms	0.1724 ms	9	625.0000	562.5000	-	4687.9 KB

---

Benchmark Results (After Optimization)

Method	Size	Mean	Error	StdDev	Gen0	Gen1	Gen2	Allocated
'ImmutableSortedSet (Larger Count)'	100000	1.605 ns	0.0427 ns	0.0379 ns	-	-	-	-
'ImmutableSortedSet (Smaller Count)'	100000	1.624 ns	0.0491 ns	0.0459 ns	-	-	-	-
'SortedSet (Smaller Count - Different Comparer)'	100000	1.690 ns	0.0456 ns	0.0404 ns	-	-	-	-
'BCL SortedSet (Smaller Count)'	100000	1.816 ns	0.0569 ns	0.0532 ns	-	-	-	-
'Array (Smaller Count)'	100000	2.155 ns	0.0625 ns	0.0522 ns	-	-	-	-
'List with Duplicates (Mismatch)'	100000	1,241,268.377 ns	9,938.3202 ns	8,810.0620 ns	9.7656	9.7656	9.7656	400150 B
'BCL SortedSet (Mismatch - Same Count)'	100000	2,656,001.235 ns	26,235.3715 ns	23,256.9735 ns	-	-	-	312 B
'BCL SortedSet (Match - Same Comparer)'	100000	2,656,167.835 ns	36,661.1748 ns	32,499.1766 ns	-	-	-	314 B
'ImmutableSortedSet (Mismatch - Same Count)'	100000	3,658,683.860 ns	43,801.7836 ns	40,972.2155 ns	-	-	-	-
'ImmutableSortedSet (Match - Same Comparer)'	100000	3,672,184.801 ns	28,641.6696 ns	22,361.5316 ns	-	-	-	-
'List (Match - Fallback)'	100000	7,607,183.416 ns	116,938.1638 ns	97,648.6630 ns	671.8750	625.0000	-	4400390 B
'List (Last Diff - Fallback)'	100000	7,713,407.057 ns	123,214.3767 ns	109,226.3356 ns	671.8750	625.0000	7.8125	4400387 B
'Array (Match - Fallback)'	100000	7,717,749.754 ns	124,099.3086 ns	116,082.5703 ns	664.0625	625.0000	7.8125	4400387 B
'LINQ (Mismatch - Lazy IEnumerable)'	100000	8,974,566.438 ns	171,672.9513 ns	160,582.9853 ns	750.0000	593.7500	15.6250	5049380 B
'LINQ (Match - Lazy IEnumerable)'	100000	9,006,743.952 ns	112,508.0211 ns	105,240.0728 ns	734.3750	640.6250	15.6250	5049380 B
'SortedSet (Different Comparer)'	100000	9,661,422.295 ns	175,748.1986 ns	164,394.9741 ns	656.2500	609.3750	-	4400729 B
'List with Duplicates (Match)'	100000	24,718,281.944 ns	253,370.8073 ns	224,606.6212 ns	656.2500	593.7500	-	4800396 B

Performance Analysis Summary (100,000 Elements)

Performance Analysis Summary (100,000 Elements)

Case	Speed Improvement	Memory Improvement
ImmutableSortedSet (Larger Count)	~4,681,000x	Zero Alloc
SortedSet (Smaller - Diff Comparer)	~4,023,000x	Zero Alloc
ImmutableSortedSet (Smaller Count)	~2,165,000x	Zero Alloc
Array (Smaller Count)	~2,278,000x	Zero Alloc
BCL SortedSet (Smaller Count)	~1,077,000x	Zero Alloc
BCL SortedSet (Mismatch - Same Count)	~5.70x	~99.99%
BCL SortedSet (Match - Same Comparer)	~5.59x	~99.99%
ImmutableSortedSet (Match - Same Comparer)	~4.70x	Zero Alloc
ImmutableSortedSet (Mismatch - Same Count)	~4.68x	Zero Alloc
List (Match - Fallback)	~1.80x	Stable
Array (Match - Fallback)	~1.79x	Stable
List (Last Diff - Fallback)	~1.80x	Stable
LINQ (Mismatch - Lazy IEnumerable)	~1.69x	Stable
LINQ (Match - Lazy IEnumerable)	~1.70x	Stable
List with Duplicates (Mismatch)	~1.56x	Stable
SortedSet (Different Comparer)	~1.57x	Stable
List with Duplicates (Match)	~1.29x	Stable

---

Testing

In addition to the fix, I have added comprehensive unit tests covering various scenarios to ensure correctness:

• Mismatched Comparers (Ordinal vs. OrdinalIgnoreCase): Verified that SetEquals returns true when logically equal but with different comparers, and false when logically different.
• ICollection with Duplicates: Verified the fallback path correctly handles collections like List<T> with duplicate elements.
• Count Optimizations:
  • Verified that mismatched comparers correctly bypass the fast-path count check.
  • Verified that SetEquals still performs early-exit when other.Count < source.Count.
• Fast-Path Validation: Ensured that when comparers match, the optimized count-based comparison still works as expected.
• Edge Cases: Included tests for empty sets with different comparers and content-specific mismatches.

[aw0lid]

Just gentle ping in case this fell through the cracks.
Ready for any review.

[aw0lid]

The count validation has been moved out of SetEqualsWithImmutableSortedset and SetEqualsWithSortedset, and the iteration logic in SetEqualsWithImmutableSortedset has been reversed. This was done to make these methods reusable for optimizing other set operations, such as IsProperSubsetOf and IsSubsetOf, in subsequent PRs after this one is merged

This refactoring enables us to implement logic like the following:

private static bool IsProperSubsetOf(IEnumerable<T> other, ImmutableSortedSet<T> source)
{
    Requires.NotNull(other, nameof(other));

    if (Source.IsEmpty)
    {
        return other.Any();
    }

    if (other is ICollection<T> otherAsICollectionGeneric)
    {
        // We check for < instead of != because other is not guaranteed to be a set, it could be a collection with duplicates.
        if (otherAsICollectionGeneric.Count <= source.Count)
        {
            return false;
        }

        switch (other)
        {
            case ImmutableSortedSet<T> otherAsImmutableSortedSet:
                if (source.EqualityComparer.Equals(otherAsImmutableSortedSet.KeyComparer))
                {
                    return SetEqualsWithImmutableSortedset(otherAsImmutableSortedSet, source);
                }
                break;

            case SortedSet<T> otherAsSortedset:
                if (source.EqualityComparer.Equals(otherAsSortedset.Comparer))
                {
                    return SetEqualsWithSortedset(otherAsHashset, source);
                }
                break;
        }
    }
    else if (other is ICollection otherAsICollection && otherAsICollection.Count <= source.Count)
    {
        return false;
    }

    var otherSet = new SortedSet<T>(other, origin.EqualityComparer);
    if (otherSet.Count <= source.Count)
    {
        return false;
    }

    return SetEqualsWithSortedset(otherSet, source);
}

[aw0lid]

CI seems to be stuck on infrastructure issues. Could someone please trigger a re-run for the failed checks? Thanks!

[eiriktsarpalis]

/ba-g test failures are unrelated

[aw0lid]

Gentle ping @eiriktsarpalis. The changes have been updated based on the feedback and the previous approval. The CI checks seem to be passing now (except for unrelated failures). Please let me know if there's anything else needed to get this merged. Thanks!

[eiriktsarpalis]

/ba-g test failures are unrelated