Reduce memory usage of "ExplicitPeakBounds" and "SpectrumMetadata" (#3374)

Add some subclasses of "ImmutableList" which can store some types of data more efficiently:
ConstantList: for when all of the elements in the list are identical
IntegerList: for when the elements are integers but their values are in a range that can be represented by less than 4 bytes.
Factor: for when the list is long but the number of unique values is small so they can be represented as integer indexes into the list of unique values
NullableList: stored as a list of struct's alongside a list of bits.

Author: nickshulman

pwiz_tools/Shared/CommonUtil/Collections/Factor.cs

@@ -0,0 +1,102 @@
+/*
+ * Original author: Nicholas Shulman <nicksh .at. u.washington.edu>,
+ *                  MacCoss Lab, Department of Genome Sciences, UW
+ *
+ * Copyright 2025 University of Washington - Seattle, WA
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace pwiz.Common.Collections
+{
+    /// <summary>
+    /// Stores items in a list as integer indexes into a list of possible values.
+    /// The list of possible values are <see cref="Levels"/>.
+    /// The indexes into the Levels list are stored in <see cref="LevelIndices"/> 
+    /// which may be stored as bits, bytes, shorts or integers depending on
+    /// the highest value required.
+    /// </summary>
+    public class Factor<T> : ImmutableList<T>
+    {
+        public static Factor<T> FromItems(IEnumerable<T> items)
+        {
+            return FromItemsWithLevels(items, ImmutableList<T>.EMPTY);
+        }
+
+        public static Factor<T> FromItemsWithLevels(IEnumerable<T> items, ImmutableList<T> startingLevels)
+        {
+            if (items is Factor<T> factor && factor.Levels.Take(startingLevels.Count).SequenceEqual(startingLevels))
+            {
+                return factor;
+            }
+            var levelsDict = new Dictionary<ValueTuple<T>, int>();
+            foreach (var level in startingLevels)
+            {
+                levelsDict.Add(ValueTuple.Create(level), levelsDict.Count);
+            }
+            var levelIndices = new List<int>();
+            foreach (var item in items)
+            {
+                var key = ValueTuple.Create(item);
+                if (!levelsDict.TryGetValue(key, out int levelIndex))
+                {
+                    levelIndex = levelsDict.Count;
+                    levelsDict.Add(key, levelIndex);
+                }
+
+                levelIndices.Add(levelIndex);
+            }
+
+            ImmutableList<T> levels;
+            if (levelsDict.Count == startingLevels.Count)
+            {
+                levels = startingLevels;
+            }
+            else
+            {
+                levels = levelsDict.OrderBy(kvp => kvp.Value).Select(kvp => kvp.Key.Item1).ToImmutable();
+            }
+            return new Factor<T>(levels, IntegerList.FromIntegers(levelIndices));
+        }
+
+        public Factor(ImmutableList<T> levels, ImmutableList<int> levelIndices)
+        {
+            Levels = levels;
+            LevelIndices = levelIndices;
+        }
+
+        public ImmutableList<T> Levels { get; }
+        public ImmutableList<int> LevelIndices { get; }
+
+        public override int Count
+        {
+            get { return LevelIndices.Count; }
+        }
+        public override IEnumerator<T> GetEnumerator()
+        {
+            return LevelIndices.Select(i => Levels[i]).GetEnumerator();
+        }
+
+        public override T this[int index]
+        {
+            get
+            {
+                return Levels[LevelIndices[index]];
+            }
+        }
+    }
+}

pwiz_tools/Shared/CommonUtil/Collections/ImmutableList.cs

@@ -140,18 +140,20 @@ public static ImmutableList<T> Singleton(T value)
         }
 
         /// <summary>
-        /// Private constructor to disallow any other implementations of this class.
+        /// GetHashCode implementation which cannot be overridden
+        /// in derived classes in order to preserve the semantics
+        /// that ImmutableList instances are equal if and only if
+        /// they contain the same items.
         /// </summary>
-        private ImmutableList()
-        {
-        }
-
-        public override int GetHashCode()
+        public sealed override int GetHashCode()
         {
             return CollectionUtil.GetHashCodeDeep(this);
         }
 
-        public override bool Equals(object o)
+        /// <summary>
+        /// Equals implementation which cannot be overridden.
+        /// </summary>
+        public sealed override bool Equals(object o)
         {
             if (o == null)
             {
@@ -161,14 +163,35 @@ public override bool Equals(object o)
             {
                 return true;
             }
+            if (GetType() == o.GetType())
+            {
+                return SameTypeEquals((ImmutableList<T>)o);
+            }
             var that = o as ImmutableList<T>;
             if (null == that)
             {
                 return false;
             }
+
+            if (Count != that.Count)
+            {
+                return false;
+            }
+
             return this.SequenceEqual(that);
         }
 
+        /// <summary>
+        /// Virtual equals implementation which is only called if the list being
+        /// compared against is the same type as this. Derived classes may override
+        /// this if they have a more efficient way of comparing against other
+        /// instances of themselves (e.g. <see cref="ConstantList{T}"/>)
+        /// </summary>
+        protected virtual bool SameTypeEquals(ImmutableList<T> list)
+        {
+            return this.SequenceEqual(list);
+        }
+
         public abstract int Count { get; }
 
         void ICollection<T>.Add(T item)
@@ -204,10 +227,32 @@ IEnumerator IEnumerable.GetEnumerator()
 
         public abstract IEnumerator<T> GetEnumerator();
 
-        public abstract bool Contains(T item);
-        public abstract void CopyTo(T[] array, int arrayIndex);
+        public virtual bool Contains(T item)
+        {
+            return this.AsEnumerable().Contains(item);
+        }
+        public virtual void CopyTo(T[] array, int arrayIndex) 
+        {
+            foreach (var item in this)
+            {
+                array[arrayIndex++] = item;
+            }
+        }
+
+        public virtual int IndexOf(T item)
+        {
+            int index = 0;
+            foreach (var v in this)
+            {
+                if (Equals(v, item))
+                {
+                    return index;
+                }
+                index++;
+            }
 
-        public abstract int IndexOf(T item);
+            return -1;
+        }
         public abstract T this[int index] { get; }
 
         T IList<T>.this[int index]
@@ -216,12 +261,12 @@ T IList<T>.this[int index]
             set { throw new InvalidOperationException(); }
         }
 
-        public abstract ImmutableList<T> ReplaceAt(int index, T value);
-
-        /// <summary>
-        /// Replaces the first item in the list that matches the predicate with the given value; throws IndexOutOfRange if no item is replaced.
-        /// </summary>
-        public abstract ImmutableList<T> ReplaceElement(T value, Func<T, bool> predicate);
+        public virtual ImmutableList<T> ReplaceAt(int index, T value)
+        {
+            var array = this.ToArray();
+            array[index] = value;
+            return ImmutableList.ValueOf(array);
+        }
 
         private class Impl : ImmutableList<T>
         {
@@ -271,12 +316,6 @@ public override ImmutableList<T> ReplaceAt(int index, T value)
                 newArray[index] = value;
                 return new Impl(newArray);
             }
-
-            public override ImmutableList<T> ReplaceElement(T value, Func<T, bool> predicate)
-            {
-                int index = Array.FindIndex(_items, t => predicate(t));
-                return ReplaceAt(index, value);
-            }
         }
 
         private class SingletonImpl : ImmutableList<T>
@@ -332,14 +371,90 @@ public override ImmutableList<T> ReplaceAt(int index, T value)
                 }
                 return new SingletonImpl(value);
             }
+        }
+    }
 
-            public override ImmutableList<T> ReplaceElement(T value, Func<T, bool> predicate)
+    public class ConstantList<T> : ImmutableList<T>
+    {
+        private int _count;
+        private T _value;
+        public ConstantList(int count, T value)
+        {
+            _count = count;
+            _value = value;
+        }
+
+        public override IEnumerator<T> GetEnumerator()
+        {
+            return Enumerable.Repeat(_value, _count).GetEnumerator();
+        }
+
+        public override int Count
+        {
+            get { return _count; }
+        }
+
+        public override T this[int index]
+        {
+            get
             {
-                if (!predicate(_item))
-                {
-                    throw new IndexOutOfRangeException();
-                }
-                return ReplaceAt(0, value);
+                if (index < 0 || index >= _count)
+                    throw new ArgumentOutOfRangeException(nameof(index));
+                return _value;
+            }
+        }
+
+        protected override bool SameTypeEquals(ImmutableList<T> list)
+        {
+            var that = (ConstantList<T>) list;
+            return Count == list.Count && Equals(_value, that._value);
+        }
+    }
+
+    /// <summary>
+    /// Efficiently stores a list of <see cref="Nullable"/> by using
+    /// <see cref="IntegerList.Bits"/>.
+    /// </summary>
+    public class NullableList<T> : ImmutableList<T?> where T : struct
+    {
+        private ImmutableList<int> _hasValueList;
+        private ImmutableList<T> _values;
+
+        public NullableList(IEnumerable<T?> items)
+        {
+            var values = new List<T>();
+            var hasValues = new List<int>();
+            foreach (var item in items)
+            {
+                values.Add(item.GetValueOrDefault());
+                hasValues.Add(item.HasValue ? 1 : 0);
+            }
+
+            _hasValueList = IntegerList.FromIntegers(hasValues);
+            if (typeof(T) == typeof(int))
+            {
+                _values = (ImmutableList<T>)(object)IntegerList.FromIntegers((List<int>)(object) values);
+            }
+            else
+            {
+                _values = values.ToImmutable();
+            }
+        }
+
+        public override int Count
+        {
+            get { return _values.Count; }
+        }
+        public override IEnumerator<T?> GetEnumerator()
+        {
+            return Enumerable.Range(0, Count).Select(i => this[i]).GetEnumerator();
+        }
+
+        public override T? this[int index]
+        {
+            get
+            {
+                return _hasValueList[index] == 0 ? (T?) null : _values[index];
             }
         }
     }