Serialization v3 aka constant suffering (space-wizards#1606)

* oops

* fixes serialization il

* copytest

* typo & misc fixes

* 139 moment

* boxing

* mesa dum

* stuff

* goodbye bad friend

* last commit before the big (4) rewrite

* adds datanodes

* kills yamlobjserializer in favor of the new system

* adds more serializers, actually implements them & removes most of the last of the old system

* changed yamlfieldattribute namespace

* adds back iselfserialize

* refactors consts&flags

* renames everything to data(field/definition)

* adds afterserialization

* help

* dataclassgen

* fuggen help me mannen

* Fix most errors on content

* Fix engine errors except map loader

* maploader & misc fix

* misc fixes

* thing

* help

* refactors datanodes

* help me mannen

* Separate ITypeSerializer into reader and writer

* Convert all type serializers

* priority

* adds alot

* il fixes

* adds robustgen

* argh

* adds array & enum serialization

* fixes dataclasses

* adds vec2i / misc fixes

* fixes inheritance

* a very notcursed todo

* fixes some custom dataclasses

* push dis

* Remove data classes

* boutta box

* yes

* Add angle and regex serializer tests

* Make TypeSerializerTest abstract

* sets up ioc etc

* remove pushinheritance

* fixes

* Merge fixes, fix yaml hot reloading

* General fixes2

* Make enum serialization ignore case

* Fix the tag not being copied in data nodes

* Fix not properly serializing flag enums

* Fix component serialization on startup

* Implement ValueDataNode ToString

* Serialization IL fixes, fix return and string equality

* Remove async from prototype manager

* Make serializing unsupported node as enum exception more descriptive

* Fix serv3 tryread casting to serializer instead of reader

* Add constructor for invalid node type exception

* Temporary fix for SERV3: Turn populate delegate into regular code

* Fix not copying the data of non primitive types

* Fix not using the data definition found in copying

* Make ISerializationHooks require explicit implementations

* Add test for serialization inheritance

* Improve IsOverridenIn method

* Fix error message when a data definition is null

* Add method to cast a read value in Serv3Manager

* Rename IServ3Manager to ISerializationManager

* Rename usages of serv3manager, add generic copy method

* Fix IL copy method lookup

* Rename old usages of serv3manager

* Add ITypeCopier

* resistance is futile

* we will conquer this codebase

* Add copy method to all serializers

* Make primitive mismatch error message more descriptive

* bing bong im going to freacking heck

* oopsie moment

* hello are you interested in my wares

* does generic serializers under new architecture

* Convert every non generic serializer to the new format, general fixes

* Update usgaes of generic serializers, cleanup

* does some pushinheritance logic

* finishes pushinheritance FRAMEWORK

* shed

* Add box2, color and component registry serializer tests

* Create more deserialized types and store prototypes with their deserialized results

* Fixes and serializer updates

* Add serialization manager extensions

* adds pushinheritance

* Update all prototypes to have a parent and have consistent id/parent properties

* Fix grammar component serialization

* Add generic serializer tests

* thonk

* Add array serializer test

* Replace logger warning calls with exceptions

* fixes

* Move redundant methods to serialization manager extensions, cleanup

* Add array serialization

* fixes context

* more fixes

* argh

* inheritance

* this should do it

* fixes

* adds copiers & fixes some stuff

* copiers use context v1

* finishing copy context

* more context fixes

* Test fixes

* funky maps

* Fix server user interface component serialization

* Fix value tuple serialization

* Add copying for value types and arrays. Fix copy internal for primitives, enums and strings

* fixes

* fixes more stuff

* yes

* Make abstract/interface skips debugs instead of warnings

* Fix typo

* Make some dictionaries readonly

* Add checks for the serialization manager initializing and already being initialized

* Add base type required and usage for MeansDataDefinition and ImplicitDataDefinitionForInheritorsAttribute

* copy by ref

* Fix exception wording

* Update data field required summary with the new forbidden docs

* Use extension in map loader

* wanna erp

* Change serializing to not use il temporarily

* Make writing work with nullable types

* pushing

* check

* cuddling slaps HARD

* Add serialization priority test

* important fix

* a serialization thing

* serializer moment

* Add validation for some type serializers

* adds context

* moar context

* fixes

* Do the thing for appearance

* yoo lmao

* push haha pp

* Temporarily make copy delegate regular c# code

* Create deserialized component registry to handle not inheriting conflicting references

* YAML LINTER BABY

* ayes

* Fix sprite component norot not being default true like in latest master

* Remove redundant todos

* Add summary doc to every ISerializationManager method

* icon fixes

* Add skip hook argument to readers and copiers

* Merge fixes

* Fix ordering of arguments in read and copy reflection call

* Fix user interface components deserialization

* pew pew

* i am going to HECK

* Add MustUseReturnValue to copy-over methods

* Make serialization log calls use the same sawmill

* gamin

* Fix doc errors in ISerializationManager.cs

* goodbye brave soldier

* fixes

* WIP merge fixes and entity serialization

* aaaaaaaaaaaaaaa

* aaaaaaaaaaaaaaa

* adds inheritancebehaviour

* test/datafield fixes

* forgot that one

* adds more verbose validation

* This fixes the YAML hot reloading

* Replace yield break with Enumerable.Empty

* adds copiers

* aaaaaaaaaaaaa

* array fix
priority fix
misc fixes

* fix(?)

* fix.

* funny map serialization (wip)

* funny map serialization (wip)

* Add TODO

* adds proper info the validation

* Make yaml linter 5 times faster (~80% less execution time)

* Improves the error message for missing fields in the linter

* Include component name in unknown component type error node

* adds alwaysrelevant usa

* fixes mapsaving

* moved surpressor to analyzers proj

* warning cleanup & moves surpressor

* removes old msbuild targets

* Revert "Make yaml linter 5 times faster (~80% less execution time)"

This reverts commit 2ee4cc2.

* Add serialization to RobustServerSimulation and mock reflection methods
Fixes container tests

* Fix nullability warnings

* Improve yaml linter message feedback

* oops moment

* Add IEquatable, IComparable, ToString and operators to DataPosition
Rename it to NodeMark
Make it a readonly struct

* Remove try catch from enum parsing

* Make dependency management in serialization less bad

* Make dependencies an argument instead of a property on the serialization manager

* Clean up type serializers

* Improve validation messages and resourc epath checking

* Fix sprite error message

* reached perfection

Co-authored-by: Paul <[email protected]>
Co-authored-by: DrSmugleaf <[email protected]>
Co-authored-by: Vera Aguilera Puerto <[email protected]>

Author: 4 people

Robust.Analyzers/Diagnostics.cs

@@ -0,0 +1,10 @@
+using Microsoft.CodeAnalysis;
+
+namespace Robust.Generators
+{
+    public static class Diagnostics
+    {
+        public static SuppressionDescriptor MeansImplicitAssignment =>
+            new SuppressionDescriptor("RADC1000", "CS0649", "Marked as implicitly assigned.");
+    }
+}

Robust.Analyzers/MeansImplicitAssigmentSuppressor.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Immutable;
+using System.Linq;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Diagnostics;
+using Robust.Generators;
+
+namespace Robust.Analyzers
+{
+    [DiagnosticAnalyzer(LanguageNames.CSharp)]
+    public class MeansImplicitAssigmentSuppressor : DiagnosticSuppressor
+    {
+        const string MeansImplicitAssignmentAttribute = "Robust.Shared.MeansImplicitAssignmentAttribute";
+
+        public override void ReportSuppressions(SuppressionAnalysisContext context)
+        {
+            var implAttr = context.Compilation.GetTypeByMetadataName(MeansImplicitAssignmentAttribute);
+            foreach (var reportedDiagnostic in context.ReportedDiagnostics)
+            {
+                if(reportedDiagnostic.Id != Diagnostics.MeansImplicitAssignment.SuppressedDiagnosticId) continue;
+
+                var node = reportedDiagnostic.Location.SourceTree?.GetRoot(context.CancellationToken).FindNode(reportedDiagnostic.Location.SourceSpan);
+                if (node == null) continue;
+
+                var symbol = context.GetSemanticModel(reportedDiagnostic.Location.SourceTree).GetDeclaredSymbol(node);
+
+                if (symbol == null || !symbol.GetAttributes().Any(a =>
+                    a.AttributeClass?.GetAttributes().Any(attr =>
+                        SymbolEqualityComparer.Default.Equals(attr.AttributeClass, implAttr)) == true))
+                {
+                    continue;
+                }
+
+                context.ReportSuppression(Suppression.Create(
+                    Diagnostics.MeansImplicitAssignment,
+                    reportedDiagnostic));
+            }
+        }
+
+        public override ImmutableArray<SuppressionDescriptor> SupportedSuppressions => ImmutableArray.Create(Diagnostics.MeansImplicitAssignment);
+    }
+}

Robust.Client/Animations/Animation.cs

@@ -13,7 +13,7 @@ namespace Robust.Client.Animations
     /// <seealso cref="AnimationPlayerComponent"/>
     public sealed class Animation
     {
-        public readonly List<AnimationTrack> AnimationTracks = new();
+        public List<AnimationTrack> AnimationTracks { get; private set; } = new();
 
         public TimeSpan Length { get; set; }
     }

Robust.Client/Animations/AnimationTrackPlaySound.cs

@@ -15,7 +15,7 @@ public sealed class AnimationTrackPlaySound : AnimationTrack
         /// <summary>
         ///     A list of key frames for when to fire flicks.
         /// </summary>
-        public readonly List<KeyFrame> KeyFrames = new();
+        public List<KeyFrame> KeyFrames { get; private set; } = new();
 
         public override (int KeyFrameIndex, float FramePlayingTime) InitPlayback()
         {

Robust.Client/Animations/AnimationTrackProperty.cs

@@ -10,7 +10,7 @@ namespace Robust.Client.Animations
     /// </summary>
     public abstract class AnimationTrackProperty : AnimationTrack
     {
-        public readonly List<KeyFrame> KeyFrames = new();
+        public List<KeyFrame> KeyFrames { get; protected set; } = new();
 
         /// <summary>
         ///     How to interpolate values when between two keyframes.

Robust.Client/Animations/AnimationTrackSpriteFlick.cs

@@ -15,7 +15,7 @@ public sealed class AnimationTrackSpriteFlick : AnimationTrack
         /// <summary>
         ///     A list of key frames for when to fire flicks.
         /// </summary>
-        public readonly List<KeyFrame> KeyFrames = new();
+        public List<KeyFrame> KeyFrames { get; private set; } = new();
 
         // TODO: Should this layer key be per keyframe maybe?
         /// <summary>

Robust.Client/ClientIoC.cs

@@ -27,6 +27,8 @@
 using Robust.Shared.Players;
 using Robust.Shared.Prototypes;
 using Robust.Shared.Reflection;
+using Robust.Shared.Serialization;
+using Robust.Shared.Serialization.Manager;
 
 namespace Robust.Client
 {
@@ -101,7 +103,6 @@ public static void RegisterIoC(GameController.DisplayMode mode)
             IoCManager.Register<IViewVariablesManagerInternal, ViewVariablesManager>();
             IoCManager.Register<IClientConGroupController, ClientConGroupController>();
             IoCManager.Register<IScriptClient, ScriptClient>();
-            //IoCManager.Register<IXamlCompiler, XamlCompiler>();
         }
     }
 }

Robust.Client/Console/Commands/Debug.cs

@@ -12,7 +12,6 @@
 using Robust.Client.Debugging;
 using Robust.Client.Graphics;
 using Robust.Client.ResourceManagement;
-using Robust.Client.ResourceManagement.ResourceTypes;
 using Robust.Client.UserInterface;
 using Robust.Client.UserInterface.Controls;
 using Robust.Client.UserInterface.CustomControls;

Robust.Client/GameController.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.IO;
 using System.Net;
 using System.Threading.Tasks;
@@ -27,6 +27,7 @@
 using Robust.Shared.Network;
 using Robust.Shared.Prototypes;
 using Robust.Shared.Serialization;
+using Robust.Shared.Serialization.Manager;
 using Robust.Shared.Timing;
 using Robust.Shared.Utility;
 
@@ -44,7 +45,7 @@ internal sealed partial class GameController : IGameControllerInternal
         [Dependency] private readonly IUserInterfaceManagerInternal _userInterfaceManager = default!;
         [Dependency] private readonly IBaseClient _client = default!;
         [Dependency] private readonly IInputManager _inputManager = default!;
-        [Dependency] private readonly IClientConsoleHost _consoleHost = default!;
+        [Dependency] private readonly IClientConsoleHost _console = default!;
         [Dependency] private readonly ITimerManager _timerManager = default!;
         [Dependency] private readonly IClientEntityManager _entityManager = default!;
         [Dependency] private readonly IPlacementManager _placementManager = default!;
@@ -154,6 +155,8 @@ public bool Startup(Func<ILogHandler>? logHandlerFactory = null)
                 _configurationManager.LoadCVarsFromAssembly(loadedModule);
             }
 
+            IoCManager.Resolve<ISerializationManager>().Initialize();
+
             // Call Init in game assemblies.
             _modLoader.BroadcastRunLevel(ModRunLevel.PreInit);
             _modLoader.BroadcastRunLevel(ModRunLevel.Init);
@@ -163,7 +166,7 @@ public bool Startup(Func<ILogHandler>? logHandlerFactory = null)
             IoCManager.Resolve<INetConfigurationManager>().SetupNetworking();
             _serializer.Initialize();
             _inputManager.Initialize();
-            _consoleHost.Initialize();
+            _console.Initialize();
             _prototypeManager.Initialize();
             _prototypeManager.LoadDirectory(new ResourcePath(@"/Prototypes/"));
             _prototypeManager.Resync();

Robust.Client/GameController/GameController.IoC.cs

@@ -17,7 +17,6 @@ private static void InitIoC(DisplayMode mode)
             RegisterReflection();
         }
 
-
         internal static void RegisterReflection()
         {
             // Gets a handle to the shared and the current (client) dll.

Robust.Client/GameObjects/ClientComponentFactory.cs

@@ -54,7 +54,6 @@ public ClientComponentFactory()
 
 #if DEBUG
             Register<DebugExceptionOnAddComponent>();
-            Register<DebugExceptionExposeDataComponent>();
             Register<DebugExceptionInitializeComponent>();
             Register<DebugExceptionStartupComponent>();
 #endif

Robust.Client/GameObjects/Components/Appearance/AppearanceComponent.cs

@@ -4,8 +4,7 @@
 using Robust.Shared.GameObjects;
 using Robust.Shared.IoC;
 using Robust.Shared.Reflection;
-using Robust.Shared.Serialization;
-using Robust.Shared.Utility;
+using Robust.Shared.Serialization.Manager.Attributes;
 using Robust.Shared.ViewVariables;
 using YamlDotNet.RepresentationModel;
 
@@ -15,13 +14,11 @@ public sealed class AppearanceComponent : SharedAppearanceComponent
     {
         [ViewVariables]
         private Dictionary<object, object> data = new();
+
         [ViewVariables]
+        [DataField("visuals")]
         internal List<AppearanceVisualizer> Visualizers = new();
 
-        [Dependency] private readonly IReflectionManager _reflectionManager = default!;
-
-        private static bool _didRegisterSerializer;
-
         [ViewVariables]
         private bool _appearanceDirty;
 
@@ -107,18 +104,6 @@ internal void UnmarkDirty()
             _appearanceDirty = false;
         }
 
-        public override void ExposeData(ObjectSerializer serializer)
-        {
-            if (!_didRegisterSerializer)
-            {
-                YamlObjectSerializer.RegisterTypeSerializer(typeof(AppearanceVisualizer),
-                    new VisualizerTypeSerializer(_reflectionManager));
-                _didRegisterSerializer = true;
-            }
-
-            serializer.DataFieldCached(ref Visualizers, "visuals", new List<AppearanceVisualizer>());
-        }
-
         public override void Initialize()
         {
             base.Initialize();
@@ -131,78 +116,6 @@ public override void Initialize()
             MarkDirty();
         }
 
-        class VisualizerTypeSerializer : YamlObjectSerializer.TypeSerializer
-        {
-            private readonly IReflectionManager _reflectionManager;
-
-            public VisualizerTypeSerializer(IReflectionManager reflectionManager)
-            {
-                _reflectionManager = reflectionManager;
-            }
-
-            public override object NodeToType(Type type, YamlNode node, YamlObjectSerializer serializer)
-            {
-                var mapping = (YamlMappingNode) node;
-                var nodeType = mapping.GetNode("type");
-                switch (nodeType.AsString())
-                {
-                    case SpriteLayerToggle.NAME:
-                        var keyString = mapping.GetNode("key").AsString();
-                        object key;
-                        if (_reflectionManager.TryParseEnumReference(keyString, out var @enum))
-                        {
-                            key = @enum;
-                        }
-                        else
-                        {
-                            key = keyString;
-                        }
-
-                        var layer = mapping.GetNode("layer").AsInt();
-                        return new SpriteLayerToggle(key, layer);
-
-                    default:
-                        var visType = _reflectionManager.LooseGetType(nodeType.AsString());
-                        if (!typeof(AppearanceVisualizer).IsAssignableFrom(visType))
-                        {
-                            throw new InvalidOperationException();
-                        }
-
-                        var vis = (AppearanceVisualizer) Activator.CreateInstance(visType)!;
-                        vis.LoadData(mapping);
-                        return vis;
-                }
-            }
-
-            public override YamlNode TypeToNode(object obj, YamlObjectSerializer serializer)
-            {
-                switch (obj)
-                {
-                    case SpriteLayerToggle spriteLayerToggle:
-                        YamlScalarNode key;
-                        if (spriteLayerToggle.Key is Enum)
-                        {
-                            var name = spriteLayerToggle.Key.GetType().FullName;
-                            key = new YamlScalarNode($"{name}.{spriteLayerToggle.Key}");
-                        }
-                        else
-                        {
-                            key = new YamlScalarNode(spriteLayerToggle.Key.ToString());
-                        }
-
-                        return new YamlMappingNode
-                        {
-                            {new YamlScalarNode("type"), new YamlScalarNode(SpriteLayerToggle.NAME)},
-                            {new YamlScalarNode("key"), key},
-                            {new YamlScalarNode("layer"), new YamlScalarNode(spriteLayerToggle.SpriteLayer.ToString())},
-                        };
-                    default:
-                        // TODO: A proper way to do serialization here.
-                        // I can't use the ExposeData system here since that's specific to entity serializers.
-                        return new YamlMappingNode();
-                }
-            }
-        }
 
         internal class SpriteLayerToggle : AppearanceVisualizer
         {
@@ -223,15 +136,9 @@ public SpriteLayerToggle(object key, int spriteLayer)
     ///     Handles the visualization of data inside of an appearance component.
     ///     Implementations of this class are NOT bound to a specific entity, they are flyweighted across multiple.
     /// </summary>
+    [ImplicitDataDefinitionForInheritors]
     public abstract class AppearanceVisualizer
     {
-        /// <summary>
-        ///     Load data from the prototype declaring this visualizer, to configure settings and such.
-        /// </summary>
-        public virtual void LoadData(YamlMappingNode node)
-        {
-        }
-
         /// <summary>
         ///     Initializes an entity to be managed by this appearance controller.
         ///     DO NOT assume this is your only entity. Visualizers are shared.

Robust.Client/GameObjects/Components/Eye/EyeComponent.cs

@@ -3,7 +3,9 @@
 using Robust.Shared.IoC;
 using Robust.Shared.Map;
 using Robust.Shared.Maths;
+using Robust.Shared.Prototypes;
 using Robust.Shared.Serialization;
+using Robust.Shared.Serialization.Manager.Attributes;
 using Robust.Shared.ViewVariables;
 
 namespace Robust.Client.GameObjects
@@ -20,7 +22,9 @@ public class EyeComponent : SharedEyeComponent
 
         // Horrible hack to get around ordering issues.
         private bool _setCurrentOnInitialize;
-        private bool _setDrawFovOnInitialize;
+        [DataField("drawFov")]
+        private bool _setDrawFovOnInitialize = true;
+        [DataField("zoom")]
         private Vector2 _setZoomOnInitialize = Vector2.One/2f;
         private Vector2 _offset = Vector2.Zero;
 
@@ -157,15 +161,6 @@ public override void OnRemove()
             Current = false;
         }
 
-        /// <inheritdoc />
-        public override void ExposeData(ObjectSerializer serializer)
-        {
-            base.ExposeData(serializer);
-
-            serializer.DataFieldCached(ref _setZoomOnInitialize, "zoom", Vector2.One/2f);
-            serializer.DataFieldCached(ref _setDrawFovOnInitialize, "drawFov", true);
-        }
-
         /// <summary>
         /// Updates the Eye of this entity with the transform position. This has to be called every frame to
         /// keep the view following the entity.