Feature/split serialize

.github/workflows/extended_checks.yml

@@ -181,6 +181,10 @@ jobs:
       run: |
         set -eux
         $BIN/daslang_static _dasroot_/dastest/dastest.das -- --color --failures-only --test ./tests
+    - name: "Test ser/deser"
+      run: |
+        $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./tests --ser serialized.bin
+        $BIN/daslang _dasroot_/dastest/dastest.das -- --color --failures-only --test ./tests --deser serialized.bin
 
     - name: "Run self-binder (bind_clangbind.das)"
       if: matrix.target == 'linux'

CMakeLists.txt

@@ -654,6 +654,7 @@ SOURCE_GROUP_FILES("vecmath" VECMATH_SRC)
 
 SET(AST_SRC
 src/ast/ast.cpp
+src/ast/ast_dispatch.cpp
 src/ast/ast_interop.cpp
 src/ast/ast_tls.cpp
 src/ast/ast_visitor.cpp

daslib/aot_cpp.das

@@ -3990,11 +3990,12 @@ def public compile_and_simulate(input : string, var access; var mg : ModuleGroup
     set_aot();
     compile_file(input, access, mg, cop) $(ok; var program : smart_ptr<Program>; issues) {
         if (!ok) {
-            print("failed to compile {input}\n{issues}\n")
+            to_log(LOG_ERROR, "failed to compile {input}\n{issues}\n")
             return
         }
         simulate(program) $(sok; var pctx : smart_ptr<Context>; serrors) {
             if (!sok) {
+                to_log(LOG_ERROR, "failed to simulate {input}\n{serrors}\n")
                 panic("Failed to simulate {serrors}")
             }
             blk(program, pctx)

dastest/dastest.das

@@ -156,6 +156,13 @@ def serialize_path(ctx : SuiteCtx, files : array<string>, out_file : string) {
         var inscope access <- make_file_access(ctx.projectPath)
         access |> add_file_access_root("dastest", ctx.dastestRoot)
         for (file in files) {
+            // Tests whose basename starts with cant_, failed_, or invalid_
+            // are expected-failure compile tests anywhere in the tests tree;
+            // they have nothing to serialize.
+            let base = base_name(file)
+            if (base |> starts_with("cant_") || base |> starts_with("failed_") || base |> starts_with("invalid_")) {
+                continue
+            }
             using() $(var mg : ModuleGroup) {
                 using() $(var cop : CodeOfPolicies) {
                     cop.aot_module = true

include/daScript/ast/ast.h

@@ -25,6 +25,7 @@
 namespace das
 {
     struct AstSerializer;
+    class Visitor;
 
     class Function;
     typedef Function * FunctionPtr;
@@ -691,7 +692,7 @@ namespace das
         virtual Expression * tail() { return this; }
         virtual bool swap_tail ( Expression *, Expression * ) { return false; }
         virtual uint32_t getEvalFlags() const { return 0; }
-        virtual void serialize ( AstSerializer & ser );
+        virtual void dispatch ( Visitor & vis );
         virtual void gc_collect ( gc_root * target, gc_root * from );
         LineInfo    at;
         TypeDeclPtr type = nullptr;
@@ -745,7 +746,7 @@ namespace das
         virtual bool rtti_isConstant() const override { return true; }
         template <typename QQ> QQ & cvalue() { return *((QQ *)&value); }
         template <typename QQ> const QQ & cvalue() const { return *((const QQ *)&value); }
-        virtual void serialize ( AstSerializer & ser ) override;
+        virtual void dispatch ( Visitor & vis ) override;
         Type    baseType = Type::none;
         vec4f   value = v_zero();
         bool    foldedNonConst = false;

include/daScript/ast/ast_serializer.h

@@ -64,6 +64,30 @@ namespace das {
         }
     };
 
+    // Composite key for serializer maps. Pairs a raw pointer with a per-
+    // AstSerializer epoch so reused addresses don't collide with prior entries.
+    // Hoisted to namespace das so daslang_hash can be specialized before the
+    // map instantiations inside AstSerializer.
+    struct SerializeNodeId {
+        void *  ptr = nullptr;
+        size_t  epoch = 0;
+        bool operator == ( const SerializeNodeId & o ) const noexcept {
+            return ptr == o.ptr && epoch == o.epoch;
+        }
+        bool operator != ( const SerializeNodeId & o ) const noexcept {
+            return !(*this == o);
+        }
+    };
+
+    template <>
+    struct daslang_hash<SerializeNodeId, void> {
+        size_t operator () ( const SerializeNodeId & s ) const noexcept {
+            const size_t pmix = (reinterpret_cast<size_t>(s.ptr) >> 4)
+                              * size_t(0x9E3779B97F4A7C15ull);
+            return pmix ^ (s.epoch * size_t(0xBF58476D1CE4E5B9ull));
+        }
+    };
+
     struct DAS_API AstSerializer {
         ~AstSerializer ();
         AstSerializer ( SerializationStorage * storage, bool isWriting );
@@ -90,26 +114,32 @@ namespace das {
         das_hash_set<FileInfo*>   doNotDelete;
     // profile data
         uint64_t totMacroTime = 0;
+    // Per-program epoch for SerializeNodeId. Bumped at the start of each
+    // serialize/deserialize call so reused pointer addresses across program
+    // boundaries don't collide with prior map entries on this persistent
+    // serializer. Must be initialized — uninitialized epoch produces garbage
+    // keys and silent map collisions.
+        size_t                                      epoch = 0;
     // pointers
-        das_hash_map<uint64_t, ExprBlock*>          exprBlockMap;
+        das_hash_map<SerializeNodeId, ExprBlock*>          exprBlockMap;
         using DataOffset = uint64_t;
-        das_hash_map<FileInfo*, DataOffset>         writingFileInfoMap;
-        das_hash_map<DataOffset, FileInfo*>         readingFileInfoMap;
-        das_hash_map<uint64_t, FileAccess*>         fileAccessMap;
+        das_hash_map<FileInfo*, DataOffset>                writingFileInfoMap;
+        das_hash_map<DataOffset, FileInfo*>                readingFileInfoMap;
+        das_hash_map<SerializeNodeId, FileAccess*>         fileAccessMap;
     // smart pointers
-        das_hash_map<uint64_t, MakeFieldDeclPtr>    smartMakeFieldDeclMap;
-        das_hash_map<uint64_t, EnumerationPtr>      smartEnumerationMap;
-        das_hash_map<uint64_t, StructurePtr>        smartStructureMap;
-        das_hash_map<uint64_t, VariablePtr>         smartVariableMap;
-        das_hash_map<uint64_t, FunctionPtr>         smartFunctionMap;
-        das_hash_map<uint64_t, MakeStructPtr>       smartMakeStructMap;
-        das_hash_map<uint64_t, TypeDeclPtr>         smartTypeDeclMap;
+        das_hash_map<SerializeNodeId, MakeFieldDeclPtr>    smartMakeFieldDeclMap;
+        das_hash_map<SerializeNodeId, EnumerationPtr>      smartEnumerationMap;
+        das_hash_map<SerializeNodeId, StructurePtr>        smartStructureMap;
+        das_hash_map<SerializeNodeId, VariablePtr>         smartVariableMap;
+        das_hash_map<SerializeNodeId, FunctionPtr>         smartFunctionMap;
+        das_hash_map<SerializeNodeId, MakeStructPtr>       smartMakeStructMap;
+        das_hash_map<SerializeNodeId, TypeDeclPtr>         smartTypeDeclMap;
     // refs
-        vector<pair<ExprBlock**,uint64_t>>          blockRefs;
-        vector<pair<Function **,uint64_t>>          functionRefs;
-        vector<pair<Variable **,uint64_t>>          variableRefs;
-        vector<pair<Structure **,uint64_t>>         structureRefs;
-        vector<pair<Enumeration **,uint64_t>>       enumerationRefs;
+        vector<pair<ExprBlock**,SerializeNodeId>>          blockRefs;
+        vector<pair<Function **,SerializeNodeId>>          functionRefs;
+        vector<pair<Variable **,SerializeNodeId>>          variableRefs;
+        vector<pair<Structure **,SerializeNodeId>>         structureRefs;
+        vector<pair<Enumeration **,SerializeNodeId>>       enumerationRefs;
         // fieldRefs tuple contains: fieldptr, module, structname, fieldname
         vector<tuple<Structure::FieldDeclarationRef*, Module *, string, string>>       fieldRefs;
         // parseModule tuple contains: moduleName, mtime, thisModule, thisModule
@@ -185,6 +215,7 @@ namespace das {
         AstSerializer & operator << ( CaptureEntry & entry );
         AstSerializer & operator << ( MakeFieldDeclPtr & ptr );
         AstSerializer & operator << ( MakeStructPtr & ptr );
+        AstSerializer & operator << ( SerializeNodeId & value );
    // Top-level
         AstSerializer & operator << ( Module & module );
         AstSerializer & serializeModule ( Module & module, bool already_exists );
@@ -194,9 +225,6 @@ namespace das {
         void serializeProgram ( ProgramPtr program, ModuleGroup & libGroup ) noexcept;
         bool serializeScript ( ProgramPtr program ) noexcept;
 
-        template<typename T>
-        void serializeSmartPtr( smart_ptr<T> & obj, das_hash_map<uint64_t, smart_ptr<T>> & objMap );
-
         template <uint64_t n>
         AstSerializer& operator << ( int (&value)[n] ) {
             serialize(value, n * sizeof(int)); return *this;
@@ -248,10 +276,10 @@ namespace das {
         void writeIdentifications ( Variable * & ptr );
         void writeIdentifications ( TypeInfoMacro * & ptr );
 
-        void fillOrPatchLater ( Function * & func, uint64_t id );
-        void fillOrPatchLater ( Enumeration * & ptr, uint64_t id );
-        void fillOrPatchLater ( Structure * & ptr, uint64_t id );
-        void fillOrPatchLater ( Variable * & ptr, uint64_t id );
+        void fillOrPatchLater ( Function * & func, SerializeNodeId id );
+        void fillOrPatchLater ( Enumeration * & ptr, SerializeNodeId id );
+        void fillOrPatchLater ( Structure * & ptr, SerializeNodeId id );
+        void fillOrPatchLater ( Variable * & ptr, SerializeNodeId id );
 
         auto readModuleAndName () -> pair<Module *, string>;
         auto readModuleAndNameHash () -> pair<Module *, uint64_t>;
@@ -262,6 +290,8 @@ namespace das {
         void findExternal ( Variable * & ptr );
         void findExternal ( TypeInfoMacro * & ptr );
 
+        SerializeNodeId getSerializeId(void *ptr) { return {ptr, epoch}; }
+
         template <typename EnumType>
         void serialize_small_enum ( EnumType & baseType ) {
             if ( writing ) {
@@ -290,6 +320,7 @@ namespace das {
     // Opaque handle to expose serializer to daslang.
     struct AstSerializerState {
         unique_ptr<SerializationStorageVector> storage;
+        unique_ptr<AstSerializer> serializer;
     };
 
     // Create a writing serializer.

include/daScript/ast/ast_visitor.h

@@ -376,13 +376,23 @@ namespace das {
         return llk == this ? vis.visit(static_cast<It *>(this)) : llk;
     }
 
+    template <typename It, typename SimNodeT, bool first>
+    void ExprTableKeysOrValues<It,SimNodeT,first>::dispatch ( Visitor & vis ) {
+        vis.preVisit(static_cast<ExprLooksLikeCall*>(this));
+    }
+
     template <typename It, typename SimNodeT>
     ExpressionPtr ExprArrayCallWithSizeOrIndex<It,SimNodeT>::visit(Visitor & vis) {
         vis.preVisit(static_cast<It *>(this));
         auto llk = ExprLooksLikeCall::visit(vis);
         return llk == this ? vis.visit(static_cast<It *>(this)) : llk;
     }
 
+    template <typename It, typename SimNodeT>
+    void ExprArrayCallWithSizeOrIndex<It,SimNodeT>::dispatch ( Visitor & vis ) {
+        vis.preVisit(static_cast<ExprLooksLikeCall*>(this));
+    }
+
     template <typename TT, typename ExprConstExt>
     ExpressionPtr ExprConstT<TT,ExprConstExt>::visit(Visitor & vis) {
         vis.preVisit((ExprConst*)this);

src/ast/ast_dispatch.cpp

@@ -0,0 +1,91 @@
+#include "daScript/misc/platform.h"
+
+#include "daScript/ast/ast.h"
+#include "daScript/ast/ast_expressions.h"
+#include "daScript/ast/ast_visitor.h"
+
+// Per-class virtual shim for SerializeVisitor-style dispatch:
+// static-type-aware single call to the matching Visitor::preVisit(ExprXxx*) overload,
+// with no child walk and no post-visit. Abstract intermediate classes and templates
+// without a matching preVisit overload fall through to preVisitExpression.
+
+namespace das
+{
+
+    void Expression::dispatch ( Visitor & vis ) {
+        vis.preVisitExpression(this);
+    }
+
+    // concrete Expression subclasses -> preVisit(ExprXxx*)
+    void ExprReader::dispatch            ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprLabel::dispatch             ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprGoto::dispatch              ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprRef2Value::dispatch         ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprRef2Ptr::dispatch           ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprPtr2Ref::dispatch           ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprAddr::dispatch              ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprNullCoalescing::dispatch    ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprDelete::dispatch            ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprAt::dispatch                ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprSafeAt::dispatch            ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprBlock::dispatch             ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprVar::dispatch               ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprTag::dispatch               ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprField::dispatch             ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprSafeAsVariant::dispatch     ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprSwizzle::dispatch           ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprSafeField::dispatch         ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprLooksLikeCall::dispatch     ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprCallMacro::dispatch         ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprOp1::dispatch               ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprOp2::dispatch               ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprCopy::dispatch              ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprMove::dispatch              ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprClone::dispatch             ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprOp3::dispatch               ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprTryCatch::dispatch          ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprReturn::dispatch            ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprConstPtr::dispatch          ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprConstEnumeration::dispatch  ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprConstBitfield::dispatch     ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprConstString::dispatch       ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprStringBuilder::dispatch     ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprLet::dispatch               ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprFor::dispatch               ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprUnsafe::dispatch            ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprWhile::dispatch             ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprWith::dispatch              ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprAssume::dispatch            ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprMakeBlock::dispatch         ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprMakeGenerator::dispatch     ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprYield::dispatch             ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprInvoke::dispatch            ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprAssert::dispatch            ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprQuote::dispatch             ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprTypeInfo::dispatch          ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprIs::dispatch                ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprAscend::dispatch            ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprCast::dispatch              ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprNew::dispatch               ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprCall::dispatch              ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprIfThenElse::dispatch        ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprNamedCall::dispatch         ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprMakeStruct::dispatch        ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprMakeVariant::dispatch       ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprMakeArray::dispatch         ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprMakeTuple::dispatch         ( Visitor & vis ) { vis.preVisit(this); }
+    void ExprArrayComprehension::dispatch( Visitor & vis ) { vis.preVisit(this); }
+    void ExprTypeDecl::dispatch          ( Visitor & vis ) { vis.preVisit(this); }
+
+    // abstract intermediate classes -> route to nearest concrete preVisit overload
+    void ExprOp::dispatch                ( Visitor & vis ) { vis.preVisit(static_cast<ExprLooksLikeCall*>(this)); }
+    void ExprCallFunc::dispatch          ( Visitor & vis ) { vis.preVisit(static_cast<ExprLooksLikeCall*>(this)); }
+    void ExprMakeLocal::dispatch         ( Visitor & vis ) { vis.preVisitExpression(this); }
+
+    void ExprConst::dispatch             ( Visitor & vis ) { vis.preVisit(this); }
+
+    // ExprTableKeysOrValues / ExprArrayCallWithSizeOrIndex dispatch live in
+    // ast_visitor.h alongside their visit() templates so every TU that
+    // instantiates a specialization picks up the definition.
+
+}