Add support for module class documentation and enhance synchronizatoin logic

README.md

@@ -37,10 +37,14 @@ Exactly one entry must be marked as the latest stable version and exactly one as
 
 ### Class reference synchronization
 
-The Classes section is generated from the XML class reference in the Redot Engine repository. At startup, the application loads valid cached snapshots and starts a shallow, partial Git checkout of `doc/classes` for every branch configured in `Versions.json` in the background. Snapshots are cached under `Redot-Documentation/App_Data/class-docs` and checked for upstream changes every 24 hours. If Git is temporarily unavailable, the application continues with the last valid cache; without a cache, class documentation remains unavailable until a synchronization succeeds.
+The Classes section is generated from the XML class reference in the Redot Engine repository. At startup, the application loads valid cached snapshots and starts a shallow, partial Git checkout of `doc/classes/*.xml` and `modules/*/doc_classes/*.xml` for every branch configured in `Versions.json` in the background. Snapshots are cached under `Redot-Documentation/App_Data/class-docs` and checked for upstream changes every 24 hours. If Git is temporarily unavailable, the application continues with the last valid cache; without a cache, class documentation remains unavailable until a synchronization succeeds.
 
 The repository URL, source path, cache path, refresh interval, and Git timeout are configured in the `ClassDocumentation` section of `Redot-Documentation/appsettings.json`. Set `ClassDocumentation__Enabled=false` to disable synchronization for an offline development session. Git must be installed on the host.
 
+Module class documentation is discovered automatically using wildcard sparse checkout; no per-module configuration or engine build is required. Core and module classes share the existing Classes index, search, and URLs. Only immediate `doc_classes/*.xml` files under each module are included, not unrelated XML or module source code. All selected XML is validated together before publication; duplicate class names or invalid module XML retain the last valid snapshot.
+
+Existing core-only caches remain available while a one-time staged refresh adds module documentation, even if the engine commit is unchanged. Cache metadata records the selection revision and documentation directories so a missing module documentation directory triggers a repair. `ClassDocumentation:RepositoryPath` continues to configure the core XML directory; module discovery always uses `modules/*/doc_classes`.
+
 `/health/class-docs` reports the active commit and class count for each documentation version and returns HTTP 503 if any configured version has no usable snapshot.
 
 ---

Redot-Documentation-Tests/ClassDocumentationParserTests.cs

@@ -128,6 +128,22 @@ public void ParseFile_PreservesEmptyStatusAttributes()
         Assert.Equal(string.Empty, entry.Deprecated);
     }
 
+    [Fact]
+    public void ParseDirectories_ReportsBothFilesForDuplicateClasses()
+    {
+        string core = Path.Combine(_directory, "core");
+        string module = Path.Combine(_directory, "module");
+        Directory.CreateDirectory(core);
+        Directory.CreateDirectory(module);
+        string first = Path.Combine(core, "Node.xml");
+        string second = Path.Combine(module, "Node.xml");
+        File.WriteAllText(first, "<class name=\"Node\" />");
+        File.WriteAllText(second, "<class name=\"node\" />");
+        var error = Assert.Throws<InvalidDataException>(() => new ClassDocumentationParser().ParseDirectories([core, module]));
+        Assert.Contains(first, error.Message);
+        Assert.Contains(second, error.Message);
+    }
+
     public void Dispose()
     {
         if (Directory.Exists(_directory))

Redot-Documentation-Tests/ClassDocumentationSyncServiceTests.cs

@@ -85,6 +85,34 @@ public async Task StartAsync_DoesNotPromoteOrPublishMalformedClassData(string xm
         Assert.False(catalog.TryGetSnapshot(manager.NextPrereleaseVersion.Slug, out _));
     }
 
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task StartAsync_ValidatesCoreAndModulesBeforePublishingTogether(bool invalidModule)
+    {
+        var manager = CreateVersionManager();
+        string core = Path.Combine(_contentRootPath, "core");
+        string module = Path.Combine(_contentRootPath, "module");
+        Directory.CreateDirectory(core);
+        Directory.CreateDirectory(module);
+        File.WriteAllText(Path.Combine(core, "Node.xml"), "<class name=\"Node\" />");
+        File.WriteAllText(Path.Combine(module, "ModuleClass.xml"), invalidModule ? "<invalid />" : "<class name=\"ModuleClass\" />");
+        var source = new TestSource(core) { DocumentationPaths = [core, module] };
+        var catalog = new ClassDocumentationCatalog();
+        using var service = CreateService(manager, source, catalog);
+
+        await service.StartAsync(CancellationToken.None);
+        await service.StopAsync(CancellationToken.None);
+
+        Assert.Equal(invalidModule ? 0 : manager.Versions.Count, source.PromotionCount);
+        foreach (var version in manager.Versions)
+        {
+            Assert.Equal(!invalidModule, catalog.TryGetSnapshot(version.Slug, out var snapshot));
+            if (!invalidModule)
+                Assert.Equal(["ModuleClass", "Node"], snapshot!.Classes.Keys.Order(StringComparer.Ordinal));
+        }
+    }
+
     private static ClassDocumentationSyncService CreateService(
         VersionManagerService manager, IClassDocumentationSource source, ClassDocumentationCatalog catalog)
         => new(manager, source, new ClassDocumentationParser(), catalog,
@@ -94,6 +122,7 @@ private static ClassDocumentationSyncService CreateService(
     private sealed class TestSource(string classesPath) : IClassDocumentationSource
     {
         public bool DenyCacheAccess { get; init; }
+        public IReadOnlyList<string>? DocumentationPaths { get; init; }
         public int PrepareCount { get; private set; }
         public int PromotionCount { get; private set; }
 
@@ -109,7 +138,7 @@ public Task<ClassDocumentationCheckout> PrepareAsync(DocumentationVersion versio
         {
             PrepareCount++;
             return Task.FromResult(ClassDocumentationCheckout.Pending(version, "new-commit", classesPath, classesPath,
-                Path.Combine(classesPath, "unused-staging"), () => PromotionCount++));
+                Path.Combine(classesPath, "unused-staging"), () => PromotionCount++, DocumentationPaths));
         }
     }
 

Redot-Documentation-Tests/GitClassDocumentationSourceTests.cs

@@ -119,6 +119,7 @@ public async Task InterruptedPromotion_RecoversMatchingCheckoutAndMetadata(bool
 
         File.WriteAllText(Path.Combine(sourcePath, "doc", "classes", "Node.xml"),
             "<class name=\"Node\"><brief_description>Updated node.</brief_description></class>");
+        AddModule(sourcePath, "alpha", "Alpha");
         RunGit(sourcePath, "add", ".");
         RunGit(sourcePath, "commit", "-m", "Update class docs");
         RunGit(sourcePath, "push", remote, "master");
@@ -145,7 +146,9 @@ public async Task InterruptedPromotion_RecoversMatchingCheckoutAndMetadata(bool
             using (recovered)
             {
                 Assert.Equal(expectedCommit, recovered!.CommitSha);
-                var node = Assert.Single(new ClassDocumentationParser().ParseDirectory(recovered.ClassDocumentationPath)).Value;
+                var classes = new ClassDocumentationParser().ParseDirectories(recovered.ClassDocumentationPaths);
+                Assert.Equal(phase >= 2, classes.ContainsKey("Alpha"));
+                var node = classes["Node"];
                 Assert.Equal(phase >= 2 ? "Updated node." : "A node.", node.BriefDescription);
             }
             using var metadata = JsonDocument.Parse(File.ReadAllText(metadataPath));
@@ -174,6 +177,175 @@ public async Task TryGetCurrent_SupportsLegacyMetadataButDoesNotFallBackFromCorr
         Assert.False(source.TryGetCurrent(version, out _));
     }
 
+    [Fact]
+    public async Task Modules_AreDiscoveredAcrossUpdatesAndMissingCachedDirectoriesAreRepaired()
+    {
+        string remote = CreateRemoteRepository(out string work);
+        AddModule(work, "alpha", "Alpha");
+        File.WriteAllText(Path.Combine(work, "modules", "alpha", "source.cpp"), "not documentation");
+        File.WriteAllText(Path.Combine(work, "modules", "alpha", "unrelated.xml"), "<not-a-class />");
+        CommitAndPush(work, remote);
+        var source = CreateSource(remote);
+        var version = CreateVersion();
+        var parser = new ClassDocumentationParser();
+        using (var candidate = await source.PrepareAsync(version, CancellationToken.None))
+        {
+            Assert.Equal(2, candidate.ClassDocumentationPaths.Count);
+            Assert.Equal(2, parser.ParseDirectories(candidate.ClassDocumentationPaths).Count);
+            Assert.False(File.Exists(Path.Combine(candidate.RepositoryPath, "modules", "alpha", "source.cpp")));
+            Assert.False(File.Exists(Path.Combine(candidate.RepositoryPath, "modules", "alpha", "unrelated.xml")));
+            candidate.Promote();
+        }
+        Assert.True(source.TryGetCurrent(version, out var cached));
+        using (cached)
+            Assert.Contains("Alpha", parser.ParseDirectories(cached!.ClassDocumentationPaths).Keys);
+
+        Directory.Delete(Path.Combine(_root, "cache", "latest", "repository", "modules", "alpha", "doc_classes"), true);
+        Assert.False(source.TryGetCurrent(version, out _));
+        using (var repair = await source.PrepareAsync(version, CancellationToken.None))
+        {
+            Assert.True(repair.IsPending);
+            Assert.Contains("Alpha", parser.ParseDirectories(repair.ClassDocumentationPaths).Keys);
+            repair.Promote();
+        }
+
+        Directory.Delete(Path.Combine(work, "modules", "alpha"), true);
+        AddModule(work, "new_module", "NewModule");
+        CommitAndPush(work, remote);
+        using var updated = await source.PrepareAsync(version, CancellationToken.None);
+        var classes = parser.ParseDirectories(updated.ClassDocumentationPaths);
+        Assert.Contains("NewModule", classes.Keys);
+        Assert.DoesNotContain("Alpha", classes.Keys);
+    }
+
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task CurrentSelection_WithOmittedModuleDirectory_IsRejectedAndRepaired(bool hasSecondModule)
+    {
+        string remote = CreateRemoteRepository(out string work);
+        AddModule(work, "alpha", "Alpha");
+        if (hasSecondModule)
+            AddModule(work, "beta", "Beta");
+        CommitAndPush(work, remote);
+        var source = CreateSource(remote);
+        var version = CreateVersion();
+        using (var initial = await source.PrepareAsync(version, CancellationToken.None))
+            initial.Promote();
+
+        string repository = Path.Combine(_root, "cache", "latest", "repository");
+        string marker = Path.Combine(repository, ".redot-class-doc-sync.json");
+        var metadata = System.Text.Json.Nodes.JsonNode.Parse(File.ReadAllText(marker))!.AsObject();
+        int revision = metadata["SelectionRevision"]!.GetValue<int>();
+        var directories = metadata["DocumentationDirectories"]!.AsArray();
+        directories.Remove(directories.Single(path => path!.GetValue<string>() == "modules/alpha/doc_classes"));
+        File.WriteAllText(marker, metadata.ToJsonString());
+
+        Assert.True(revision > 0);
+        Assert.True(Directory.Exists(Path.Combine(repository, "modules", "alpha", "doc_classes")));
+        Assert.False(source.TryGetCurrent(version, out _));
+
+        using var repair = await source.PrepareAsync(version, CancellationToken.None);
+        Assert.True(repair.IsPending);
+        Assert.Equal(metadata["CommitSha"]!.GetValue<string>(), repair.CommitSha);
+        Assert.Equal(hasSecondModule ? 3 : 2, repair.ClassDocumentationPaths.Count);
+        Assert.Contains("Alpha", new ClassDocumentationParser().ParseDirectories(repair.ClassDocumentationPaths).Keys);
+        repair.Promote();
+        using var current = await source.PrepareAsync(version, CancellationToken.None);
+        Assert.False(current.IsPending);
+        var repairedMetadata = System.Text.Json.Nodes.JsonNode.Parse(File.ReadAllText(marker))!;
+        Assert.Equal(revision, repairedMetadata["SelectionRevision"]!.GetValue<int>());
+    }
+
+    [Fact]
+    public async Task LegacyCoreOnlyCache_IsReadableButRefreshedAtTheSameCommit()
+    {
+        string remote = CreateRemoteRepository(out string work);
+        AddModule(work, "alpha", "Alpha");
+        CommitAndPush(work, remote);
+        var source = CreateSource(remote);
+        var version = CreateVersion();
+        using (var initial = await source.PrepareAsync(version, CancellationToken.None))
+            initial.Promote();
+        string root = Path.Combine(_root, "cache", "latest");
+        string marker = Path.Combine(root, "repository", ".redot-class-doc-sync.json");
+        var metadata = System.Text.Json.Nodes.JsonNode.Parse(File.ReadAllText(marker))!.AsObject();
+        metadata.Remove("SelectionRevision");
+        metadata.Remove("CoreDocumentationPath");
+        metadata.Remove("DocumentationDirectories");
+        File.WriteAllText(marker, metadata.ToJsonString());
+        Directory.Delete(Path.Combine(root, "repository", "modules"), true);
+
+        Assert.True(source.TryGetCurrent(version, out var cached));
+        using (cached)
+            Assert.Single(cached!.ClassDocumentationPaths);
+        using var upgrade = await source.PrepareAsync(version, CancellationToken.None);
+        Assert.True(upgrade.IsPending);
+        Assert.Equal(metadata["CommitSha"]!.GetValue<string>(), upgrade.CommitSha);
+        Assert.Contains("Alpha", new ClassDocumentationParser().ParseDirectories(upgrade.ClassDocumentationPaths).Keys);
+        upgrade.Promote();
+        using var current = await source.PrepareAsync(version, CancellationToken.None);
+        Assert.False(current.IsPending);
+    }
+
+    [Theory]
+    [InlineData("<not-a-class />")]
+    [InlineData("<class name=\"Node\" />")]
+    public async Task InvalidModule_LeavesPreviousSnapshotAvailable(string xml)
+    {
+        string remote = CreateRemoteRepository(out string work);
+        var source = CreateSource(remote);
+        var version = CreateVersion();
+        using (var initial = await source.PrepareAsync(version, CancellationToken.None))
+            initial.Promote();
+        AddModule(work, "bad", "Bad");
+        File.WriteAllText(Path.Combine(work, "modules", "bad", "doc_classes", "Bad.xml"), xml);
+        CommitAndPush(work, remote);
+        using (var candidate = await source.PrepareAsync(version, CancellationToken.None))
+            Assert.Throws<InvalidDataException>(() => new ClassDocumentationParser().ParseDirectories(candidate.ClassDocumentationPaths));
+        Assert.True(source.TryGetCurrent(version, out var cached));
+        using (cached)
+            Assert.Single(new ClassDocumentationParser().ParseDirectories(cached!.ClassDocumentationPaths));
+    }
+
+    [Fact]
+    public async Task Versions_DiscoverTheirOwnModules()
+    {
+        string remote = CreateRemoteRepository(out string work);
+        RunGit(work, "push", remote, "HEAD:refs/heads/26.1");
+        AddModule(work, "new_feature", "NewFeature");
+        CommitAndPush(work, remote);
+        var source = CreateSource(remote);
+        var stableVersion = new DocumentationVersion { Slug = "26.1", FriendlyName = "Stable", BranchName = "26.1" };
+        using var stable = await source.PrepareAsync(stableVersion, CancellationToken.None);
+        using var latest = await source.PrepareAsync(CreateVersion(), CancellationToken.None);
+        var parser = new ClassDocumentationParser();
+        Assert.Single(parser.ParseDirectories(stable.ClassDocumentationPaths));
+        var latestClasses = parser.ParseDirectories(latest.ClassDocumentationPaths);
+        Assert.Contains("NewFeature", latestClasses.Keys);
+        var snapshot = new ClassDocumentationSnapshot(latest.Version, latest.CommitSha, latest.SynchronizedAt, latestClasses);
+        var catalog = new ClassDocumentationCatalog();
+        catalog.Publish(snapshot);
+        Assert.Contains(catalog.GetClassesAlphabetically("latest"), entry => entry.Name == "NewFeature");
+        Assert.True(catalog.TryGetClass("latest", "NewFeature", out var module));
+        string html = new ClassDocumentationRenderer().RenderPage(module!, snapshot);
+        Assert.Contains("/en/latest/Classes/Node", html);
+    }
+
+    private static void AddModule(string work, string module, string name)
+    {
+        string path = Path.Combine(work, "modules", module, "doc_classes");
+        Directory.CreateDirectory(path);
+        File.WriteAllText(Path.Combine(path, name + ".xml"), $"<class name=\"{name}\" inherits=\"Node\" />");
+    }
+
+    private static void CommitAndPush(string work, string remote)
+    {
+        RunGit(work, "add", ".");
+        RunGit(work, "commit", "-m", "Update modules");
+        RunGit(work, "push", remote, "master");
+    }
+
     private GitClassDocumentationSource CreateSource(string repositoryUrl)
     {
         var options = Options.Create(new ClassDocumentationOptions

Redot-Documentation/ClassDocumentation/ClassDocumentationParser.cs

@@ -15,20 +15,34 @@ public sealed class ClassDocumentationParser
     /// <exception cref="IOException">A class file cannot be read.</exception>
     /// <exception cref="UnauthorizedAccessException">A class file cannot be accessed.</exception>
     public IReadOnlyDictionary<string, ClassDocumentationEntry> ParseDirectory(string classDocumentationPath)
-    {
-        if (!Directory.Exists(classDocumentationPath))
-            throw new DirectoryNotFoundException($"Class documentation directory was not found: {classDocumentationPath}");
+        => ParseDirectories([classDocumentationPath]);
 
+    /// <summary>Parses and merges core and module directories into one class namespace.</summary>
+    public IReadOnlyDictionary<string, ClassDocumentationEntry> ParseDirectories(IEnumerable<string> paths)
+    {
         var classes = new Dictionary<string, ClassDocumentationEntry>(StringComparer.OrdinalIgnoreCase);
-        foreach (string filePath in Directory.EnumerateFiles(classDocumentationPath, "*.xml", SearchOption.TopDirectoryOnly))
+        var sources = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+        foreach (string directory in paths.Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal))
         {
-            ClassDocumentationEntry entry = ParseFile(filePath);
-            if (!classes.TryAdd(entry.Name, entry))
-                throw new InvalidDataException($"Duplicate class documentation entry '{entry.Name}'.");
+            if (!Directory.Exists(directory))
+                throw new DirectoryNotFoundException($"Class documentation directory was not found: {directory}");
+
+            string[] files = Directory.GetFiles(directory, "*.xml", SearchOption.TopDirectoryOnly);
+            if (files.Length == 0)
+                throw new InvalidDataException($"No class XML files were found in '{directory}'.");
+
+            foreach (string filePath in files.Order(StringComparer.Ordinal))
+            {
+                ClassDocumentationEntry entry = ParseFile(filePath);
+                if (!classes.TryAdd(entry.Name, entry))
+                    throw new InvalidDataException(
+                        $"Duplicate class documentation entry '{entry.Name}' in '{sources[entry.Name]}' and '{filePath}'.");
+                sources.Add(entry.Name, filePath);
+            }
         }
 
         if (classes.Count == 0)
-            throw new InvalidDataException($"No class XML files were found in '{classDocumentationPath}'.");
+            throw new InvalidDataException("No class documentation directories were supplied.");
 
         return new ReadOnlyDictionary<string, ClassDocumentationEntry>(classes);
     }

Redot-Documentation/ClassDocumentation/ClassDocumentationSyncService.cs

@@ -262,7 +262,7 @@ private ClassDocumentationSnapshot CreateSnapshot(ClassDocumentationCheckout che
             checkout.Version,
             checkout.CommitSha,
             checkout.SynchronizedAt,
-            _parser.ParseDirectory(checkout.ClassDocumentationPath));
+            _parser.ParseDirectories(checkout.ClassDocumentationPaths));
 
     /// <summary>Shortens a commit identifier for logging.</summary>
     /// <param name="commitSha">The full commit identifier.</param>