Remove the assert->writeln pipeline in favor of the DDoc wrapper

ddoc/.gitignore

@@ -0,0 +1,4 @@
+/libddoc_preprocessor.a
+/ddoc_preprocessor
+/ddoc_preprocessor-test-application
+/ddoc_preprocessor-test-library

ddoc/dub.sdl

@@ -0,0 +1,11 @@
+name "ddoc_preprocessor"
+description "Preprocesses source code before running Ddoc over it"
+dependency "libdparse" version="0.7.2-alpha.4"
+dependency "dmd" path="../../dmd"
+versions "DdocOptions"
+configuration "executable" {
+	versions "IsExecutable"
+	targetType "executable"
+}
+configuration "library" {
+}

ddoc/dub.selections.json

@@ -0,0 +1,6 @@
+{
+	"fileVersion": 1,
+	"versions": {
+		"libdparse": "0.7.2-alpha.4"
+	}
+}

assert_writeln_magic.d → ddoc/source/assert_writeln_magic.d

@@ -177,7 +177,7 @@ private:
     Out fl;
 }
 
-void parseString(Visitor)(ubyte[] sourceCode, string fileName, Visitor visitor)
+void parseString(Visitor)(ubyte[] sourceCode, Visitor visitor)
 {
     import dparse.lexer;
     import dparse.parser : parseModule;
@@ -188,89 +188,36 @@ void parseString(Visitor)(ubyte[] sourceCode, string fileName, Visitor visitor)
     const(Token)[] tokens = getTokensForParser(sourceCode, config, &cache).array;
 
     RollbackAllocator rba;
-    auto m = parseModule(tokens, fileName, &rba);
+    auto m = parseModule(tokens, "magic.d", &rba);
     visitor.visit(m);
 }
 
-void parseFile(string fileName, string destFile)
+private auto assertWritelnModuleImpl(string fileText)
 {
-    import std.array : uninitializedArray;
-
-    auto inFile = File(fileName);
-    if (inFile.size == 0)
-        warningf("%s is empty", inFile.name);
-
-    ubyte[] sourceCode = uninitializedArray!(ubyte[])(to!size_t(inFile.size));
-    if (sourceCode.length == 0)
-        return;
-
-    inFile.rawRead(sourceCode);
-    auto fl = FileLines(fileName, destFile);
+    import std.string : representation;
+    auto fl = FileLines(fileText);
     auto visitor = new TestVisitor!(typeof(fl))(fl);
-    parseString(sourceCode, fileName, visitor);
+    // libdparse doesn't allow to work on immutable source code
+    parseString(cast(ubyte[]) fileText.representation, visitor);
     delete visitor;
+    return fl;
 }
 
-// Modify a path under oldBase to a new path with the same subpath under newBase.
-// E.g.: `/foo/bar`.rebasePath(`/foo`, `/quux`) == `/quux/bar`
-string rebasePath(string path, string oldBase, string newBase)
+auto assertWritelnModule(string fileText)
 {
-    import std.path : absolutePath, buildPath, relativePath;
-    return buildPath(newBase, path.absolutePath.relativePath(oldBase.absolutePath));
+    return assertWritelnModuleImpl(fileText).buildLines;
 }
-
-version(unittest) { void main(){} } else
-void main(string[] args)
+auto assertWritelnBlock(string fileText)
 {
-    import std.file;
-    import std.getopt;
-    import std.path;
-
-    string inputDir, outputDir;
-    string[] ignoredFiles;
-
-    auto helpInfo = getopt(args, config.required,
-            "inputdir|i", "Folder to start the recursive search for unittest blocks (can be a single file)", &inputDir,
-            "outputdir|o", "Alternative folder to use as output (can be a single file)", &outputDir,
-            "ignore", "List of files to exclude (partial matching is supported)", &ignoredFiles);
-
-    if (helpInfo.helpWanted)
-    {
-        return defaultGetoptPrinter(`assert_writeln_magic
-Tries to lower EqualExpression in AssertExpressions of Unittest blocks to commented writeln calls.
-`, helpInfo.options);
-    }
-
-    inputDir = inputDir.asNormalizedPath.array;
-
-    DirEntry[] files;
-
-    // inputDir as default output directory
-    if (!outputDir.length)
-        outputDir = inputDir;
-
-    if (inputDir.isFile)
-    {
-        files = [DirEntry(inputDir)];
-        inputDir = "";
-    }
-    else
-    {
-        files = dirEntries(inputDir, SpanMode.depth).filter!(
-                a => a.name.endsWith(".d") && !a.name.canFind(".git")).array;
-    }
-
-    foreach (file; files)
+    auto source = "unittest{\n" ~ fileText ~ "}\n";
+    auto fl = assertWritelnModuleImpl(source);
+    auto app = appender!string;
+    foreach (line; fl.lines[1 .. $ - 2])
     {
-        if (!ignoredFiles.any!(x => file.name.canFind(x)))
-        {
-            // single files
-            if (inputDir.length == 0)
-                parseFile(file.name, outputDir);
-            else
-                parseFile(file.name, file.name.rebasePath(inputDir, outputDir));
-        }
+        app ~= line;
+        app ~= "\n";
     }
+    return app.data;
 }
 
 /**
@@ -284,47 +231,26 @@ struct FileLines
 
     string[] lines;
     string destFile;
-    bool overwriteInputFile;
     bool hasWrittenChanges;
 
-    this(string inputFile, string destFile)
-    {
-        stderr.writefln("%s -> %s", inputFile, destFile);
-        this.overwriteInputFile = inputFile == destFile;
-        this.destFile = destFile;
-        lines = File(inputFile).byLineCopy.array;
-
-        destFile.dirName.mkdirRecurse;
-    }
-
-    // dumps all changes
-    ~this()
+    this(string inputText)
     {
-        if (overwriteInputFile)
-        {
-            if (hasWrittenChanges)
-            {
-                auto tmpFile = File(destFile ~ ".tmp", "w");
-                writeLinesToFile(tmpFile);
-                tmpFile.close;
-                tmpFile.name.rename(destFile);
-            }
-        }
-        else
-        {
-            writeLinesToFile(File(destFile, "w"));
-        }
+        lines = inputText.split("\n");
     }
 
     // writes all changes to a random, temporary file
-    void writeLinesToFile(File outFile) {
+    auto buildLines() {
+        auto app = appender!string;
         // dump file
         foreach (line; lines)
-            outFile.writeln(line);
+        {
+            app ~= line;
+            app ~= "\n";
+        }
         // within the docs we automatically inject std.stdio (hence we need to do the same here)
         // writeln needs to be @nogc, @safe, pure and nothrow (we just fake it)
-        outFile.writeln("// \nprivate void writeln(T)(T l) { }");
-        outFile.flush;
+        app ~= "// \nprivate void writeln(T)(T l) { }";
+        return app.data;
     }
 
     string opIndex(size_t i) { return lines[i]; }
@@ -349,7 +275,7 @@ version(unittest)
         import std.string : representation;
         auto mock = FileLinesMock(sourceCode.split("\n"));
         auto visitor = new TestVisitor!(typeof(mock))(mock);
-        parseString(sourceCode.representation.dup, "testmodule", visitor);
+        parseString(sourceCode.representation.dup, visitor);
         delete visitor;
         return mock;
     }

ddoc_preprocessor.d → ddoc/source/preprocessor.d

@@ -24,12 +24,14 @@ struct Config
 {
     string dmdBinPath = "dmd";
     string outputFile;
-    string cwd = __FILE_FULL_PATH__.dirName;
+    string cwd = __FILE_FULL_PATH__.dirName.dirName.dirName;
 }
 Config config;
 
+version(IsExecutable)
 int main(string[] rootArgs)
 {
+    import assert_writeln_magic;
     import std.getopt;
     auto helpInformation = getopt(
         rootArgs, std.getopt.config.passThrough,
@@ -49,33 +51,71 @@ All unknown options are passed to the compiler.
     assert(pos >= 0, "An input file (.d or .dd) must be provided");
     auto inputFile = args[pos];
     auto text = inputFile.readText;
-    // replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017
-    args = args[0..pos].chain("-".only, args[pos..$].dropOne).array;
+
+    // for now only non-package modules are supported
+    if (!inputFile.endsWith("index.d"))
+        // replace only works with 2.078.1, see: https://github.com/dlang/phobos/pull/6017
+        args = args[0..pos].chain("-".only, args[pos..$].dropOne).array;
 
     // transform and extend the ddoc page
     text = genGrammar(text);
     text = genHeader(text);
     text = genChangelogVersion(inputFile, text);
     text = genSwitches(text);
+    text = fixDdocBugs(inputFile, text);
+
+    // Phobos index.d should have been named index.dd
+    if (inputFile.endsWith(".d") && !inputFile.endsWith("index.d"))
+        text = assertWritelnModule(text);
+
+    string[string] macros;
+    macros["SRC_FILENAME"] = "%s\n".format(inputFile.buildNormalizedPath);
+    return compile(text, args, inputFile, macros);
+}
 
-    // inject custom, "dynamic" macros
-    text ~= "\nSRC_FILENAME=%s\n".format(inputFile.buildNormalizedPath);
-    return compile(text, args);
+auto createTmpDir()
+{
+    import std.uuid : randomUUID;
+    auto dir = tempDir.buildPath("ddoc_preprocessor_" ~ randomUUID.toString.replace("-", ""));
+    mkdir(dir);
+    return dir;
 }
 
-auto compile(R)(R buffer, string[] arguments)
+auto compile(R)(R buffer, string[] arguments, string inputFile, string[string] macros = null)
 {
+    import core.time : usecs;
+    import core.thread : Thread;
     import std.process : pipeProcess, Redirect, wait;
     auto args = [config.dmdBinPath] ~ arguments;
-    foreach (arg; ["-c", "-o-", "-"])
+
+    // Note: ideally we could pass in files directly on stdin.
+    // However, for package.d files, we need to imitate package directory layout to avoid conflicts
+    auto tmpDir = createTmpDir;
+    auto inputTmpFile = tmpDir.buildPath(inputFile);
+    inputTmpFile.dirName.mkdirRecurse;
+    std.file.write(inputTmpFile, buffer);
+    args = args.replace("-", inputTmpFile);
+    scope(exit) tmpDir.rmdirRecurse;
+
+    if (macros !is null)
+    {
+        auto macroString = macros.byPair.map!(a => "%s=%s".format(a[0], a[1])).join("\n");
+        auto macroFile = tmpDir.buildPath("macros.ddoc");
+        std.file.write(macroFile, macroString);
+        args ~= macroFile;
+    }
+
+    foreach (arg; ["-c", "-o-"])
     {
         if (!args.canFind(arg))
             args ~= arg;
     }
     auto pipes = pipeProcess(args, Redirect.stdin);
-    pipes.stdin.write(buffer);
-    pipes.stdin.close;
-    return wait(pipes.pid);
+    import std.process : execute;
+    auto ret = execute(args);
+    if (ret.status != 0)
+        stderr.writeln(ret.output);
+    return ret.status;
 }
 
 // replaces the content of a DDoc macro call
@@ -384,3 +424,18 @@ private void highlightSpecialWords(ref string flag, ref string helpText)
             .to!string;
     }
 }
+
+// Fix ddoc bugs
+string fixDdocBugs(string inputFile, string text)
+{
+
+    // https://github.com/dlang/dlang.org/pull/2069#issuecomment-363154934
+    // can be removed once the Phobos PR is in stable and master
+    // https://github.com/dlang/phobos/pull/6126
+    if (inputFile.endsWith("exception.d"))
+    {
+        text = text.replace(`typeof(new E("", __FILE__, __LINE__)`, `typeof(new E("", string.init, size_t.init)`);
+        text = text.replace(`typeof(new E(__FILE__, __LINE__)`, `typeof(new E(string.init, size_t.init)`);
+    }
+    return text;
+}

dpl-docs/dub.json

@@ -1,7 +1,10 @@
 {
 	"name": "dpl-docs",
 	"dependencies": {
-		"ddox": "~>0.16.7"
+		"ddox": "~>0.16.7",
+		"ddoc_preprocessor": {
+		  "path": "../ddoc"
+		}
 	},
 	"versions": ["VibeNoSSL"],
 	"subConfigurations": { "eventcore": "libasync"}

dpl-docs/source/app.d

@@ -1,6 +1,6 @@
 module app;
 
-import ddox.main;
+import ddox_main;
 import std.getopt;
 import std.process;
 import vibe.core.log;