BrsFile ast toString()

src/astUtils/creators.ts

@@ -8,6 +8,7 @@ import { LiteralExpression, CallExpression, DottedGetExpression, VariableExpress
 import { AssignmentStatement, Block, DottedSetStatement, IfStatement, IndexedSetStatement, MethodStatement } from '../parser/Statement';
 
 const tokenDefaults = {
+    [TokenKind.At]: '@',
     [TokenKind.BackTick]: '`',
     [TokenKind.Backslash]: '\\',
     [TokenKind.BackslashEqual]: '\\=',
@@ -28,6 +29,8 @@ const tokenDefaults = {
     [TokenKind.EndSub]: 'end sub',
     [TokenKind.EndTry]: 'end try',
     [TokenKind.EndWhile]: 'end while',
+    [TokenKind.ExitWhile]: 'exit while',
+    [TokenKind.ForEach]: 'for each',
     [TokenKind.Equal]: '=',
     [TokenKind.Greater]: '>',
     [TokenKind.GreaterEqual]: '>=',

src/lexer/Lexer.ts

@@ -391,6 +391,8 @@ export class Lexer {
                 ...DiagnosticMessages.unexpectedCharacter(c),
                 location: this.locationOf()
             });
+            //keep the character as leading trivia (so the source code can be rebuilt from the AST), but not in the token list, so the parser never sees it
+            this.addToken(TokenKind.UnexpectedCharacter);
         }
     }
 
@@ -1209,7 +1211,7 @@ export class Lexer {
             token.leadingTrivia = [...this.leadingTrivia];
             this.leadingTrivia = [];
         }
-        if (kind !== TokenKind.Comment) {
+        if (kind !== TokenKind.Comment && kind !== TokenKind.UnexpectedCharacter) {
             this.tokens.push(token);
         }
         this.sync();

src/lexer/TokenKind.ts

@@ -186,6 +186,11 @@ export enum TokenKind {
     // structural
     Whitespace = 'Whitespace',
     Newline = 'Newline',
+    /**
+     * A character the lexer does not recognize. These are only ever included in leading trivia (never in the token list),
+     * so the parser ignores them, but they are kept so the source code can be rebuilt from the AST
+     */
+    UnexpectedCharacter = 'UnexpectedCharacter',
     Eof = 'Eof'
 }
 
@@ -746,7 +751,8 @@ export const AllowedTriviaTokens: ReadonlyArray<TokenKind> = [
     TokenKind.Newline,
     TokenKind.Whitespace,
     TokenKind.Comment,
-    TokenKind.Colon
+    TokenKind.Colon,
+    TokenKind.UnexpectedCharacter
 ];
 
 

src/parser/AstNode.spec.ts

@@ -2174,4 +2174,41 @@ describe('AstNode', () => {
             });
         });
     });
+
+    describe('toString', () => {
+        it('retains full fidelity', () => {
+            const text = `
+                thing = true
+
+                if true
+                    thing = true
+                end if
+
+                if true
+                    thing = true
+                else
+                    thing = true
+                end if
+
+                if true
+                    thing = true
+                else if true
+                    thing = true
+                else
+                    thing = true
+                end if
+
+                for i = 0 to 10 step 1
+                    print true,false;3
+                end for
+
+                for each item in thing
+                    print 1
+                end for
+            `;
+            expect(
+                Parser.parse(text).ast.toString()
+            ).to.eql(text);
+        });
+    });
 });

src/parser/AstNode.ts

@@ -12,6 +12,8 @@ import { DynamicType } from '../types/DynamicType';
 import type { BscType } from '../types/BscType';
 import type { Token } from '../lexer/Token';
 import { isBlock, isBody, isFunctionParameterExpression } from '../astUtils/reflection';
+import type { SourceNode } from 'source-map';
+import { TranspileState } from './TranspileState';
 
 /**
  * A BrightScript AST node
@@ -276,6 +278,21 @@ export abstract class AstNode {
      */
     public abstract clone(): AstNode;
 
+    /**
+     * Generate a SourceNode that represents the source code of this node (including all leading trivia such as whitespace and comments).
+     * Unlike `transpile`, this does not convert BrighterScript into BrightScript, it produces the code exactly as it appears in the AST.
+     * Annotations are written by `TranspileState.nodeToSourceNode`, so implementations should use that for writing child nodes.
+     */
+    public abstract toSourceNode(state: TranspileState): SourceNode;
+
+    /**
+     * Get the source code for this node, exactly as it appears in the AST (including all leading trivia such as whitespace and comments).
+     * For an unmodified AST, calling this on the root node produces the exact text that was parsed.
+     */
+    public toString(): string {
+        return new TranspileState('', {}).nodeToSourceNode(this).toString();
+    }
+
     /**
      * Helper function for creating a clone. This will clone any attached annotations, as well as reparent the cloned node's children to the clone
      */