Adding support for annotating until we reach a certain end

Sources/Syntax/Implementations/Common/BinaryOperation/BinaryOperationParser.swift

@@ -101,18 +101,18 @@ extension BinaryOperationParser {
         let leftMost = try member(scanner)
         var representations = [IntermediateRepresentation]()
         while true {
-            scanner.begin()
-            do {
+            let added: Bool = scanner.attempt { scanner in
                 scanner.enterNode()
                 try scanner.parse(using: current.parser)
                 scanner.exitNode()
                 scanner.configureNode(kind: .binaryOperator)
                 scanner.pruneNode(strategy: .separate)
-                try scanner.commit()
+            }
+
+            if added {
                 let value = try member(scanner)
                 representations.append(value)
-            } catch {
-                try scanner.rollback()
+            } else {
                 break
             }
         }

Sources/Syntax/Implementations/Modifications/Maybe.swift

@@ -41,12 +41,11 @@ extension Maybe: InternalParser {
     }
 
     func parse(using scanner: Scanner) throws {
-        scanner.begin()
-        do {
+        let parsed: Bool = scanner.attempt { scanner in
             try scanner.parse(using: content)
-            try scanner.commit()
-        } catch {
-            try scanner.rollback()
+        }
+
+        if !parsed {
             scanner.store(value: nil as Output)
         }
     }

Sources/Syntax/Implementations/Modifications/Parser+repeatUntil.swift

@@ -48,13 +48,13 @@ extension RepeatUntil: InternalParser {
 
         while (true) {
             let index = scanner.range.lowerBound
-            scanner.begin()
-            do {
-                try scanner.parse(using: end)
-                try scanner.commit()
+            let hasParsedEnd: Bool = scanner.attempt { scanner in
+                try scanner.parse(using: self.end)
+            }
+
+            if hasParsedEnd {
                 break
-            } catch { }
-            try scanner.rollback()
+            }
 
             scanner.begin()
             do {

Sources/Syntax/Implementations/Structure/Annotated.swift

@@ -32,20 +32,17 @@ extension Annotated: InternalParser {
 
         if let pattern = pattern {
             let match = try scanner.take(pattern: pattern)
-            scanner.beginScanning(in: match.range, for: Content.Output.self)
+            scanner.beginScanning(in: match.range, clipToLast: false, for: Content.Output.self)
         } else {
-            scanner.beginScanning(in: scanner.range, for: Content.Output.self)
+            scanner.beginScanning(in: scanner.range, clipToLast: false, for: Content.Output.self)
         }
 
         while (true) {
-            scanner.begin()
-            do {
+            let hasParsed: Bool = scanner.attempt { scanner in
                 try parser.parse(using: scanner)
-                try scanner.commit()
-                try scanner.commit()
-            } catch {
-                scanner.exitNode()
-                try scanner.rollback()
+            }
+
+            if !hasParsed {
                 break
             }
         }

Sources/Syntax/Implementations/Structure/AnnotatedUntil.swift

@@ -0,0 +1,105 @@
+
+import Foundation
+import SyntaxTree
+
+public struct AnnotatedUntil<Content : Parser, End: Parser>: Parser {
+    private enum AnnotationValue {
+        case content(Content.Output)
+        case end(End.Output)
+    }
+
+    let id = UUID()
+
+    let content: InternalParser
+    let end: InternalParser
+
+    public init(@ParserBuilder content: () -> Content, @ParserBuilder end: () -> End) {
+        self.content = content().map(AnnotationValue.content)
+        self.end = end().map(AnnotationValue.end)
+    }
+
+    public var body: AnyParser<(AnnotatedString<Content.Output>, End.Output)> {
+        return neverBody()
+    }
+}
+
+extension AnnotatedUntil: InternalParser {
+
+    func prefixes() -> Set<String> {
+        return []
+    }
+
+    func parse(using scanner: Scanner) throws {
+        scanner.enterNode()
+        scanner.beginScanning(in: scanner.range, clipToLast: true, for: AnnotationValue.self)
+
+        while (true) {
+            let nextContentRange = try scanner.range(for: content)
+            let nextEndRange = try scanner.range(for: end)
+
+            switch (nextContentRange, nextEndRange) {
+
+            case (.none, .none), (.none, .some):
+                break
+
+            case (.some(let nextContentRange), .some(let nextEndRange)):
+                if nextContentRange.lowerBound >= nextEndRange.lowerBound {
+                    break
+                } else {
+                    fallthrough
+                }
+
+            default:
+                scanner.begin()
+                try content.parse(using: scanner)
+                try scanner.commit()
+                continue
+            }
+
+            break
+        }
+
+        scanner.begin()
+        try end.parse(using: scanner)
+        try scanner.commit()
+
+        try scanner.commit()
+
+        let annotatedString = try scanner.pop(of: AnnotatedString<AnnotationValue>.self)
+        let last = annotatedString.annotations.last!
+
+        let text = annotatedString.text[..<last.range.lowerBound]
+
+        guard case .end(let end) = last.value else { fatalError() }
+
+        let annotations = annotatedString.annotations.dropLast().map { annotation -> AnnotatedString<Content.Output>.Annotation in
+            guard case .content(let content) = annotation.value else { fatalError() }
+            return AnnotatedString<Content.Output>.Annotation(range: annotation.range, value: content)
+        }
+
+        let newString = AnnotatedString(text: text, annotations: annotations)
+        scanner.store(value: (newString, end))
+        scanner.exitNode()
+    }
+
+}
+
+extension Scanner {
+
+    fileprivate func range(for parser: InternalParser) throws -> Range<Location>? {
+        begin()
+        enterNode()
+        do {
+            try parser.parse(using: self)
+            exitNode()
+        } catch {
+            try rollback()
+            return nil
+        }
+
+        let range = locationOfNode()
+        try rollback()
+        return range
+    }
+
+}

Sources/Syntax/Internals/Scanner.swift

@@ -20,7 +20,7 @@ protocol Scanner {
     func prefix(_ length: Int) throws -> Substring?
 
     func begin()
-    func beginScanning<T>(in range: Range<String.Index>, for type: T.Type)
+    func beginScanning<T>(in range: Range<String.Index>, clipToLast: Bool, for type: T.Type)
 
     func commit() throws
 
@@ -36,6 +36,32 @@ protocol Scanner {
     func store(error: DiagnosticError)
 }
 
+extension Scanner {
+
+    func attempt(_ closure: (Scanner) throws -> Void) throws {
+        begin()
+        do {
+            try closure(self)
+            try commit()
+        } catch {
+            try rollback()
+        }
+    }
+
+    func attempt(_ closure: (Scanner) throws -> Void) -> Bool {
+        begin()
+        do {
+            try closure(self)
+            try commit()
+            return true
+        } catch {
+            try! rollback()
+            return false
+        }
+    }
+
+}
+
 extension Scanner {
     func take(word: String) throws -> String {
         do {

Sources/Syntax/Internals/ScannerState.swift

@@ -150,8 +150,8 @@ final class ScannerState {
         return storage.begin(self)
     }
 
-    func beginScanning<T>(in range: Range<String.Index>, for type: T.Type) -> ScannerState {
-        return ScannerState(range: range, parent: self, storage: ScanningStorage<T>())
+    func beginScanning<T>(in range: Range<String.Index>, clipToLast: Bool, for type: T.Type) -> ScannerState {
+        return ScannerState(range: range, parent: self, storage: ScanningStorage<T>(clipToLast: clipToLast))
     }
 
     func rollback() -> ScannerState? {
@@ -221,13 +221,13 @@ private final class InPlaceStorage: ScannerStateStorage {
         let currentStack = values
 
         try parser.parse(using: scanner)
-        var values = self.values
-        values.remove(from: currentStack)
+        var newValuesStack = self.values
+        newValuesStack.remove(from: currentStack)
         let memoized = MemoizedState(range: state.range,
-                                          node: state.node,
-                                          values: values,
-                                          ids: ids,
-                                          lastDiagnosticError: state.lastDiagnosticError)
+                                     node: state.node,
+                                     values: newValuesStack,
+                                     ids: ids,
+                                     lastDiagnosticError: state.lastDiagnosticError)
 
         memoizationStorage[key] = memoized
     }
@@ -352,6 +352,8 @@ private protocol ScanningStorageProtocol {
 
 private final class ScanningStorage<T>: ScannerStateStorage, ScanningStorageProtocol {
     typealias Annotation = AnnotatedString<T>.Annotation
+
+    private let clipToLast: Bool
     private let fromParent: [Range<String.Index>]
     var annotations: [Annotation] = []
     var ids: Set<UUID>
@@ -371,13 +373,15 @@ private final class ScanningStorage<T>: ScannerStateStorage, ScanningStorageProt
         }
     }
 
-    init() {
+    init(clipToLast: Bool) {
+        self.clipToLast = clipToLast
         fromParent = []
         ids = []
     }
 
     init(parent: ScanningStorage<T>) {
         self.parent = parent
+        self.clipToLast = parent.clipToLast
         fromParent = parent.annotations.map(\.range) + parent.fromParent
         ids = parent.ids
     }
@@ -413,19 +417,19 @@ private final class ScanningStorage<T>: ScannerStateStorage, ScanningStorageProt
         }
 
         // Upon a match start with in place storage
-        let nextState = ScannerState(range: result.range.lowerBound..<state.range.upperBound, parent: state, storage: InPlaceStorage())
-        nextState.range = result.range.upperBound..<state.range.upperBound
-
-        let (firstNode, rest) = state.node.backToFirst()
-
-        state.node = firstNode
-        if let rest = rest {
-            rest.update(from: state.range.lowerBound, to: result.range.lowerBound)
-            nextState.node = rest
-        }
+//        let nextState = ScannerState(range: result.range.lowerBound..<state.range.upperBound, parent: state, storage: InPlaceStorage())
+//        nextState.range = result.range.upperBound..<state.range.upperBound
+//
+//        let (firstNode, rest) = state.node.backToFirst()
+//
+//        state.node = firstNode
+//        if let rest = rest {
+//            rest.update(from: state.range.lowerBound, to: result.range.lowerBound)
+//            nextState.node = rest
+//        }
 
         rangeMatchStart = result.range.lowerBound
-        return ExpressionScanResult(match: result, state: nextState)
+        return ExpressionScanResult(match: result, state: state)
     }
 
     func begin(_ state: ScannerState) -> ScannerState {
@@ -441,8 +445,11 @@ private final class ScanningStorage<T>: ScannerStateStorage, ScanningStorageProt
 
         switch parent.storage.effective() {
         case let storage as InPlaceStorage:
-            parent.range = state.range.upperBound..<parent.range.upperBound
-            storage.values.append(AnnotatedString(text: state.text[state.range], annotations: annotations.sorted { $0.range.lowerBound < $1.range.lowerBound }))
+            let start = state.range.lowerBound
+            let end = clipToLast ? annotations.map(\.range.upperBound).max() ?? start : state.range.upperBound
+
+            parent.range = end..<parent.range.upperBound
+            storage.values.append(AnnotatedString(text: state.text[start..<end], annotations: annotations.sorted { $0.range.lowerBound < $1.range.lowerBound }))
             storage.ids = []
 
             if parent.node.start >= state.node.originalStart && state.node.originalStart < state.node.start {
@@ -508,6 +515,11 @@ private class StackedScanningStateStorage<T>: ScannerStateStorage {
         return current.lastDiagnosticError
     }
 
+    init(clipToLast: Bool) {
+        self.scanning = ScanningStorage(clipToLast: clipToLast)
+        self.inplace = InPlaceStorage()
+    }
+
     init(parent: ScanningStorage<T>) {
         self.scanning = ScanningStorage(parent: parent)
         self.inplace = InPlaceStorage()
@@ -539,13 +551,8 @@ private class StackedScanningStateStorage<T>: ScannerStateStorage {
         }
 
         let result = try current.take(expression: expression, in: text, for: state)
-
-        var state: ScannerState? = state
-        while let storage = state?.storage as? StackedScanningStateStorage<T> {
-            storage.isInPlace = true
-            state = state?.parent
-        }
-
+        isInPlace = true
+        state.range = result.match.range.upperBound..<state.range.upperBound
         return result
     }
 
@@ -558,6 +565,9 @@ private class StackedScanningStateStorage<T>: ScannerStateStorage {
     }
 
     func commit(_ state: ScannerState) throws -> ScannerState? {
+        if let parent = state.parent, let storage = parent.storage as? StackedScanningStateStorage<T> {
+            storage.isInPlace = self.isInPlace
+        }
         return try current.commit(state)
     }