TokenizerTests.fs

#nowarn "57" // FSharpLexer.Tokenize is experimental
module FSharp.Compiler.Service.Tests.TokenizerTests

open FSharp.Compiler.Tokenization
open FSharp.Test
open Xunit

let rec parseLine(line: string, state: FSharpTokenizerLexState ref, tokenizer: FSharpLineTokenizer) = seq {
  match tokenizer.ScanToken(state.Value) with
  | Some(tok), nstate ->
      let str = line.Substring(tok.LeftColumn, tok.RightColumn - tok.LeftColumn + 1)
      yield str, tok
      state.Value <- nstate
      yield! parseLine(line, state, tokenizer)
  | None, nstate -> 
      state.Value <- nstate }

let tokenizeLines (lines:string[]) =
  let sourceTok = FSharpSourceTokenizer([], Some "C:\\test.fsx", None, None)
  [ let state = ref FSharpTokenizerLexState.Initial
    for n, line in lines |> Seq.zip [ 0 .. lines.Length-1 ] do

      let tokenizer = sourceTok.CreateLineTokenizer(line)
      yield n, parseLine(line, state, tokenizer) |> List.ofSeq ]

[<Fact>]
let ``Tokenizer test - simple let with string``() =
    let tokenizedLines = 
      tokenizeLines
        [| "// Sets the hello world variable"
           "let hello = \"Hello world\" " |]

    let actual = 
        [ for lineNo, lineToks in tokenizedLines do
            yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
    let expected = 
      [(0,
        [("LINE_COMMENT", "//"); ("LINE_COMMENT", " "); ("LINE_COMMENT", "Sets");
         ("LINE_COMMENT", " "); ("LINE_COMMENT", "the"); ("LINE_COMMENT", " ");
         ("LINE_COMMENT", "hello"); ("LINE_COMMENT", " ");
         ("LINE_COMMENT", "world"); ("LINE_COMMENT", " ");
         ("LINE_COMMENT", "variable")]);
       (1,
        [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello");
         ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " ");
         ("STRING_TEXT", "\""); ("STRING_TEXT", "Hello"); ("STRING_TEXT", " ");
         ("STRING_TEXT", "world"); ("STRING", "\""); ("WHITESPACE", " ")])]

    if actual <> expected then 
        printfn "actual   = %A" actual
        printfn "expected = %A" expected
        actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)

[<Fact>]
let ``Tokenizer test 2 - single line non-nested string interpolation``() =
    let tokenizedLines = 
      tokenizeLines
        [| "// Tests tokenizing string interpolation"
           "let hello0 = $\"\""
           "let hello1 = $\"Hello world\"  "
           "let hello2 = $\"Hello world {1+1} = {2}\" "
           "let hello0v = @$\"\""
           "let hello1v = @$\"Hello world\"  "
           "let hello2v = @$\"Hello world {1+1} = {2}\" " |]

    let actual = 
        [ for lineNo, lineToks in tokenizedLines do
            yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
    let expected = 
        [(0,
          [("LINE_COMMENT", "//"); ("LINE_COMMENT", " "); ("LINE_COMMENT", "Tests");
           ("LINE_COMMENT", " "); ("LINE_COMMENT", "tokenizing"); ("LINE_COMMENT", " ");
           ("LINE_COMMENT", "string"); ("LINE_COMMENT", " ");
           ("LINE_COMMENT", "interpolation")]);
         (1,
          [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello0"); ("WHITESPACE", " ");
           ("EQUALS", "="); ("WHITESPACE", " "); ("STRING_TEXT", "$\"");
           ("INTERP_STRING_BEGIN_END", "\"")]);
         (2,
          [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello1"); ("WHITESPACE", " ");
           ("EQUALS", "="); ("WHITESPACE", " "); ("STRING_TEXT", "$\"");
           ("STRING_TEXT", "Hello"); ("STRING_TEXT", " "); ("STRING_TEXT", "world");
           ("INTERP_STRING_BEGIN_END", "\""); ("WHITESPACE", "  ")]);
         (3,
          [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello2"); ("WHITESPACE", " ");
           ("EQUALS", "="); ("WHITESPACE", " "); ("STRING_TEXT", "$\"");
           ("STRING_TEXT", "Hello"); ("STRING_TEXT", " "); ("STRING_TEXT", "world");
           ("STRING_TEXT", " "); ("INTERP_STRING_BEGIN_PART", "{"); ("INT32", "1");
           ("PLUS_MINUS_OP", "+"); ("INT32", "1"); ("STRING_TEXT", "}");
           ("STRING_TEXT", " "); ("STRING_TEXT", "="); ("STRING_TEXT", " ");
           ("INTERP_STRING_PART", "{"); ("INT32", "2"); ("STRING_TEXT", "}");
           ("INTERP_STRING_END", "\""); ("WHITESPACE", " ")]);
         (4,
          [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello0v");
           ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " ");
           ("STRING_TEXT", "@$\""); ("INTERP_STRING_BEGIN_END", "\"")]);
         (5,
          [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello1v");
           ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " ");
           ("STRING_TEXT", "@$\""); ("STRING_TEXT", "Hello"); ("STRING_TEXT", " ");
           ("STRING_TEXT", "world"); ("INTERP_STRING_BEGIN_END", "\"");
           ("WHITESPACE", "  ")]);
         (6,
          [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello2v");
           ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " ");
           ("STRING_TEXT", "@$\""); ("STRING_TEXT", "Hello"); ("STRING_TEXT", " ");
           ("STRING_TEXT", "world"); ("STRING_TEXT", " ");
           ("INTERP_STRING_BEGIN_PART", "{"); ("INT32", "1"); ("PLUS_MINUS_OP", "+");
           ("INT32", "1"); ("STRING_TEXT", "}"); ("STRING_TEXT", " ");
           ("STRING_TEXT", "="); ("STRING_TEXT", " "); ("INTERP_STRING_PART", "{");
           ("INT32", "2"); ("STRING_TEXT", "}"); ("INTERP_STRING_END", "\"");
           ("WHITESPACE", " ")]);]
  
    if actual <> expected then 
        printfn "actual   = %A" actual
        printfn "expected = %A" expected
        actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)

[<Fact>]
let ``Tokenizer test - multiline non-nested string interpolation``() =
    let tokenizedLines = 
      tokenizeLines
        [| "let hello1t = $\"\"\"abc {1+"
           " 1} def\"\"\"" |]

    let actual = 
        [ for lineNo, lineToks in tokenizedLines do
            yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
    let expected = 
        [(0,
          [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello1t");
           ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " ");
           ("STRING_TEXT", "$\"\"\""); ("STRING_TEXT", "abc"); ("STRING_TEXT", " ");
           ("INTERP_STRING_BEGIN_PART", "{"); ("INT32", "1"); ("PLUS_MINUS_OP", "+")]);
         (1,
          [("WHITESPACE", " "); ("INT32", "1"); ("STRING_TEXT", "}");
           ("STRING_TEXT", " "); ("STRING_TEXT", "def"); ("INTERP_STRING_END", "\"\"\"")])]
  
    if actual <> expected then 
        printfn "actual   = %A" actual
        printfn "expected = %A" expected
        actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)

[<Fact>]
// checks nested '{' and nested single-quote strings
let ``Tokenizer test - multi-line nested string interpolation``() =
    let tokenizedLines = 
      tokenizeLines
        [| "let hello1t = $\"\"\"abc {\"a\" +               "
           "                          {                     "
           "                           contents = \"b\"     "
           "                          }.contents            "
           "                         } def\"\"\"" |]

    let actual = 
        [ for lineNo, lineToks in tokenizedLines do
            yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
    let expected = 
        [(0,
          [("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello1t");
           ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " ");
           ("STRING_TEXT", "$\"\"\""); ("STRING_TEXT", "abc"); ("STRING_TEXT", " ");
           ("INTERP_STRING_BEGIN_PART", "{"); ("STRING_TEXT", "\""); ("STRING_TEXT", "a");
           ("STRING", "\""); ("WHITESPACE", " "); ("PLUS_MINUS_OP", "+");
           ("WHITESPACE", "               ")]);
         (1,
          [("WHITESPACE", "                          "); ("LBRACE", "{");
           ("WHITESPACE", "                     ")]);
         (2,
          [("WHITESPACE", "                           "); ("IDENT", "contents");
           ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " ");
           ("STRING_TEXT", "\""); ("STRING_TEXT", "b"); ("STRING", "\"");
           ("WHITESPACE", "     ")]);
         (3,
          [("WHITESPACE", "                          "); ("RBRACE", "}"); ("DOT", ".");
           ("IDENT", "contents"); ("WHITESPACE", "            ")]);
         (4,
          [("WHITESPACE", "                         "); ("STRING_TEXT", "}");
           ("STRING_TEXT", " "); ("STRING_TEXT", "def"); ("INTERP_STRING_END", "\"\"\"")])]
  
    if actual <> expected then 
        printfn "actual   = %A" actual
        printfn "expected = %A" expected
        actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)

[<Fact>]
let ``Tokenizer test - single-line nested string interpolation``() =
    let tokenizedLines = 
      tokenizeLines
        [| " $\"abc { { contents = 1 } }\"     " |]

    let actual = 
        [ for lineNo, lineToks in tokenizedLines do
            yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
    let expected = 
        [(0,
          [("WHITESPACE", " "); ("STRING_TEXT", "$\""); ("STRING_TEXT", "abc");
           ("STRING_TEXT", " "); ("INTERP_STRING_BEGIN_PART", "{"); ("WHITESPACE", " ");
           ("LBRACE", "{"); ("WHITESPACE", " "); ("IDENT", "contents");
           ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " "); ("INT32", "1");
           ("WHITESPACE", " "); ("RBRACE", "}"); ("WHITESPACE", " ");
           ("STRING_TEXT", "}"); ("INTERP_STRING_END", "\""); ("WHITESPACE", "     ")])]
  
    if actual <> expected then 
        printfn "actual   = %A" actual
        printfn "expected = %A" expected
        actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)

[<Fact>]
let ``Tokenizer test - elif directive produces HASH_ELIF token``() =
    let defines = ["DEBUG"]
    let sourceTok = FSharpSourceTokenizer(defines, Some "C:\\test.fsx", None, None)
    let lines =
        [| "#if DEBUG"
           "let x = 1"
           "#elif RELEASE"
           "let x = 2"
           "#endif" |]
    let state = ref FSharpTokenizerLexState.Initial
    let allTokens =
        [ for line in lines do
            let tokenizer = sourceTok.CreateLineTokenizer(line)
            let lineTokens = parseLine(line, state, tokenizer) |> List.ofSeq
            yield lineTokens ]

    // Line 0: #if DEBUG → HASH_IF keyword
    let line0Names = allTokens.[0] |> List.map (fun (_, tok) -> tok.TokenName)
    Assert.Contains("HASH_IF", line0Names)

    // Line 1: let x = 1 → active code, should have LET token
    let line1Names = allTokens.[1] |> List.map (fun (_, tok) -> tok.TokenName)
    Assert.Contains("LET", line1Names)

    // Line 2: #elif RELEASE → split into HASH_IF + WHITESPACE + IDENT by processHashIfLine
    let line2Names = allTokens.[2] |> List.map (fun (_, tok) -> tok.TokenName)
    Assert.Contains("HASH_IF", line2Names)

    // Line 3: let x = 2 → should be INACTIVECODE (since DEBUG is defined, #elif branch is skipped)
    let line3Names = allTokens.[3] |> List.map (fun (_, tok) -> tok.TokenName)
    Assert.Contains("INACTIVECODE", line3Names)

[<Fact>]
let ``FSharpLexer.Tokenize produces HashElif token kind``() =
    let source = """#if DEBUG
let x = 1
#elif RELEASE
let x = 2
#endif"""
    let tokens = ResizeArray<FSharpToken>()
    let flags = FSharpLexerFlags.Default &&& ~~~FSharpLexerFlags.SkipTrivia
    FSharpLexer.Tokenize(FSharp.Compiler.Text.SourceText.ofString source, tokens.Add, langVersion = "preview", conditionalDefines = ["DEBUG"], flags = flags)
    let hasHashElif = tokens |> Seq.exists (fun t -> t.Kind = FSharpTokenKind.HashElif)
    Assert.True(hasHashElif, "Expected at least one token with Kind = FSharpTokenKind.HashElif")

[<Fact>]
let ``Unfinished idents``() =
    let tokenizedLines =
      tokenizeLines
        [| "`"; "``"; "``a"; "``a`"; "```" |]

    let actual =
        [ for lineTokens in tokenizedLines |> List.map snd do
            [ for str, info in lineTokens -> info.TokenName, str ] ]

    let expected =
        [["IDENT", "`"]
         ["IDENT", "``"]
         ["IDENT", "``a"]
         ["IDENT", "``a`"]
         ["IDENT", "```"]]

    actual |> Assert.shouldBe expected

[<Fact>]
let ``Tokenizer test - optional parameters with question mark``() =
    let tokenizedLines =
      tokenizeLines
        [| "member _.memb(?optional:string) = optional" |]

    let actual =
        [ for lineNo, lineToks in tokenizedLines do
            yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
    
    let expected =
        [(0,
          [("MEMBER", "member"); ("WHITESPACE", " "); ("UNDERSCORE", "_"); ("DOT", ".");
           ("IDENT", "memb"); ("LPAREN", "("); ("QMARK", "?");
           ("IDENT", "optional"); ("COLON", ":"); ("IDENT", "string");
           ("RPAREN", ")"); ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " ");
           ("IDENT", "optional")])]
    
    if actual <> expected then
        printfn "actual   = %A" actual
        printfn "expected = %A" expected
        actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)