feat: semantic highlighting support (#640)

* feat: add semantic token

* fix: custom token and enhancment

* fix: unit tests

* fix: coverage

* chore: change set

* fix: no semantic for none xml view files

.changeset/curvy-worms-rhyme.md

@@ -0,0 +1,8 @@
+---
+"@ui5-language-assistant/vscode-ui5-language-assistant-bas-ext": patch
+"vscode-ui5-language-assistant": patch
+"@ui5-language-assistant/binding-parser": patch
+"@ui5-language-assistant/binding": patch
+---
+
+add semantic highlighting support

packages/binding-parser/src/api.ts

@@ -10,7 +10,8 @@ export {
   isBindingExpression,
   isMetadataPath,
   isModel,
-  isPropertyBindingInfo,
+  isBindingAllowed,
+  extractBindingSyntax,
 } from "./utils";
 
 import {

packages/binding-parser/src/lexer/token.ts

@@ -61,7 +61,7 @@ const colon = createToken({
 });
 const stringValue = createToken({
   name: STRING_VALUE,
-  pattern: /(?:'|"|'|")(?:.*?)(?:'|"|'|")/,
+  pattern: /(?:'|"|'|")(\\'|\\"|.)*?(?:'|"|'|")/,
 });
 
 const booleanValue = createToken({

packages/binding-parser/src/types/index.ts

@@ -1 +1,7 @@
 export * as BindingParserTypes from "./binding-parser";
+
+export interface ExtractBindingSyntax {
+  startIndex: number;
+  endIndex: number;
+  expression: string;
+}

packages/binding-parser/src/utils/expression.ts

@@ -1,3 +1,4 @@
+import { ExtractBindingSyntax } from "../types";
 import type {
   ParseResultErrors,
   StructureValue,
@@ -92,7 +93,7 @@ export const isMetadataPath = (
 };
 
 /**
- * An input is considered property binding syntax when
+ * An input is considered as an allowed binding when
  *
  * a. is empty curly bracket e.g  `{}` or `{   }`
  *
@@ -104,7 +105,7 @@ export const isMetadataPath = (
  *
  * e. is not OData path e.g {/path/to/...} or {path/to/...}
  */
-export const isPropertyBindingInfo = (
+export const isBindingAllowed = (
   input: string,
   binding?: StructureValue,
   errors?: ParseResultErrors
@@ -146,3 +147,67 @@ export const isPropertyBindingInfo = (
   }
   return false;
 };
+
+/**
+ * Regular expression to extract binding syntax.
+ *
+ * Also handles escaping of '{' and '}'.
+ */
+// eslint-disable-next-line no-useless-escape
+const start = /(\\[\\\{\}])|(\{)/g;
+// eslint-disable-next-line no-useless-escape
+const end = /(\\[\\\{\}])|(\})/g;
+
+export const extractBindingSyntax = (input: string): ExtractBindingSyntax[] => {
+  const result: ExtractBindingSyntax[] = [];
+  let startRegResult: RegExpExecArray | null;
+  let endRegResult: RegExpExecArray | null;
+  // resetting
+  start.lastIndex = 0;
+  let startIndex = 0;
+  let lastIndex = 0;
+  let endIndex = 0;
+  const text = input;
+  if (text.trim() === "") {
+    return [{ startIndex, endIndex, expression: input }];
+  }
+  while ((startRegResult = start.exec(input)) !== null) {
+    // scape special chars
+    if (startRegResult[1]) {
+      continue;
+    }
+    const startInput = input.slice(startRegResult.index);
+    // collect all closing bracket(s)
+    end.lastIndex = 0;
+    while ((endRegResult = end.exec(startInput)) !== null) {
+      // scape special chars
+      if (endRegResult[1]) {
+        break;
+      }
+      lastIndex = endRegResult.index;
+    }
+    if (lastIndex === startRegResult.index) {
+      // missing closing bracket
+      const expression = startInput.slice(0, input.length);
+      result.push({
+        startIndex: startRegResult.index,
+        endIndex: input.length,
+        expression,
+      });
+      input = startInput.slice(input.length);
+    } else {
+      const expression = startInput.slice(0, lastIndex + 1);
+      startIndex = endIndex + startRegResult.index;
+      endIndex = startIndex + lastIndex + 1;
+      result.push({
+        startIndex,
+        endIndex,
+        expression,
+      });
+      input = startInput.slice(lastIndex + 1);
+      // resetting
+      start.lastIndex = 0;
+    }
+  }
+  return result;
+};

packages/binding-parser/src/utils/index.ts

@@ -10,5 +10,6 @@ export {
   isBindingExpression,
   isMetadataPath,
   isModel,
-  isPropertyBindingInfo,
+  isBindingAllowed,
+  extractBindingSyntax,
 } from "./expression";

packages/binding-parser/test/unit/data/quotes/key-colon-value/escaped/double-quotes/ast.json

@@ -0,0 +1,64 @@
+{
+  "bindings": [
+    {
+      "leftCurly": {
+        "type": "left-curly",
+        "text": "{",
+        "range": "[(0,0)..(0,1)]"
+      },
+      "elements": [
+        {
+          "key": {
+            "type": "key",
+            "text": "$filter",
+            "range": "[(1,4)..(1,11)]",
+            "originalText": "$filter"
+          },
+          "colon": {
+            "type": "colon",
+            "text": ":",
+            "range": "[(1,12)..(1,13)]"
+          },
+          "value": {
+            "type": "string-value",
+            "text": "\"TravelStatus_code eq \\\"O\\\" and IsActiveEntity eq false or SiblingEntity/IsActiveEntity eq null\"",
+            "range": "[(1,14)..(1,110)]"
+          },
+          "range": "[(1,4)..(1,110)]",
+          "type": "structure-element"
+        }
+      ],
+      "rightCurly": {
+        "type": "right-curly",
+        "text": "}",
+        "range": "[(2,0)..(2,1)]"
+      },
+      "range": "[(0,0)..(2,1)]",
+      "commas": [],
+      "type": "structure-value"
+    }
+  ],
+  "spaces": [
+    {
+      "type": "white-space",
+      "text": "\n    ",
+      "range": "[(0,1)..(1,4)]"
+    },
+    {
+      "type": "white-space",
+      "text": " ",
+      "range": "[(1,11)..(1,12)]"
+    },
+    {
+      "type": "white-space",
+      "text": " ",
+      "range": "[(1,13)..(1,14)]"
+    },
+    {
+      "type": "white-space",
+      "text": "\n",
+      "range": "[(1,110)..(1,111)]"
+    }
+  ],
+  "type": "template"
+}

packages/binding-parser/test/unit/data/quotes/key-colon-value/escaped/double-quotes/cst.json

@@ -0,0 +1,82 @@
+{
+  "name": "template",
+  "children": {
+    "object": [
+      {
+        "name": "object",
+        "children": {
+          "left-curly": [
+            {
+              "image": "{",
+              "startColumn": 1,
+              "endColumn": 1,
+              "tokenTypeName": "left-curly"
+            }
+          ],
+          "object-item": [
+            {
+              "name": "object-item",
+              "children": {
+                "key": [
+                  {
+                    "image": "$filter",
+                    "startColumn": 5,
+                    "endColumn": 11,
+                    "tokenTypeName": "key"
+                  }
+                ],
+                "colon": [
+                  {
+                    "image": ":",
+                    "startColumn": 13,
+                    "endColumn": 13,
+                    "tokenTypeName": "colon"
+                  }
+                ],
+                "value": [
+                  {
+                    "name": "value",
+                    "children": {
+                      "string-value": [
+                        {
+                          "image": "\"TravelStatus_code eq \\\"O\\\" and IsActiveEntity eq false or SiblingEntity/IsActiveEntity eq null\"",
+                          "startColumn": 15,
+                          "endColumn": 110,
+                          "tokenTypeName": "string-value"
+                        }
+                      ]
+                    },
+                    "location": {
+                      "startColumn": 15,
+                      "endColumn": 110
+                    }
+                  }
+                ]
+              },
+              "location": {
+                "startColumn": 5,
+                "endColumn": 110
+              }
+            }
+          ],
+          "right-curly": [
+            {
+              "image": "}",
+              "startColumn": 1,
+              "endColumn": 1,
+              "tokenTypeName": "right-curly"
+            }
+          ]
+        },
+        "location": {
+          "startColumn": 1,
+          "endColumn": 1
+        }
+      }
+    ]
+  },
+  "location": {
+    "startColumn": 1,
+    "endColumn": 1
+  }
+}

packages/binding-parser/test/unit/data/quotes/key-colon-value/escaped/double-quotes/input.txt

@@ -0,0 +1,3 @@
+{
+    $filter : "TravelStatus_code eq \"O\" and IsActiveEntity eq false or SiblingEntity/IsActiveEntity eq null"
+}

packages/binding-parser/test/unit/data/quotes/key-colon-value/escaped/double-quotes/lexer-errors.json

@@ -0,0 +1 @@
+[]

packages/binding-parser/test/unit/data/quotes/key-colon-value/escaped/double-quotes/parse-errors.json

@@ -0,0 +1 @@
+[]

packages/binding-parser/test/unit/data/quotes/key-colon-value/escaped/single-quotes/ast.json

@@ -0,0 +1,70 @@
+{
+  "bindings": [
+    {
+      "leftCurly": {
+        "type": "left-curly",
+        "text": "{",
+        "range": "[(0,0)..(0,1)]"
+      },
+      "elements": [
+        {
+          "key": {
+            "type": "key",
+            "text": "$filter",
+            "range": "[(1,4)..(1,11)]",
+            "originalText": "$filter"
+          },
+          "colon": {
+            "type": "colon",
+            "text": ":",
+            "range": "[(1,12)..(1,13)]"
+          },
+          "value": {
+            "type": "string-value",
+            "text": "'TravelStatus_code eq \\'O\\' and IsActiveEntity eq false or SiblingEntity/IsActiveEntity eq null'",
+            "range": "[(1,14)..(1,110)]"
+          },
+          "range": "[(1,4)..(1,110)]",
+          "type": "structure-element"
+        }
+      ],
+      "rightCurly": {
+        "type": "right-curly",
+        "text": "}",
+        "range": "[(2,0)..(2,1)]"
+      },
+      "range": "[(0,0)..(2,1)]",
+      "commas": [
+        {
+          "type": "comma",
+          "text": ",",
+          "range": "[(1,110)..(1,111)]"
+        }
+      ],
+      "type": "structure-value"
+    }
+  ],
+  "spaces": [
+    {
+      "type": "white-space",
+      "text": "\n    ",
+      "range": "[(0,1)..(1,4)]"
+    },
+    {
+      "type": "white-space",
+      "text": " ",
+      "range": "[(1,11)..(1,12)]"
+    },
+    {
+      "type": "white-space",
+      "text": " ",
+      "range": "[(1,13)..(1,14)]"
+    },
+    {
+      "type": "white-space",
+      "text": "\n",
+      "range": "[(1,111)..(1,112)]"
+    }
+  ],
+  "type": "template"
+}