Fix false negatives for escape sequence (#32)

- Fix false negatives for escape sequence
- Fix false negatives for octal and binary literals in JSON5.

Author: ota-meshi

package.json

@@ -15,7 +15,7 @@
         "test:base": "mocha --require ts-node/register \"tests/src/**/*.ts\" --reporter dot --timeout 60000",
         "test": "npm run test:base",
         "test:nyc": "nyc --reporter=lcov npm run test:base",
-        "test:debug": "mocha --require ts-node/register --inspect \"tests/src/**/*.ts\" --reporter dot",
+        "test:debug": "mocha --require ts-node/register/transpile-only --inspect \"tests/src/**/*.ts\" --reporter dot",
         "update": "ts-node ./tools/update.ts && npm run eslint-fix && npm run test:nyc",
         "preversion": "npm test && npm run update && git add .",
         "version": "npm run eslint-fix && git add .",
@@ -67,6 +67,7 @@
         "vue-eslint-parser": "^7.2.0"
     },
     "dependencies": {
+        "eslint-utils": "^2.1.0",
         "eslint-visitor-keys": "^2.0.0",
         "espree": "^6.0.0 || ^7.2.0"
     }

src/parser/convert.ts

@@ -13,6 +13,7 @@ import type {
     Expression,
 } from "estree"
 import type { AST } from "eslint"
+import { PatternMatcher } from "eslint-utils"
 import type {
     JSONNode,
     JSONProgram,
@@ -48,6 +49,10 @@ import type { TokenStore, MaybeNodeOrToken } from "./token-store"
 import { isComma } from "./token-store"
 
 const lineBreakPattern = /\r\n|[\r\n\u2028\u2029]/u
+const codePointEscapeMatcher = new PatternMatcher(/\\u\{[\da-fA-F]+\}/gu)
+const octalNumericLiteralPattern = /^0[oO]/u
+const legacyOctalNumericLiteralPattern = /^0[0-7]/u
+const binaryNumericLiteralPattern = /^0[bB]/u
 
 export type JSONSyntaxContext = {
     trailingCommas: boolean
@@ -59,6 +64,9 @@ export type JSONSyntaxContext = {
     infinities: boolean
     nans: boolean
     numericSeparators: boolean
+    binaryNumericLiterals: boolean
+    octalNumericLiterals: boolean
+    legacyOctalNumericLiterals: boolean
     invalidJsonNumbers: boolean
     //
     multilineStrings: boolean
@@ -70,6 +78,8 @@ export type JSONSyntaxContext = {
     regExpLiterals: boolean
     templateLiterals: boolean
     bigintLiterals: boolean
+    unicodeCodepointEscapes: boolean
+    escapeSequenceInIdentifier: boolean
 }
 
 export function convertNode(
@@ -141,7 +151,7 @@ export function convertNode(
         return convertUnaryExpressionNode(node, tokens, ctx)
     }
     if (node.type === "Identifier") {
-        return convertIdentifierNode(node, tokens)
+        return convertIdentifierNode(node, tokens, ctx)
     }
     if (node.type === "TemplateLiteral") {
         return convertTemplateLiteralNode(node, tokens, ctx)
@@ -461,6 +471,24 @@ function validateLiteral(node: Literal, ctx: JSONSyntaxContext) {
                 })
             }
         }
+        if (!ctx.octalNumericLiterals) {
+            if (octalNumericLiteralPattern.test(text)) {
+                return throwUnexpectedError("octal numeric literal", node)
+            }
+        }
+        if (!ctx.legacyOctalNumericLiterals) {
+            if (legacyOctalNumericLiteralPattern.test(text)) {
+                return throwUnexpectedError(
+                    "legacy octal numeric literal",
+                    node,
+                )
+            }
+        }
+        if (!ctx.binaryNumericLiterals) {
+            if (binaryNumericLiteralPattern.test(text)) {
+                return throwUnexpectedError("binary numeric literal", node)
+            }
+        }
         if (!ctx.invalidJsonNumbers) {
             try {
                 JSON.parse(text)
@@ -470,7 +498,9 @@ function validateLiteral(node: Literal, ctx: JSONSyntaxContext) {
         }
     }
     if (
-        (!ctx.multilineStrings || !ctx.singleQuotes) &&
+        (!ctx.multilineStrings ||
+            !ctx.singleQuotes ||
+            !ctx.unicodeCodepointEscapes) &&
         typeof value === "string"
     ) {
         if (!ctx.singleQuotes) {
@@ -483,6 +513,11 @@ function validateLiteral(node: Literal, ctx: JSONSyntaxContext) {
                 return throwUnexpectedError("multiline string", node)
             }
         }
+        if (!ctx.unicodeCodepointEscapes) {
+            if (codePointEscapeMatcher.test(node.raw!)) {
+                return throwUnexpectedError("unicode codepoint escape", node)
+            }
+        }
     }
 
     return undefined
@@ -547,11 +582,18 @@ function convertUnaryExpressionNode(
 function convertIdentifierNode(
     node: Identifier,
     tokens: TokenStore,
+    ctx: JSONSyntaxContext,
 ): JSONIdentifier {
     /* istanbul ignore next */
     if (node.type !== "Identifier") {
         return throwUnexpectedNodeError(node, tokens)
     }
+
+    if (!ctx.escapeSequenceInIdentifier) {
+        if (node.name.length < node.range![1] - node.range![0]) {
+            return throwUnexpectedError("escape sequence", node)
+        }
+    }
     const nn: JSONIdentifier = {
         type: "JSONIdentifier",
         name: node.name,
@@ -591,6 +633,11 @@ function convertTemplateLiteralNode(
         return throwUnexpectedTokenError("$", loc)
     }
 
+    if (!ctx.unicodeCodepointEscapes) {
+        if (codePointEscapeMatcher.test(node.quasis[0].value.raw)) {
+            return throwUnexpectedError("unicode codepoint escape", node)
+        }
+    }
     const quasis: [JSONTemplateElement] = [
         convertTemplateElementNode(node.quasis[0], tokens),
     ]

src/parser/parser.ts

@@ -170,6 +170,9 @@ function getJSONSyntaxContext(str?: string | null): JSONSyntaxContext {
             infinities: false,
             nans: false,
             numericSeparators: false,
+            binaryNumericLiterals: false,
+            octalNumericLiterals: false,
+            legacyOctalNumericLiterals: false,
             invalidJsonNumbers: false,
             multilineStrings: false,
             unquoteProperties: false,
@@ -180,6 +183,8 @@ function getJSONSyntaxContext(str?: string | null): JSONSyntaxContext {
             regExpLiterals: false,
             templateLiterals: false,
             bigintLiterals: false,
+            unicodeCodepointEscapes: false,
+            escapeSequenceInIdentifier: false,
         }
     }
     if (upperCase === "JSONC") {
@@ -192,6 +197,9 @@ function getJSONSyntaxContext(str?: string | null): JSONSyntaxContext {
             infinities: false,
             nans: false,
             numericSeparators: false,
+            binaryNumericLiterals: false,
+            octalNumericLiterals: false,
+            legacyOctalNumericLiterals: false,
             invalidJsonNumbers: false,
             multilineStrings: false,
             unquoteProperties: false,
@@ -202,6 +210,8 @@ function getJSONSyntaxContext(str?: string | null): JSONSyntaxContext {
             regExpLiterals: false,
             templateLiterals: false,
             bigintLiterals: false,
+            unicodeCodepointEscapes: false,
+            escapeSequenceInIdentifier: false,
         }
     }
     if (upperCase === "JSON5") {
@@ -214,6 +224,9 @@ function getJSONSyntaxContext(str?: string | null): JSONSyntaxContext {
             infinities: true,
             nans: true,
             numericSeparators: false,
+            binaryNumericLiterals: false,
+            octalNumericLiterals: false,
+            legacyOctalNumericLiterals: false,
             invalidJsonNumbers: true,
             multilineStrings: true,
             unquoteProperties: true,
@@ -224,6 +237,8 @@ function getJSONSyntaxContext(str?: string | null): JSONSyntaxContext {
             regExpLiterals: false,
             templateLiterals: false,
             bigintLiterals: false,
+            unicodeCodepointEscapes: false,
+            escapeSequenceInIdentifier: false,
         }
     }
     return {
@@ -235,6 +250,9 @@ function getJSONSyntaxContext(str?: string | null): JSONSyntaxContext {
         infinities: true,
         nans: true,
         numericSeparators: true,
+        binaryNumericLiterals: true,
+        octalNumericLiterals: true,
+        legacyOctalNumericLiterals: true,
         invalidJsonNumbers: true,
         multilineStrings: true,
         unquoteProperties: true,
@@ -245,5 +263,7 @@ function getJSONSyntaxContext(str?: string | null): JSONSyntaxContext {
         regExpLiterals: true,
         templateLiterals: true,
         bigintLiterals: true,
+        unicodeCodepointEscapes: true,
+        escapeSequenceInIdentifier: true,
     }
 }

tests/fixtures/parser/ast/unicode-codepoint-escape01-input.json6

@@ -0,0 +1 @@
+{"\u{31}":"\u{31}"}