diff --git a/lark/exceptions.py b/lark/exceptions.py index b45bfe4ac..a0fdff018 100644 --- a/lark/exceptions.py +++ b/lark/exceptions.py @@ -1,6 +1,6 @@ from .utils import logger, NO_VALUE from typing import Mapping, Iterable, Callable, Union, TypeVar, Tuple, Any, List, Set, Optional, Collection, TYPE_CHECKING - +import unicodedata if TYPE_CHECKING: from .lexer import Token from .parsers.lalr_interactive_parser import InteractiveParser @@ -20,6 +20,11 @@ def assert_config(value, options: Collection, msg='Got %r, expected one of %s'): if value not in options: raise ConfigurationError(msg % (value, options)) +def _display_width(s): + width = 0 + for ch in s: + width += 2 if unicodedata.east_asian_width(ch) in ('F', 'W') else 1 + return width class GrammarError(LarkError): pass @@ -66,7 +71,7 @@ def get_context(self, text: str, span: int=40) -> str: if not isinstance(text, bytes): before = text[start:pos].rsplit('\n', 1)[-1] after = text[pos:end].split('\n', 1)[0] - return before + after + '\n' + ' ' * len(before.expandtabs()) + '^\n' + return before + after + '\n' + ' ' * _display_width(before.expandtabs()) + '^\n' else: before = text[start:pos].rsplit(b'\n', 1)[-1] after = text[pos:end].split(b'\n', 1)[0] diff --git a/tests/test_parser.py b/tests/test_parser.py index 533ae7890..c89e2a10e 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -204,6 +204,36 @@ def NUM(self, token): result = parser.parse(text) self.assertEqual(result, expected) + def test_fullwidth_error_context_alignment(self): + + grammar = r""" + start: CHINESE WORD WORD + + CHINESE: "你好" + WORD: /[a-zA-Z]+/ + + %ignore " " + """ + + parser = Lark(grammar) + + text = "你好 hello 123" + + try: + parser.parse(text) + assert False + except Exception as e: + context = e.get_context(text) + + lines = context.splitlines() + + assert len(lines) >= 2 + + caret_pos = lines[1].index("^") + + # ensure full-width chars are counted properly + assert caret_pos >= 11 + def test_vargs_meta(self): @v_args(meta=True)