From 5aee68840621d22616e480adc7afbcf789732202 Mon Sep 17 00:00:00 2001
From: Vincent Gao <gaobing1230@gmail.com>
Date: Sat, 20 Jun 2026 22:09:07 +0200
Subject: [PATCH] Preserve end_line/end_column/end_pos when copying or pickling
 a Token

Token.__reduce__ and Token.__deepcopy__ passed only the first five
positional attributes (type, value, start_pos, line, column) to the
constructor, so copy.copy, copy.deepcopy and pickle round-trips silently
reset end_line, end_column and end_pos to None. new_borrow_pos already
carries all eight attributes; these two methods predate the end_* fields
and were never updated to match.
---
 lark/lexer.py       |  4 ++--
 tests/test_lexer.py | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/lark/lexer.py b/lark/lexer.py
index 9aa10c2e2..2f92966a8 100644
--- a/lark/lexer.py
+++ b/lark/lexer.py
@@ -254,13 +254,13 @@ def new_borrow_pos(cls: Type[_T], type_: str, value: Any, borrow_t: 'Token') ->
         return cls(type_, value, borrow_t.start_pos, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos)
 
     def __reduce__(self):
-        return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column))
+        return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column, self.end_line, self.end_column, self.end_pos))
 
     def __repr__(self):
         return 'Token(%r, %r)' % (self.type, self.value)
 
     def __deepcopy__(self, memo):
-        return Token(self.type, self.value, self.start_pos, self.line, self.column)
+        return Token(self.type, self.value, self.start_pos, self.line, self.column, self.end_line, self.end_column, self.end_pos)
 
     def __eq__(self, other):
         if isinstance(other, Token) and self.type != other.type:
diff --git a/tests/test_lexer.py b/tests/test_lexer.py
index cf9dcc48e..cb1251413 100644
--- a/tests/test_lexer.py
+++ b/tests/test_lexer.py
@@ -1,12 +1,26 @@
+import copy
+import pickle
 from unittest import TestCase, main
 
 from lark import Lark, Tree, TextSlice
+from lark.lexer import Token
 
 
 class TestLexer(TestCase):
     def setUp(self):
         pass
 
+    def test_token_copy_preserves_end_position(self):
+        # deepcopy/copy/pickle round-trips must keep the end_* position
+        # attributes, like new_borrow_pos does, instead of dropping them.
+        t = Token("WORD", "hello", start_pos=0, line=1, column=1,
+                  end_line=2, end_column=6, end_pos=5)
+        attrs = ("type", "value", "start_pos", "line", "column",
+                 "end_line", "end_column", "end_pos")
+        expected = [getattr(t, a) for a in attrs]
+        for copied in (copy.deepcopy(t), copy.copy(t), pickle.loads(pickle.dumps(t))):
+            self.assertEqual([getattr(copied, a) for a in attrs], expected)
+
     def test_basic(self):
         p = Lark("""
             start: "a" "b" "c" "d"