Preserve end_line/end_column/end_pos when copying or pickling a Token

lark/lexer.py

@@ -254,13 +254,13 @@ def new_borrow_pos(cls: Type[_T], type_: str, value: Any, borrow_t: 'Token') ->
         return cls(type_, value, borrow_t.start_pos, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos)
 
     def __reduce__(self):
-        return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column))
+        return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column, self.end_line, self.end_column, self.end_pos))
 
     def __repr__(self):
         return 'Token(%r, %r)' % (self.type, self.value)
 
     def __deepcopy__(self, memo):
-        return Token(self.type, self.value, self.start_pos, self.line, self.column)
+        return Token(self.type, self.value, self.start_pos, self.line, self.column, self.end_line, self.end_column, self.end_pos)
 
     def __eq__(self, other):
         if isinstance(other, Token) and self.type != other.type:

tests/test_lexer.py

@@ -1,12 +1,26 @@
+import copy
+import pickle
 from unittest import TestCase, main
 
 from lark import Lark, Tree, TextSlice
+from lark.lexer import Token
 
 
 class TestLexer(TestCase):
     def setUp(self):
         pass
 
+    def test_token_copy_preserves_end_position(self):
+        # deepcopy/copy/pickle round-trips must keep the end_* position
+        # attributes, like new_borrow_pos does, instead of dropping them.
+        t = Token("WORD", "hello", start_pos=0, line=1, column=1,
+                  end_line=2, end_column=6, end_pos=5)
+        attrs = ("type", "value", "start_pos", "line", "column",
+                 "end_line", "end_column", "end_pos")
+        expected = [getattr(t, a) for a in attrs]
+        for copied in (copy.deepcopy(t), copy.copy(t), pickle.loads(pickle.dumps(t))):
+            self.assertEqual([getattr(copied, a) for a in attrs], expected)
+
     def test_basic(self):
         p = Lark("""
             start: "a" "b" "c" "d"