Skip to content

Commit a0fe4d0

Browse files
syzsunshine219MemOS AutoDevclaudeendxxxx
authored
fix(memory): decode legacy JSON info metadata (#2602)
* fix(graph_dbs): deserialize internal_info / info on Neo4j read path (#2288) `_sanitize_neo4j_value` serializes every dict-typed metadata value into a JSON string on write, but `_parse_node` / `_parse_nodes` only reversed the `sources` transform. `internal_info` and `info` therefore came back as `str`, breaking `TextualMemoryItem`'s `dict | None` validation during recall on document-ingest / Dream-enabled deployments. Adds a small `_deserialize_dict_field` helper (symmetric with the write-side sanitizer) and wires it into both `Neo4jGraphDB._parse_node` and `Neo4jCommunityGraphDB._parse_node` / `_parse_nodes` for the two known dict-typed fields. JSON-object-shaped strings deserialize to dicts, malformed strings fall back to None so pydantic still accepts them, all other values pass through unchanged. Write path and other graph backends untouched. Adds 16 regression tests including an end-to-end guard that feeds the parser output into `TreeNodeTextualMemoryMetadata` to close the loop with the failure point. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(graph_dbs): log dropped internal_info deserialize + tidy test helper Address two OCR findings on #2289: 1. src/memos/graph_dbs/neo4j.py: `_deserialize_dict_field` silently dropped malformed dict-shaped strings to None, making the event indistinguishable from a legitimately-None field. Emit a `logger.warning` before returning None so operators can trace the corruption. Uses the module-level logger already declared at line 13. 2. tests/graph_dbs/test_neo4j_internal_info_roundtrip.py: `TestDeserializeDictField._helper` is a stable module-level lookup with no `self` use — promote it to `@staticmethod` for idiomatic pytest style. Verified: pytest tests/graph_dbs/ (47 passed, 3 pre-existing skips) + ruff check/format clean on both files. --------- Co-authored-by: MemOS AutoDev <autodev@memos.local> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: Wenqiang Wei <46308778+endxxxx@users.noreply.github.com>
1 parent ad9cf7b commit a0fe4d0

3 files changed

Lines changed: 137 additions & 15 deletions

File tree

‎src/memos/memories/textual/item.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from datetime import datetime
88
from typing import Any, Literal
99

10-
from pydantic import BaseModel, ConfigDict, Field, field_validator
10+
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
1111

1212

1313
ALLOWED_ROLES = {"user", "assistant", "system"}
@@ -159,14 +159,14 @@ class TextualMemoryMetadata(BaseModel):
159159
description="Internal algorithm metadata reserved for system use.",
160160
)
161161

162-
@field_validator("internal_info", mode="before")
162+
@field_validator("internal_info", "info", mode="before")
163163
@classmethod
164-
def _decode_internal_info(cls, value: Any) -> Any:
164+
def _decode_dict_metadata(cls, value: Any, validation_info: ValidationInfo) -> Any:
165165
# Neo4j stores dictionaries as JSON strings; decode them before model validation.
166166
if isinstance(value, str):
167167
value = json.loads(value)
168168
if not isinstance(value, dict):
169-
raise ValueError("internal_info JSON must decode to an object")
169+
raise ValueError(f"{validation_info.field_name} JSON must decode to an object")
170170
return value
171171

172172
model_config = ConfigDict(extra="allow")
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
"""Legacy info decoding shares the existing strict metadata validation boundary."""
2+
3+
import json
4+
5+
from types import SimpleNamespace
6+
from unittest.mock import MagicMock
7+
from uuid import uuid4
8+
9+
import pytest
10+
11+
from pydantic import ValidationError
12+
13+
from memos.graph_dbs.neo4j import (
14+
Neo4jGraphDB,
15+
_flatten_info_fields,
16+
_sanitize_neo4j_metadata,
17+
)
18+
from memos.graph_dbs.neo4j_community import Neo4jCommunityGraphDB
19+
from memos.memories.textual.item import TextualMemoryItem
20+
21+
22+
@pytest.fixture(params=["neo4j", "community", "community_batch"])
23+
def parse_nodes(request):
24+
cls = Neo4jGraphDB if request.param == "neo4j" else Neo4jCommunityGraphDB
25+
db = cls.__new__(cls)
26+
if isinstance(db, Neo4jCommunityGraphDB):
27+
db.vec_db = MagicMock()
28+
db.vec_db.get_by_id.return_value = None
29+
db.vec_db.get_by_ids.return_value = []
30+
if request.param == "community_batch":
31+
return db._parse_nodes
32+
33+
def parse(nodes):
34+
return [db._parse_node(node) for node in nodes]
35+
36+
return parse
37+
38+
39+
def make_node():
40+
return {
41+
"id": str(uuid4()),
42+
"memory": "Synthetic legacy memory",
43+
"memory_type": "LongTermMemory",
44+
"internal_info": json.dumps({"chunk_index": 0}),
45+
"legacy_flat_key": "preserved",
46+
}
47+
48+
49+
@pytest.mark.parametrize(
50+
("stored_info", "expected"),
51+
[
52+
(None, None),
53+
({}, {}),
54+
({"nested": {"values": [True, None]}}, {"nested": {"values": [True, None]}}),
55+
("{}", {}),
56+
(
57+
' {"nested": {"text": "历史数据", "values": [true, null]}}\n',
58+
{"nested": {"text": "历史数据", "values": [True, None]}},
59+
),
60+
],
61+
)
62+
def test_legacy_info_validates_on_all_graph_read_paths(parse_nodes, stored_info, expected):
63+
node = make_node()
64+
node["info"] = stored_info
65+
parsed = parse_nodes([node])[0]
66+
67+
item = TextualMemoryItem.from_dict(parsed)
68+
69+
assert item.id == node["id"]
70+
assert item.memory == node["memory"]
71+
assert item.metadata.info == expected
72+
assert item.metadata.internal_info == {"chunk_index": 0}
73+
assert item.metadata.legacy_flat_key == "preserved"
74+
assert node["info"] == stored_info
75+
76+
77+
@pytest.mark.parametrize("field", ["internal_info", "info"])
78+
@pytest.mark.parametrize("value", ["{not json}", "plain text", "[]", "null", "1", '""'])
79+
def test_corrupt_dict_metadata_is_rejected_without_dropping_it(parse_nodes, field, value):
80+
node = make_node()
81+
node[field] = value
82+
parsed = parse_nodes([node])[0]
83+
84+
assert parsed["metadata"][field] == value
85+
with pytest.raises(ValidationError):
86+
TextualMemoryItem.from_dict(parsed)
87+
88+
89+
def test_missing_info_and_modern_flattened_data_remain_readable(parse_nodes):
90+
node = make_node()
91+
node.update(_sanitize_neo4j_metadata(_flatten_info_fields({"info": {"client_key": "value"}})))
92+
93+
parsed = parse_nodes([node])[0]
94+
item = TextualMemoryItem.from_dict(parsed)
95+
96+
assert "info" not in parsed["metadata"]
97+
assert item.metadata.info is None
98+
assert item.metadata.client_key == "value"
99+
100+
101+
def test_community_batch_preserves_embedding_and_info_alignment():
102+
db = Neo4jCommunityGraphDB.__new__(Neo4jCommunityGraphDB)
103+
db.vec_db = MagicMock()
104+
nodes = [make_node(), make_node()]
105+
for index, node in enumerate(nodes):
106+
node["info"] = json.dumps({"index": index})
107+
db.vec_db.get_by_ids.return_value = [
108+
SimpleNamespace(id=nodes[1]["id"], vector=[0.2]),
109+
SimpleNamespace(id=nodes[0]["id"], vector=[0.1]),
110+
]
111+
112+
items = [TextualMemoryItem.from_dict(node) for node in db._parse_nodes(nodes)]
113+
114+
assert [item.metadata.info for item in items] == [{"index": 0}, {"index": 1}]
115+
assert [item.metadata.embedding for item in items] == [[0.1], [0.2]]

‎tests/memories/textual/test_item.py‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,38 +26,42 @@
2626
PreferenceTextualMemoryMetadata,
2727
],
2828
)
29-
def test_internal_info_decodes_json_object(metadata_cls):
29+
@pytest.mark.parametrize("field", ["internal_info", "info"])
30+
def test_dict_metadata_decodes_json_object(metadata_cls, field):
3031
value = {"ingest_batch_id": "test-batch", "nested": {"chunk_index": 0, "values": [True, None]}}
3132

32-
metadata = metadata_cls(internal_info=json.dumps(value))
33+
metadata = metadata_cls(**{field: " " + json.dumps(value) + "\n"})
3334

34-
assert metadata.internal_info == value
35-
assert metadata.model_dump()["internal_info"] == value
35+
assert getattr(metadata, field) == value
36+
assert metadata.model_dump()[field] == value
3637

3738

3839
@pytest.mark.parametrize("value", [None, {}, {"ingest_batch_id": "test-batch", "nested": [1, 2]}])
39-
def test_internal_info_preserves_existing_values(value):
40+
@pytest.mark.parametrize("field", ["internal_info", "info"])
41+
def test_dict_metadata_preserves_existing_values(value, field):
4042
original = copy.deepcopy(value)
4143

42-
metadata = TextualMemoryMetadata(internal_info=value)
44+
metadata = TextualMemoryMetadata(**{field: value})
4345

44-
assert metadata.internal_info == original
46+
assert getattr(metadata, field) == original
4547
assert value == original
4648

4749

4850
@pytest.mark.parametrize(
4951
"value",
5052
["{", "not-json", "[]", "null", "1", "true", '"text"', json.dumps('{"batch": "test"}')],
5153
)
52-
def test_internal_info_rejects_malformed_json_and_encoded_non_objects(value):
54+
@pytest.mark.parametrize("field", ["internal_info", "info"])
55+
def test_dict_metadata_rejects_malformed_json_and_encoded_non_objects(value, field):
5356
with pytest.raises(ValidationError):
54-
TextualMemoryMetadata(internal_info=value)
57+
TextualMemoryMetadata(**{field: value})
5558

5659

5760
@pytest.mark.parametrize("value", [[], [["batch", "test"]], 1, False])
58-
def test_internal_info_preserves_rejection_of_invalid_types(value):
61+
@pytest.mark.parametrize("field", ["internal_info", "info"])
62+
def test_dict_metadata_preserves_rejection_of_invalid_types(value, field):
5963
with pytest.raises(ValidationError):
60-
TextualMemoryMetadata(internal_info=value)
64+
TextualMemoryMetadata(**{field: value})
6165

6266

6367
def test_search_item_from_dict_decodes_internal_info_without_changing_other_fields():
@@ -68,6 +72,7 @@ def test_search_item_from_dict_decodes_internal_info_without_changing_other_fiel
6872
"metadata": {
6973
"memory_type": "LongTermMemory",
7074
"internal_info": json.dumps(value),
75+
"info": json.dumps({"client_metadata": True}),
7176
"relativity": 0.75,
7277
"sources": [{"type": "chat", "role": "user", "content": "Synthetic source."}],
7378
"session_id": "test-session",
@@ -78,6 +83,7 @@ def test_search_item_from_dict_decodes_internal_info_without_changing_other_fiel
7883
item = TextualMemoryItem.from_dict(record)
7984

8085
assert item.metadata.internal_info == value
86+
assert item.metadata.info == {"client_metadata": True}
8187
assert item.id == record["id"]
8288
assert item.memory == record["memory"]
8389
assert item.metadata.sources[0].content == "Synthetic source."
@@ -88,4 +94,5 @@ def test_search_item_from_dict_decodes_internal_info_without_changing_other_fiel
8894
# Graph storage serializes dictionaries as JSON strings again on the next write.
8995
stored = item.to_dict()
9096
stored["metadata"]["internal_info"] = json.dumps(stored["metadata"]["internal_info"])
97+
stored["metadata"]["info"] = json.dumps(stored["metadata"]["info"])
9198
assert TextualMemoryItem.from_dict(stored).to_dict() == item.to_dict()

0 commit comments

Comments
 (0)