fix(offload): tracing/token hardening — astral token fix, L1.5 null fallback, createRequire

docs/README.md

@@ -0,0 +1,34 @@
+# fix(offload): Tracing & Token Hardening
+
+拆分自 #391，per [YOMXXX review](https://github.com/TencentCloud/TencentDB-Agent-Memory/pull/391#issuecomment-4923202779)。
+
+## 修复范围
+
+聚焦 `src/offload/` 层：token estimate、local LLM、Opik tracing。不涉及 store、runtime、gateway 层。
+
+## 具体修复
+
+### `src/offload/fast-token-estimate.ts` — Astral-平面 Token 修正
+
+| 问题 | 修复 |
+|:---|:---|
+| astral-plane 字符（emoji / CJK extension B，U+10000+）编码为 surrogate pair，原实现把 high surrogate 和 low surrogate 各计为一个 token | high surrogate 检测后跳过低代理（`i += 2`），修正 ~2× token 高估 |
+
+### `src/offload/local-llm/index.ts` — L1.5 Fallback 修正
+
+| 问题 | 修复 |
+|:---|:---|
+| 解析失败返回 `{taskCompleted: false, ...}` — normalizeJudgment 把 `false` 当有效判断，永远不触发 "LLM 不可用" 重试 | 改为 `{taskCompleted: null, ...}`，`== null` 检查触发重试路径 |
+
+### `src/offload/opik-tracer.ts` — ESM 兼容 + Trace 截断
+
+| 问题 | 修复 |
+|:---|:---|
+| ESM 下 `require("opik")` 为 `undefined`，tracer 永久静默关闭 | 改为 `createRequire(import.meta.url)` |
+| 未脱敏 tool 输出（文件内容、环境变量）完整写入 trace，泄露到 Opik backend | 统一截断到 2000 字符 |
+
+## 验证
+
+- 3 files, +41/-15
+- 无 API 变更，无新增依赖
+- 与 #39 / #232 / #242 / #287 / #288 / #289 / #347 均不重叠

src/offload/fast-token-estimate.ts

@@ -283,7 +283,14 @@ export function fastEstimateTokens(text: string): number {
     if (cp >= 0x21 && cp <= 0x7E) { tokens += 0.6; i++; continue; }
 
     // ── Other Unicode (emoji etc.) ──
-    if (cp > 0x7F) { tokens += 2.5; i++; continue; }
+    if (cp > 0x7F) {
+      tokens += 2.5;
+      // Astral-plane characters (U+10000+) are encoded as a surrogate pair;
+      // skip the low surrogate so it isn't counted again as a second token
+      // (previously emoji / CJK extension-B were ~2× over-estimated).
+      i += cp >= 0xD800 && cp <= 0xDBFF && i + 1 < n ? 2 : 1;
+      continue;
+    }
 
     i++;
   }

src/offload/local-llm/index.ts

@@ -115,11 +115,13 @@ export class LocalLlmClient {
     if (!result) {
       this.logger?.warn?.(`${TAG} L1.5: failed to parse judgment from LLM response (${raw.length} chars)`);
       // Return all-null to trigger normalizeJudgment's "LLM unavailable" path
+      // (which checks `== null` on each field). Returning `false` here would be
+      // treated as a real no-op judgment and never trigger the retry path.
       return {
-        taskCompleted: false,
-        isContinuation: false,
-        isLongTask: false,
-      } as L15Response;
+        taskCompleted: null,
+        isContinuation: null,
+        isLongTask: null,
+      } as unknown as L15Response;
     }
 
     return result as L15Response;

src/offload/opik-tracer.ts

@@ -4,6 +4,7 @@
  */
 import type { PluginLogger } from "./types.js";
 import { getEnv } from "../utils/env.js";
+import { createRequire } from "node:module";
 
 // Opik client types (minimal shape to avoid hard dependency)
 interface OpikClient {
@@ -24,6 +25,18 @@ let client: OpikClient | null = null;
 let tracerEnabled = false;
 let tracerInitTried = false;
 
+/**
+ * Max characters of any single string field emitted to Opik. Tool I/O (file
+ * contents, env vars, command output) routinely contains secrets; truncating
+ * limits (though does not eliminate) leakage to the Opik backend.
+ */
+const MAX_TRACE_FIELD_CHARS = 2000;
+
+function truncateForTrace(s: string): string {
+  if (typeof s !== "string" || s.length <= MAX_TRACE_FIELD_CHARS) return s;
+  return s.slice(0, MAX_TRACE_FIELD_CHARS) + `…[truncated ${s.length - MAX_TRACE_FIELD_CHARS} chars]`;
+}
+
 function extractLayerTag(stage: string): string {
   const match = stage.match(/^(L\d+(?:\.\d+)?)/i);
   if (!match) return "Lx-unknown";
@@ -96,8 +109,11 @@ export function initOffloadOpikTracer(
     // Dynamic import — graceful when opik is not installed
     let OpikConstructor: new (params: Record<string, unknown>) => OpikClient;
     try {
-      // eslint-disable-next-line @typescript-eslint/no-require-imports
-      const opikModule = require("opik") as { Opik: new (params: Record<string, unknown>) => OpikClient };
+      // createRequire lets an ESM module load a CommonJS dependency. A bare
+      // require() is undefined under Node ESM and silently disabled the tracer
+      // even when opik was installed.
+      const moduleRequire = createRequire(import.meta.url);
+      const opikModule = moduleRequire("opik") as { Opik: new (params: Record<string, unknown>) => OpikClient };
       OpikConstructor = opikModule.Opik;
     } catch {
       logger.debug?.("[context-offload] opik package not available, tracer disabled");
@@ -173,7 +189,8 @@ export function traceOffloadDecision(params: {
 
 /**
  * Serialize a single message into a diagnostic object for tracing.
- * Outputs full content text (no truncation) for debugging purposes.
+ * Content is truncated per MAX_TRACE_FIELD_CHARS so tool I/O (which may carry
+ * secrets) is not shipped verbatim to the Opik backend.
  */
 function serializeMessageForTrace(msg: any, index: number): Record<string, unknown> {
   const role = msg.role ?? msg.message?.role ?? msg.type ?? "unknown";
@@ -187,18 +204,18 @@ function serializeMessageForTrace(msg: any, index: number): Record<string, unkno
   let contentLength: number;
   if (typeof content === "string") {
     contentLength = content.length;
-    contentText = content;
+    contentText = truncateForTrace(content);
   } else if (Array.isArray(content)) {
     const parts: string[] = [];
     for (const c of content) {
       if (typeof c !== "object" || c === null) continue;
       if (c.type === "text" && typeof c.text === "string") {
-        parts.push(c.text);
+        parts.push(truncateForTrace(c.text));
       } else if (c.type === "tool_use") {
-        const inputStr = c.input != null ? JSON.stringify(c.input) : "";
+        const inputStr = c.input != null ? truncateForTrace(JSON.stringify(c.input)) : "";
         parts.push(`[tool_use: ${c.name ?? "?"} id=${c.id ?? "?"} input=${inputStr}]`);
       } else if (c.type === "tool_result") {
-        const resultStr = typeof c.content === "string" ? c.content : JSON.stringify(c.content ?? "");
+        const resultStr = typeof c.content === "string" ? truncateForTrace(c.content) : truncateForTrace(JSON.stringify(c.content ?? ""));
         parts.push(`[tool_result: id=${c.tool_use_id ?? "?"} content=${resultStr}]`);
       } else {
         parts.push(`[${c.type ?? "unknown_block"}]`);
@@ -339,8 +356,8 @@ export function traceOffloadModelIo(params: {
       provider: params.provider,
       input: {
         url: params.url,
-        systemPrompt: params.systemPrompt,
-        userPrompt: params.userPrompt,
+        systemPrompt: truncateForTrace(params.systemPrompt),
+        userPrompt: truncateForTrace(params.userPrompt),
       },
       metadata: {
         stage: params.stage,
@@ -352,7 +369,7 @@ export function traceOffloadModelIo(params: {
     });
     span.update({
       output: {
-        responseContent: params.responseContent,
+        responseContent: truncateForTrace(params.responseContent),
         usage: params.usage,
         durationMs: dur,
         duration: durStr,