Skip to content

Commit 46e73a3

Browse files
author
Sorra
committed
SB-0MNM9ZYAZ000R2IZ: Support inline/reply ingestion; document usage and add tests
1 parent 88b4876 commit 46e73a3

File tree

3 files changed

+291
-0
lines changed

3 files changed

+291
-0
lines changed

README.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,29 @@ SourceBase automatically extracts URLs from Discord messages, fetches content me
2020
- 📊 **Backfill Queue**: Automatic retry for failed operations with SLA tracking
2121
- 🎭 **Discord Reactions**: Success/failure feedback on message processing
2222

23+
### Discord Inline Ingestion (ob add)
24+
25+
- The bot supports ingesting raw text directly from Discord using the `ob add` trigger.
26+
- Two supported patterns:
27+
- Inline: `ob add <text>` — paste the text you want the bot to ingest on the same message.
28+
- Reply: Post the text in one message, then reply to that message with `ob add` to instruct the bot to ingest the referenced message's content.
29+
30+
- Behavior and limits:
31+
- The bot writes the provided text to a temporary file and calls the OpenBrain CLI (`ob add`) with a `file://` URL so the existing CLI-based ingestion pipeline is reused.
32+
- To avoid abuse, the bot enforces a conservative default size limit of 64 KiB for direct text ingestion. You can override this limit with the environment variable `OB_ADD_MAX_BYTES` (value in bytes).
33+
- If the bot cannot fetch the referenced message (reply flow), it will reply with a helpful message explaining the permission issue and how to proceed.
34+
- If the CLI is unavailable, the bot will notify the user with a friendly error message.
35+
36+
Example:
37+
38+
```text
39+
Paste long text into a message and then reply to it with:
40+
ob add
41+
42+
Or post the text inline:
43+
ob add The quick brown fox jumps over the lazy dog.
44+
```
45+
2346
## Quick Start
2447

2548
Want to get running quickly? Here's the minimal setup:

src/index.ts

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1858,6 +1858,112 @@ const bot = new DiscordBot({
18581858
authorId: message.author.id
18591859
});
18601860

1861+
// Handle inline `ob add <text>` message triggers. This allows users to
1862+
// paste raw text and instruct the bot to ingest it via the OpenBrain CLI.
1863+
try {
1864+
// Match either `ob add <text>` or a bare `ob add` (so users can reply
1865+
// to an existing message with `ob add`). Capture group 1 is the
1866+
// optional inline payload when provided.
1867+
const obAddMatch =
1868+
typeof message.content === "string" &&
1869+
message.content.match(/^\s*ob\s+add(?:\s+([\s\S]*))?$/i);
1870+
if (obAddMatch) {
1871+
let payload = String(obAddMatch[1] || "").trim();
1872+
1873+
// If payload is empty, attempt to use the referenced/replied-to message's content
1874+
let fetchRefFailed = false;
1875+
if (!payload && message.reference && (message.reference as any).messageId) {
1876+
try {
1877+
const refId = (message.reference as any).messageId;
1878+
const chAny = message.channel as any;
1879+
if (chAny && chAny.messages && typeof chAny.messages.fetch === "function") {
1880+
const refMsg = await chAny.messages.fetch(refId);
1881+
payload = (refMsg?.content || "").trim();
1882+
}
1883+
} catch (err) {
1884+
fetchRefFailed = true;
1885+
logger.warn("Failed to fetch referenced message for ob add", {
1886+
messageId: message.id,
1887+
referencedMessageId: (message.reference as any).messageId,
1888+
error: err instanceof Error ? err.message : String(err),
1889+
});
1890+
}
1891+
}
1892+
1893+
if (!payload) {
1894+
if (fetchRefFailed) {
1895+
// Inform the user the bot couldn't fetch the referenced message so
1896+
// they know to either paste the text or check channel permissions.
1897+
await message.reply(
1898+
"\u26a0\ufe0f I couldn't fetch the message you replied to. Please paste the text you want to add, or ensure the bot has permission to read message history in this channel, then try `ob add` again."
1899+
);
1900+
} else {
1901+
await message.reply("\u274c Please provide text to add, for example: `ob add <text>` or reply to a message with `ob add`.");
1902+
}
1903+
return;
1904+
}
1905+
1906+
// Enforce a conservative size limit to avoid large payload abuse. Default
1907+
// to 64KB but allow overriding via environment for tests or special hosts.
1908+
const MAX_ADD_BYTES = Number(process.env.OB_ADD_MAX_BYTES || 64 * 1024);
1909+
if (Buffer.byteLength(payload, "utf8") > MAX_ADD_BYTES) {
1910+
logger.warn("ob add payload too large", { messageId: message.id, size: Buffer.byteLength(payload, "utf8"), max: MAX_ADD_BYTES });
1911+
await message.reply(`\u26a0\ufe0f Text too large to ingest directly (max ${MAX_ADD_BYTES} bytes). Please provide a URL or split the text into smaller pieces.`);
1912+
return;
1913+
}
1914+
1915+
// Check CLI availability before creating temporary files
1916+
if (!(await checkCliAvailability(message))) {
1917+
logger.warn("ob add requested but CLI unavailable", { messageId: message.id });
1918+
return;
1919+
}
1920+
1921+
// Write payload to a secure temporary file and invoke the existing
1922+
// threaded progress flow by passing a file:// URL to the add command.
1923+
const { makeTempFileName } = await import("./discord/utils.js");
1924+
const fs = await import("fs/promises");
1925+
const tmpName = makeTempFileName("ob-add", "txt");
1926+
1927+
try {
1928+
await fs.writeFile(tmpName, payload, { encoding: "utf8", mode: 0o600 });
1929+
} catch (err) {
1930+
logger.error("Failed to write temporary file for ob add", { messageId: message.id, error: err instanceof Error ? err.message : String(err) });
1931+
await message.reply("\u274c Failed to prepare temporary file for ingestion. Please try again or report this to the maintainers.");
1932+
return;
1933+
}
1934+
1935+
const fileUrl = pathToFileURL(tmpName).toString();
1936+
1937+
try {
1938+
try {
1939+
await processUrlWithProgress(message, fileUrl);
1940+
} catch (err) {
1941+
// processUrlWithProgress performs a lot of its own error handling
1942+
// but be defensive here in case it throws unexpectedly.
1943+
logger.error("Error during ob add processing", { messageId: message.id, error: err instanceof Error ? err.message : String(err) });
1944+
try {
1945+
await message.reply("\u274c Failed to ingest text — an internal error occurred. Please try again later.");
1946+
} catch {
1947+
// ignore reply failures
1948+
}
1949+
throw err;
1950+
}
1951+
} finally {
1952+
try {
1953+
await fs.unlink(tmpName).catch(() => {});
1954+
} catch {
1955+
// best-effort cleanup
1956+
}
1957+
}
1958+
1959+
return;
1960+
}
1961+
} catch (err) {
1962+
// Defensive: ensure any unexpected error during ob add handling does not
1963+
// prevent processing of other message types.
1964+
logger.warn("Error while attempting to handle ob add message", { messageId: message.id, error: err instanceof Error ? err.message : String(err) });
1965+
}
1966+
18611967
// Handle crawl commands
18621968
if (isCrawlCommand(message.content)) {
18631969
logger.info("Crawl command detected", { messageId: message.id });

tests/index.ob-add.test.ts

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
import { beforeEach, describe, expect, it, vi } from "vitest";
2+
3+
// Minimal helper copied/adapted from tests/index.test.ts to capture the
4+
// onMonitoredMessage handler exported to the Discord bot initializer.
5+
async function loadMonitoredMessageHandler(setupAdditionalMocks?: () => Promise<void>) {
6+
let capturedOptions: { onMonitoredMessage?: (message: any) => Promise<void> } | null = null;
7+
8+
await vi.doMock("../src/discord/client.js", async () => {
9+
class MockDiscordBot {
10+
constructor(options: any) {
11+
capturedOptions = options;
12+
}
13+
14+
async start(): Promise<void> {
15+
return;
16+
}
17+
}
18+
19+
return { DiscordBot: MockDiscordBot };
20+
});
21+
22+
const processOnSpy = vi.spyOn(process, "on").mockImplementation(() => process as any);
23+
24+
try {
25+
if (setupAdditionalMocks) {
26+
await setupAdditionalMocks();
27+
}
28+
29+
// Import the module under test after mocking to allow our mocks to be
30+
// picked up by the module loader.
31+
await import("../src/index.js");
32+
} finally {
33+
processOnSpy.mockRestore();
34+
}
35+
36+
if (!capturedOptions) throw new Error("Failed to capture onMonitoredMessage handler");
37+
38+
const handler = (capturedOptions as any).onMonitoredMessage;
39+
if (typeof handler !== "function") throw new Error("Failed to capture onMonitoredMessage handler");
40+
41+
return handler as (message: any) => Promise<void>;
42+
}
43+
44+
function createFakeMessage(content: string, overrides: { channelId?: string; messageId?: string; authorId?: string } = {}) {
45+
const replies: string[] = [];
46+
const threadMessages: string[] = [];
47+
48+
const thread = {
49+
id: "thread-1",
50+
send: vi.fn(async (text: string) => {
51+
threadMessages.push(String(text));
52+
}),
53+
setArchived: vi.fn(async (_archived: boolean) => undefined),
54+
};
55+
56+
const message: any = {
57+
content,
58+
author: { id: overrides.authorId ?? "author-1" },
59+
id: overrides.messageId ?? "message-1",
60+
channelId: overrides.channelId ?? "channel-1",
61+
client: { user: { id: "bot-user-1" } },
62+
react: vi.fn(async (_emoji: string) => undefined),
63+
reply: vi.fn(async (text: string) => {
64+
replies.push(String(text));
65+
}),
66+
startThread: vi.fn(async (_opts: { name: string; autoArchiveDuration: number }) => thread),
67+
reactions: { cache: new Map<string, { users: { remove: (id: string) => Promise<void> } }>() },
68+
};
69+
70+
return { message, replies, thread, threadMessages };
71+
}
72+
73+
describe("ob add message handler failure modes", () => {
74+
beforeEach(() => {
75+
vi.clearAllMocks();
76+
vi.resetModules();
77+
});
78+
79+
it("replies with helpful message when referenced message cannot be fetched", async () => {
80+
const onMonitoredMessage = await loadMonitoredMessageHandler(async () => {
81+
await vi.doMock("../src/bot/cli-runner.js", () => {
82+
class MockCliRunnerError extends Error {}
83+
return {
84+
runAddCommand: vi.fn(),
85+
runQueueCommand: vi.fn(),
86+
runSummaryCommand: vi.fn(),
87+
isCliAvailable: vi.fn(async () => true),
88+
CliRunnerError: MockCliRunnerError,
89+
};
90+
});
91+
});
92+
93+
const { message, replies } = createFakeMessage("ob add");
94+
95+
// Simulate a reply reference but fetching the referenced message fails
96+
message.reference = { messageId: "ref-1" };
97+
message.channel = { messages: { fetch: vi.fn().mockRejectedValue(new Error("nope")) } };
98+
99+
await onMonitoredMessage(message);
100+
101+
expect(replies.length).toBeGreaterThan(0);
102+
expect(replies[0]).toContain("couldn't fetch the message");
103+
});
104+
105+
it("rejects oversized inline payloads with an explanatory message", async () => {
106+
const onMonitoredMessage = await loadMonitoredMessageHandler(async () => {
107+
await vi.doMock("../src/bot/cli-runner.js", () => {
108+
class MockCliRunnerError extends Error {}
109+
return {
110+
runAddCommand: vi.fn(),
111+
runQueueCommand: vi.fn(),
112+
runSummaryCommand: vi.fn(),
113+
isCliAvailable: vi.fn(async () => true),
114+
CliRunnerError: MockCliRunnerError,
115+
};
116+
});
117+
});
118+
119+
// Create a payload larger than the default 64KiB limit
120+
const largeText = "ob add " + "x".repeat(70 * 1024);
121+
const { message, replies } = createFakeMessage(largeText);
122+
123+
await onMonitoredMessage(message);
124+
125+
expect(replies.length).toBeGreaterThan(0);
126+
expect(replies[0]).toContain("Text too large to ingest directly");
127+
});
128+
129+
it("reports temp-file write failures to the user", async () => {
130+
const onMonitoredMessage = await loadMonitoredMessageHandler(async () => {
131+
await vi.doMock("../src/bot/cli-runner.js", () => {
132+
class MockCliRunnerError extends Error {}
133+
return {
134+
runAddCommand: vi.fn(),
135+
runQueueCommand: vi.fn(),
136+
runSummaryCommand: vi.fn(),
137+
isCliAvailable: vi.fn(async () => true),
138+
CliRunnerError: MockCliRunnerError,
139+
};
140+
});
141+
142+
// Ensure we return a deterministic temp file path
143+
await vi.doMock("../src/discord/utils.js", () => ({
144+
makeTempFileName: (prefix = "briefing", ext = "md") => "/tmp/fake-ob-add.txt",
145+
buildCliErrorReport: () => "",
146+
}));
147+
148+
// Mock fs/promises to simulate write failure
149+
await vi.doMock("fs/promises", () => ({
150+
writeFile: vi.fn().mockRejectedValue(new Error("no space")),
151+
unlink: vi.fn().mockResolvedValue(undefined),
152+
}));
153+
});
154+
155+
const { message, replies } = createFakeMessage("ob add small payload");
156+
157+
await onMonitoredMessage(message);
158+
159+
expect(replies.length).toBeGreaterThan(0);
160+
expect(replies[0]).toContain("Failed to prepare temporary file for ingestion");
161+
});
162+
});

0 commit comments

Comments
 (0)