Skip to content

fix(chatgpt): support current tool menu rows #2117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lavapapa wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(chatgpt): support current tool menu rows #2117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 110 additions & 10 deletions clis/chatgpt/utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -765,20 +765,58 @@ export async function getCurrentChatGPTTool(page) {
return rect.width > 0 && rect.height > 0;
};
const normalize = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
const escapeRegExp = (value) => String(value).replace(/[|\\\\{}()[\\]^$+*?.]/g, '\\\\$&');
const textMatchesLabel = (text, label) => {
const normalizedText = normalize(text);
const normalizedLabel = normalize(label);
if (!normalizedText || !normalizedLabel) return false;
if (normalizedText === normalizedLabel) return true;
if (normalizedText.startsWith(normalizedLabel + ' ')) return true;
if (/^[\\x00-\\x7F]+$/.test(normalizedLabel)) {
return new RegExp('(^|\\\\b)' + escapeRegExp(normalizedLabel) + '(\\\\b|$)', 'i').test(normalizedText);
}
return normalizedText.replace(/\\s+/g, '').startsWith(normalizedLabel.replace(/\\s+/g, ''));
};
const findEntryForText = (text) => {
const matches = [];
for (const [key, value] of Object.entries(labels)) {
for (const item of value.labels || []) {
if (textMatchesLabel(text, item)) {
matches.push({ key, value, length: normalize(item).length });
}
}
}
matches.sort((a, b) => b.length - a.length);
return matches[0] || null;
};
const labels = ${JSON.stringify(CHATGPT_TOOL_OPTIONS)};
const form = Array.from(document.querySelectorAll('form')).find((node) => node instanceof HTMLElement && isVisible(node));
const root = form || document.body;
const nodes = Array.from(root.querySelectorAll('button, [role="button"], [role="menuitemradio"], span, div'));
const composerSelector = '#prompt-textarea, [data-testid="prompt-textarea"], [contenteditable="true"][role="textbox"]';
const isEditablePromptText = (node) => {
const editable = node.closest(composerSelector);
if (!editable) return false;
return !node.closest('[contenteditable="false"]');
};
const nodes = Array.from(root.querySelectorAll([
'button',
'[role="button"]',
'[role="menuitemradio"][aria-checked="true"]',
'[contenteditable="false"]',
'[data-testid*="selected" i]',
'[data-testid*="tool" i]',
].join(',')));
const node = nodes.find((candidate) => {
if (!isVisible(candidate)) return false;
if (isEditablePromptText(candidate)) return false;
const text = normalize(candidate.textContent);
return Object.values(labels).some((entry) => entry.labels.includes(text));
return !!findEntryForText(text);
});
const label = normalize(node?.textContent || '');
const entry = Object.entries(labels).find(([, value]) => value.labels.includes(label));
const entry = findEntryForText(label);
return {
tool: entry?.[0] ?? null,
label: entry?.[1]?.label ?? null,
tool: entry?.key ?? null,
label: entry?.value?.label ?? null,
};
})()`)), 'chatgpt current tool');
}
Expand Down Expand Up @@ -832,8 +870,38 @@ export async function selectChatGPTTool(page, tool) {
};
const normalize = (value) => String(value || '').replace(/\\s+/g, ' ').trim();
const labels = ${JSON.stringify(target.labels)};
const options = Array.from(document.querySelectorAll('[role="menuitemradio"]'));
const option = options.find((node) => node instanceof HTMLElement && isVisible(node) && labels.includes(normalize(node.textContent)));
const escapeRegExp = (value) => String(value).replace(/[|\\\\{}()[\\]^$+*?.]/g, '\\\\$&');
const matchScore = (text, label) => {
const normalizedText = normalize(text);
const normalizedLabel = normalize(label);
if (!normalizedText || !normalizedLabel) return 0;
if (normalizedText === normalizedLabel) return 4;
if (normalizedText.startsWith(normalizedLabel + ' ')) return 3;
if (/^[\\x00-\\x7F]+$/.test(normalizedLabel)) {
return new RegExp('(^|\\\\b)' + escapeRegExp(normalizedLabel) + '(\\\\b|$)', 'i').test(normalizedText) ? 2 : 0;
}
return normalizedText.replace(/\\s+/g, '').startsWith(normalizedLabel.replace(/\\s+/g, '')) ? 2 : 0;
};
const bestScore = (node) => Math.max(0, ...labels.map(label => matchScore(node.textContent, label)));
const clickableSelector = '[role="menuitemradio"], [role="menuitem"], [role="option"], button, div[tabindex="0"]';
const composerSelector = '#prompt-textarea, [data-testid="prompt-textarea"], [contenteditable="true"][role="textbox"]';
const candidates = new Set(Array.from(document.querySelectorAll(clickableSelector)));
for (const labelNode of Array.from(document.querySelectorAll('span, div, p'))) {
if (!(labelNode instanceof HTMLElement) || !isVisible(labelNode)) continue;
if (!bestScore(labelNode)) continue;
const clickable = labelNode.closest(clickableSelector);
if (clickable) candidates.add(clickable);
}
const ranked = Array.from(candidates)
.filter((node) =>
node instanceof HTMLElement
&& isVisible(node)
&& !node.closest(composerSelector)
&& bestScore(node) > 0
)
.map((node) => ({ node, score: bestScore(node), textLength: normalize(node.textContent).length }))
.sort((a, b) => b.score - a.score || a.textLength - b.textLength);
const option = ranked[0]?.node || null;
if (!(option instanceof HTMLElement)) return { found: false };
const checked = option.getAttribute('aria-checked') === 'true';
option.scrollIntoView({ block: 'center', inline: 'center' });
Expand Down Expand Up @@ -943,9 +1011,41 @@ export async function sendChatGPTMessage(page, text) {
composer.focus();
if (composer instanceof HTMLTextAreaElement || composer instanceof HTMLInputElement) {
composer.value = '';
} else if (composer.isContentEditable) {
composer.textContent = '';
composer.innerHTML = '<p><br></p>';
} else if (composer.isContentEditable || composer.getAttribute('contenteditable') === 'true') {
const preservedSelector = '[contenteditable="false"]';
const walker = document.createTreeWalker(composer, NodeFilter.SHOW_TEXT, {
acceptNode(node) {
const parent = node.parentElement;
if (!parent || parent.closest(preservedSelector)) return NodeFilter.FILTER_REJECT;
return NodeFilter.FILTER_ACCEPT;
},
});
const textNodes = [];
while (walker.nextNode()) textNodes.push(walker.currentNode);
for (const node of textNodes) node.nodeValue = '';

for (const node of Array.from(composer.querySelectorAll('*')).reverse()) {
if (!(node instanceof HTMLElement)) continue;
if (node.matches(preservedSelector) || node.closest(preservedSelector)) continue;
if (['BR', 'IMG', 'INPUT', 'TEXTAREA'].includes(node.tagName)) continue;
if (node.childNodes.length === 0) node.remove();
}

const lastElement = composer.lastElementChild;
if (!(lastElement instanceof HTMLElement) || lastElement.matches(preservedSelector) || lastElement.closest(preservedSelector)) {
const paragraph = document.createElement('p');
paragraph.appendChild(document.createElement('br'));
composer.appendChild(paragraph);
}

const selection = window.getSelection();
if (selection) {
const range = document.createRange();
range.selectNodeContents(composer);
range.collapse(false);
selection.removeAllRanges();
selection.addRange(range);
}
} else {
composer.textContent = '';
}
Expand Down
100 changes: 99 additions & 1 deletion clis/chatgpt/utils.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ function createDomEvaluatePage(html) {
url: 'https://chatgpt.com/',
runScripts: 'outside-only',
});
for (const node of dom.window.document.querySelectorAll('form, button, [role="menuitemradio"], [role="menuitem"], [role="option"], #prompt-textarea, [data-testid]')) {
for (const node of dom.window.document.querySelectorAll('form, button, [role="menuitemradio"], [role="menuitem"], [role="option"], [tabindex], [contenteditable="false"], #prompt-textarea, [data-testid]')) {
node.getBoundingClientRect = () => ({ width: 120, height: 36 });
node.scrollIntoView = () => {};
}
Expand Down Expand Up @@ -656,6 +656,46 @@ describe('chatgpt tool selection validation', () => {
await expect(selectChatGPTTool({}, 'deep-research'))
.rejects.toThrow('ChatGPT tool selection requires native browser click support.');
});

it('selects Deep Research from current custom tabbable tool rows', async () => {
const page = createDomEvaluatePage(`
<form>
<button type="button" data-testid="composer-plus-btn">+</button>
<div id="prompt-textarea" contenteditable="true"><p><br></p></div>
</form>
<div role="menu" data-open="false" style="display: none">
<div tabindex="0">
<div>添加文件</div>
<div>Upload a local file</div>
</div>
<div tabindex="0">
<div>深度研究</div>
<div>进行多步骤研究并引用来源</div>
</div>
</div>
`);
let clickCount = 0;
page.nativeClick = vi.fn().mockImplementation(async () => {
clickCount += 1;
if (clickCount === 1) {
const menu = page.dom.window.document.querySelector('[role="menu"]');
menu.style.display = 'block';
for (const node of page.dom.window.document.querySelectorAll('[tabindex]')) {
node.getBoundingClientRect = () => ({ width: 220, height: 44 });
node.scrollIntoView = () => {};
}
}
if (clickCount === 2) {
const composer = page.dom.window.document.querySelector('#prompt-textarea');
composer.innerHTML = '<span contenteditable="false">深度研究</span><p><br></p>';
composer.querySelector('[contenteditable="false"]').getBoundingClientRect = () => ({ width: 120, height: 36 });
page.dom.window.document.querySelector('[role="menu"]').style.display = 'none';
}
});

await expect(selectChatGPTTool(page, 'deep-research')).resolves.toEqual({ Status: 'Success', Tool: 'Deep Research' });
expect(page.nativeClick).toHaveBeenCalledTimes(2);
});
});

describe('chatgpt detail completion state', () => {
Expand Down Expand Up @@ -934,6 +974,39 @@ describe('chatgpt current tool detection', () => {
label: null,
});
});

it('ignores prompt text that mentions tool labels', async () => {
const page = createDomEvaluatePage(`
<form>
<button type="button" data-testid="composer-plus-btn">+</button>
<div id="prompt-textarea" contenteditable="true">
<p>OpenCLI Deep Research selector trace smoke / 深度研究</p>
</div>
</form>
`);

await expect(getCurrentChatGPTTool(page)).resolves.toEqual({
tool: null,
label: null,
});
});

it('detects selected tool pills inside the composer', async () => {
const page = createDomEvaluatePage(`
<form>
<button type="button" data-testid="composer-plus-btn">+</button>
<div id="prompt-textarea" contenteditable="true">
<span contenteditable="false">深度研究</span>
<p><br></p>
</div>
</form>
`);

await expect(getCurrentChatGPTTool(page)).resolves.toEqual({
tool: 'deep-research',
label: 'Deep Research',
});
});
});

describe('chatgpt send selectors', () => {
Expand Down Expand Up @@ -998,6 +1071,31 @@ describe('chatgpt send selectors', () => {
await expect(sendChatGPTMessage(page, 'hello')).resolves.toBe(true);
});

it('preserves selected tool pills when clearing prompt text before typing', async () => {
const page = createDomEvaluatePage(`
<form>
<div id="prompt-textarea" contenteditable="true">
<span contenteditable="false">深度研究</span>
<p>stale draft</p>
</div>
<button type="button" data-testid="send-button">Send</button>
</form>
`);
page.nativeClick = vi.fn().mockResolvedValue(undefined);
page.nativeType = vi.fn().mockImplementation(async (text) => {
const composer = page.dom.window.document.querySelector('#prompt-textarea');
const paragraph = composer.querySelector('p') || composer.appendChild(page.dom.window.document.createElement('p'));
paragraph.textContent = text;
});

await expect(sendChatGPTMessage(page, 'fresh prompt')).resolves.toBe(true);

const composer = page.dom.window.document.querySelector('#prompt-textarea');
expect(composer.querySelector('[contenteditable="false"]')?.textContent).toBe('深度研究');
expect(composer.textContent).toContain('fresh prompt');
expect(composer.textContent).not.toContain('stale draft');
});

it('keeps zh-CN aria and placeholder fallbacks without replacing English selectors', () => {
expect(__test__.COMPOSER_SELECTORS).toEqual(expect.arrayContaining([
'[aria-label="Chat with ChatGPT"]',
Expand Down
Loading