feat: accept clipboard images — paste screenshot → vision request

This commit is contained in:
aedes
2026-04-21 19:30:43 +03:00
parent cf1ee42c3b
commit 8dfa51d92e
2 changed files with 46 additions and 16 deletions

View File

@@ -316,8 +316,10 @@
} }
let inFlight = false; let inFlight = false;
async function send(text) { async function send(payload) {
if (inFlight || !text.trim()) return; if (inFlight) return;
const hasText = payload.text && payload.text.trim();
if (!hasText && !payload.image) return;
inFlight = true; inFlight = true;
outputEl.innerHTML = ''; outputEl.innerHTML = '';
let fullText = ''; let fullText = '';
@@ -325,7 +327,7 @@
const resp = await fetch('/api/chat', { const resp = await fetch('/api/chat', {
method: 'POST', method: 'POST',
headers: {'Content-Type':'application/json'}, headers: {'Content-Type':'application/json'},
body: JSON.stringify({text, model: 'sonnet'}), body: JSON.stringify({ model: 'sonnet', ...payload }),
}); });
const reader = resp.body.getReader(); const reader = resp.body.getReader();
const decoder = new TextDecoder(); const decoder = new TextDecoder();
@@ -353,12 +355,29 @@
} catch(e) {} finally { inFlight = false; } } catch(e) {} finally { inFlight = false; }
} }
// Auto-send on paste // Auto-send on paste (text or image)
inputEl.addEventListener('paste', () => { inputEl.addEventListener('paste', (e) => {
const items = (e.clipboardData || window.clipboardData)?.items || [];
let imgItem = null;
for (const it of items) {
if (it.kind === 'file' && it.type.startsWith('image/')) { imgItem = it; break; }
}
if (imgItem) {
e.preventDefault();
const file = imgItem.getAsFile();
const fr = new FileReader();
fr.onload = () => {
const dataUrl = fr.result;
const base64 = dataUrl.split(',')[1];
send({ image: { data: base64, media_type: file.type || 'image/png' } });
};
fr.readAsDataURL(file);
} else {
setTimeout(() => { setTimeout(() => {
const t = inputEl.value.trim(); const t = inputEl.value.trim();
if (t) send(t); if (t) send({ text: t });
}, 50); }, 50);
}
}); });
</script> </script>

View File

@@ -51,7 +51,7 @@ const SYSTEM_PROMPT = [
"Respond with the answer only. Nothing else.", "Respond with the answer only. Nothing else.",
].join("\n"); ].join("\n");
async function callClaude(modelId, text, accessToken) { async function callClaude(modelId, content, accessToken) {
return fetch('https://api.anthropic.com/v1/messages', { return fetch('https://api.anthropic.com/v1/messages', {
method: 'POST', method: 'POST',
dispatcher: anthropicAgent, dispatcher: anthropicAgent,
@@ -68,14 +68,25 @@ async function callClaude(modelId, text, accessToken) {
max_tokens: 8192, max_tokens: 8192,
stream: true, stream: true,
system: SYSTEM_PROMPT, system: SYSTEM_PROMPT,
messages: [{ role: 'user', content: text }], messages: [{ role: 'user', content }],
}), }),
}); });
} }
app.post('/api/chat', express.json({ limit: '10mb' }), async (req, res) => { app.post('/api/chat', express.json({ limit: '25mb' }), async (req, res) => {
const { text, model = 'sonnet' } = req.body; const { text, model = 'sonnet', image } = req.body;
if (!text?.trim()) return res.status(400).json({ error: 'text required' }); const hasText = text?.trim();
if (!hasText && !image) return res.status(400).json({ error: 'text or image required' });
let content;
if (image?.data && image?.media_type) {
content = [
{ type: 'image', source: { type: 'base64', media_type: image.media_type, data: image.data } },
{ type: 'text', text: hasText || 'Analyze the image and answer any questions shown in it.' },
];
} else {
content = text;
}
const accessToken = getAccessToken(); const accessToken = getAccessToken();
if (!accessToken) { if (!accessToken) {
@@ -92,9 +103,9 @@ app.post('/api/chat', express.json({ limit: '10mb' }), async (req, res) => {
const fallbackId = MODELS.haiku; const fallbackId = MODELS.haiku;
try { try {
let apiResp = await callClaude(primaryId, text, accessToken); let apiResp = await callClaude(primaryId, content, accessToken);
if (apiResp.status === 429 && primaryId !== fallbackId) { if (apiResp.status === 429 && primaryId !== fallbackId) {
apiResp = await callClaude(fallbackId, text, accessToken); apiResp = await callClaude(fallbackId, content, accessToken);
} }
if (!apiResp.ok) { if (!apiResp.ok) {
const errText = await apiResp.text(); const errText = await apiResp.text();