feat: accept clipboard images — paste screenshot → vision request
This commit is contained in:
@@ -316,8 +316,10 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
let inFlight = false;
|
let inFlight = false;
|
||||||
async function send(text) {
|
async function send(payload) {
|
||||||
if (inFlight || !text.trim()) return;
|
if (inFlight) return;
|
||||||
|
const hasText = payload.text && payload.text.trim();
|
||||||
|
if (!hasText && !payload.image) return;
|
||||||
inFlight = true;
|
inFlight = true;
|
||||||
outputEl.innerHTML = '';
|
outputEl.innerHTML = '';
|
||||||
let fullText = '';
|
let fullText = '';
|
||||||
@@ -325,7 +327,7 @@
|
|||||||
const resp = await fetch('/api/chat', {
|
const resp = await fetch('/api/chat', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {'Content-Type':'application/json'},
|
headers: {'Content-Type':'application/json'},
|
||||||
body: JSON.stringify({text, model: 'sonnet'}),
|
body: JSON.stringify({ model: 'sonnet', ...payload }),
|
||||||
});
|
});
|
||||||
const reader = resp.body.getReader();
|
const reader = resp.body.getReader();
|
||||||
const decoder = new TextDecoder();
|
const decoder = new TextDecoder();
|
||||||
@@ -353,12 +355,29 @@
|
|||||||
} catch(e) {} finally { inFlight = false; }
|
} catch(e) {} finally { inFlight = false; }
|
||||||
}
|
}
|
||||||
|
|
||||||
// Auto-send on paste
|
// Auto-send on paste (text or image)
|
||||||
inputEl.addEventListener('paste', () => {
|
inputEl.addEventListener('paste', (e) => {
|
||||||
|
const items = (e.clipboardData || window.clipboardData)?.items || [];
|
||||||
|
let imgItem = null;
|
||||||
|
for (const it of items) {
|
||||||
|
if (it.kind === 'file' && it.type.startsWith('image/')) { imgItem = it; break; }
|
||||||
|
}
|
||||||
|
if (imgItem) {
|
||||||
|
e.preventDefault();
|
||||||
|
const file = imgItem.getAsFile();
|
||||||
|
const fr = new FileReader();
|
||||||
|
fr.onload = () => {
|
||||||
|
const dataUrl = fr.result;
|
||||||
|
const base64 = dataUrl.split(',')[1];
|
||||||
|
send({ image: { data: base64, media_type: file.type || 'image/png' } });
|
||||||
|
};
|
||||||
|
fr.readAsDataURL(file);
|
||||||
|
} else {
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
const t = inputEl.value.trim();
|
const t = inputEl.value.trim();
|
||||||
if (t) send(t);
|
if (t) send({ text: t });
|
||||||
}, 50);
|
}, 50);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
25
server.js
25
server.js
@@ -51,7 +51,7 @@ const SYSTEM_PROMPT = [
|
|||||||
"Respond with the answer only. Nothing else.",
|
"Respond with the answer only. Nothing else.",
|
||||||
].join("\n");
|
].join("\n");
|
||||||
|
|
||||||
async function callClaude(modelId, text, accessToken) {
|
async function callClaude(modelId, content, accessToken) {
|
||||||
return fetch('https://api.anthropic.com/v1/messages', {
|
return fetch('https://api.anthropic.com/v1/messages', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
dispatcher: anthropicAgent,
|
dispatcher: anthropicAgent,
|
||||||
@@ -68,14 +68,25 @@ async function callClaude(modelId, text, accessToken) {
|
|||||||
max_tokens: 8192,
|
max_tokens: 8192,
|
||||||
stream: true,
|
stream: true,
|
||||||
system: SYSTEM_PROMPT,
|
system: SYSTEM_PROMPT,
|
||||||
messages: [{ role: 'user', content: text }],
|
messages: [{ role: 'user', content }],
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
app.post('/api/chat', express.json({ limit: '10mb' }), async (req, res) => {
|
app.post('/api/chat', express.json({ limit: '25mb' }), async (req, res) => {
|
||||||
const { text, model = 'sonnet' } = req.body;
|
const { text, model = 'sonnet', image } = req.body;
|
||||||
if (!text?.trim()) return res.status(400).json({ error: 'text required' });
|
const hasText = text?.trim();
|
||||||
|
if (!hasText && !image) return res.status(400).json({ error: 'text or image required' });
|
||||||
|
|
||||||
|
let content;
|
||||||
|
if (image?.data && image?.media_type) {
|
||||||
|
content = [
|
||||||
|
{ type: 'image', source: { type: 'base64', media_type: image.media_type, data: image.data } },
|
||||||
|
{ type: 'text', text: hasText || 'Analyze the image and answer any questions shown in it.' },
|
||||||
|
];
|
||||||
|
} else {
|
||||||
|
content = text;
|
||||||
|
}
|
||||||
|
|
||||||
const accessToken = getAccessToken();
|
const accessToken = getAccessToken();
|
||||||
if (!accessToken) {
|
if (!accessToken) {
|
||||||
@@ -92,9 +103,9 @@ app.post('/api/chat', express.json({ limit: '10mb' }), async (req, res) => {
|
|||||||
const fallbackId = MODELS.haiku;
|
const fallbackId = MODELS.haiku;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
let apiResp = await callClaude(primaryId, text, accessToken);
|
let apiResp = await callClaude(primaryId, content, accessToken);
|
||||||
if (apiResp.status === 429 && primaryId !== fallbackId) {
|
if (apiResp.status === 429 && primaryId !== fallbackId) {
|
||||||
apiResp = await callClaude(fallbackId, text, accessToken);
|
apiResp = await callClaude(fallbackId, content, accessToken);
|
||||||
}
|
}
|
||||||
if (!apiResp.ok) {
|
if (!apiResp.ok) {
|
||||||
const errText = await apiResp.text();
|
const errText = await apiResp.text();
|
||||||
|
|||||||
Reference in New Issue
Block a user