diff --git a/lib/ai.js b/lib/ai.js index f25c1f5..c107082 100644 --- a/lib/ai.js +++ b/lib/ai.js @@ -40,6 +40,15 @@ class ReplyError extends Error {} const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +// Cuts text to at most max UTF-16 units without splitting an emoji in half. A lone +// half (surrogate) is still sent as JSON, but some APIs (DeepSeek) refuse to parse it. +function clip(text, max) { + const s = String(text); + if (s.length <= max) return s; + const cut = s.slice(0, max); + return /[\uD800-\uDBFF]$/.test(cut) ? cut.slice(0, -1) : cut; +} + // Settings values (ai_provider, ai_base_url, …) -> a client config, filling the // gaps from the provider's preset. function configFrom(values = {}) { @@ -88,7 +97,7 @@ function modelSummary(model, knownTags = []) { const size = modelSize(model); if (size) lines.push(`Size: ${size}`); if (model.tags && model.tags.length) lines.push(`Tags so far: ${model.tags.join(', ')}`); - if (model.description && model.description.trim()) lines.push(`Description so far: ${model.description.trim().slice(0, 800)}`); + if (model.description && model.description.trim()) lines.push(`Description so far: ${clip(model.description.trim(), 800)}`); if (knownTags.length) lines.push(`Tags already used in this library: ${knownTags.join(', ')}`); return lines.join('\n'); } @@ -108,6 +117,8 @@ function buildMessages(model, { + (describe ? DESCRIBE_PROMPT : '') + (suggestName ? NAME_PROMPT : '') + (suggestCollections ? COLLECTIONS_PROMPT : ''); + // Stored names and descriptions may already hold a broken emoji; send a stand-in instead. + text = text.toWellFormed(); const content = image ? [{ type: 'text', text }, { type: 'image_url', image_url: { url: image } }] : text; @@ -118,12 +129,11 @@ function normalizeTags(value) { const list = Array.isArray(value) ? value : typeof value === 'string' ? value.split(',') : []; const tags = []; for (const raw of list) { - const tag = String(raw).toLowerCase() + const tag = clip(String(raw).toLowerCase() .replace(/^#+/, '') .replace(/[_\s]+/g, ' ') .replace(/[^\p{L}\p{N} +&'-]/gu, '') - .trim() - .slice(0, 40) + .trim(), 40) .trim(); if (tag && !GENERIC_TAGS.has(tag) && !tags.includes(tag)) tags.push(tag); } @@ -137,11 +147,10 @@ const stripQuotes = (s) => String(s).replace(/^["'“”‘’]+|["'“”‘’ // written differently. function cleanName(value, currentName) { if (typeof value !== 'string') return null; - const name = stripQuotes(value) + const name = clip(stripQuotes(value) .replace(/\.(stl|3mf|obj|fbx|step|stp|3dm|gcode|zip|7z|rar)$/i, '') .replace(/\s+/g, ' ') - .trim() - .slice(0, 80) + .trim(), 80) .trim(); return name && comparable(name) !== comparable(currentName) ? name : null; } @@ -151,7 +160,7 @@ function normalizeCollections(value) { const list = Array.isArray(value) ? value : typeof value === 'string' ? value.split(',') : []; const names = []; for (const raw of list) { - const name = stripQuotes(raw).replace(/\s+/g, ' ').trim().slice(0, 60).trim(); + const name = clip(stripQuotes(raw).replace(/\s+/g, ' ').trim(), 60).trim(); if (name && !names.some((n) => comparable(n) === comparable(name))) names.push(name); } return names.slice(0, 3); @@ -171,7 +180,7 @@ function parseReply(content, options = {}) { } if (!data || typeof data !== 'object') throw new ReplyError('The AI reply wasn’t the JSON asked for'); const tags = describe ? normalizeTags(data.tags) : []; - const description = describe && typeof data.description === 'string' ? data.description.trim().slice(0, 600) : ''; + const description = describe && typeof data.description === 'string' ? clip(data.description.trim(), 600) : ''; if (describe && !tags.length && !description) throw new ReplyError('The AI reply had no tags or description'); return { tags, diff --git a/test/ai.test.js b/test/ai.test.js index 4a59435..d4705f1 100644 --- a/test/ai.test.js +++ b/test/ai.test.js @@ -146,6 +146,23 @@ test('a reply that isn’t JSON is asked again once, and both answers count towa } }); +test('text sent to the AI never holds half an emoji', () => { + // The panda starts at unit 799, so a plain cut at 800 would split it. + const description = `${'a'.repeat(799)}🐼 red panda`; + const [, user] = ai.buildMessages({ name: 'Red panda', files: [], tags: [], description }); + assert.ok(user.content.isWellFormed()); + assert.ok(!user.content.includes('\uFFFD'), 'clipped before the emoji, not replaced'); + assert.match(user.content, /Description so far: a{799}$/m); + + // Text that's already broken in the library is made safe before it's sent. + const [, broken] = ai.buildMessages({ name: 'Bad \uD83D name', files: [], tags: ['ok'], description: '' }, { knownTags: ['tag\uDC3C'] }); + assert.ok(broken.content.isWellFormed()); + + const reply = JSON.stringify({ name: `x${'🐼'.repeat(50)}`, tags: ['panda'], description: `x${'🐼'.repeat(400)}`, collections: [`x${'🐼'.repeat(40)}`] }); + const parsed = ai.parseReply(reply, { currentName: 'Red panda', collections: true }); + assert.ok(parsed.name.isWellFormed() && parsed.description.isWellFormed() && parsed.collections[0].isWellFormed()); +}); + test('a rejected key is fatal for a run; a bad reply is not', async () => { const provider = await fakeProvider([() => [401, { error: { message: 'Authentication Fails' } }]]); try {