From 2bbdd5d5c389ec5dbf0915c27ed917409d43db14 Mon Sep 17 00:00:00 2001 From: dlawler489 <104159223@student.swin.edu.au> Date: Tue, 15 Sep 2026 19:48:07 +1000 Subject: [PATCH] Don't split emoji when shortening text for the AI Descriptions sent to the AI are cut to 800 characters. When the cut landed inside an emoji (stored as two UTF-16 halves), the request carried half of it, which DeepSeek rejects: "unexpected end of hex escape". Text is now cut before the emoji, anything already broken in the library is replaced before sending, and the tags, names, collection ideas and descriptions read back are cut the same safe way. Co-Authored-By: Claude Opus 5 --- lib/ai.js | 27 ++++++++++++++++++--------- test/ai.test.js | 17 +++++++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/lib/ai.js b/lib/ai.js index f25c1f5..c107082 100644 --- a/lib/ai.js +++ b/lib/ai.js @@ -40,6 +40,15 @@ class ReplyError extends Error {} const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +// Cuts text to at most max UTF-16 units without splitting an emoji in half. A lone +// half (surrogate) is still sent as JSON, but some APIs (DeepSeek) refuse to parse it. +function clip(text, max) { + const s = String(text); + if (s.length <= max) return s; + const cut = s.slice(0, max); + return /[\uD800-\uDBFF]$/.test(cut) ? cut.slice(0, -1) : cut; +} + // Settings values (ai_provider, ai_base_url, …) -> a client config, filling the // gaps from the provider's preset. function configFrom(values = {}) { @@ -88,7 +97,7 @@ function modelSummary(model, knownTags = []) { const size = modelSize(model); if (size) lines.push(`Size: ${size}`); if (model.tags && model.tags.length) lines.push(`Tags so far: ${model.tags.join(', ')}`); - if (model.description && model.description.trim()) lines.push(`Description so far: ${model.description.trim().slice(0, 800)}`); + if (model.description && model.description.trim()) lines.push(`Description so far: ${clip(model.description.trim(), 800)}`); if (knownTags.length) lines.push(`Tags already used in this library: ${knownTags.join(', ')}`); return lines.join('\n'); } @@ -108,6 +117,8 @@ function buildMessages(model, { + (describe ? DESCRIBE_PROMPT : '') + (suggestName ? NAME_PROMPT : '') + (suggestCollections ? COLLECTIONS_PROMPT : ''); + // Stored names and descriptions may already hold a broken emoji; send a stand-in instead. + text = text.toWellFormed(); const content = image ? [{ type: 'text', text }, { type: 'image_url', image_url: { url: image } }] : text; @@ -118,12 +129,11 @@ function normalizeTags(value) { const list = Array.isArray(value) ? value : typeof value === 'string' ? value.split(',') : []; const tags = []; for (const raw of list) { - const tag = String(raw).toLowerCase() + const tag = clip(String(raw).toLowerCase() .replace(/^#+/, '') .replace(/[_\s]+/g, ' ') .replace(/[^\p{L}\p{N} +&'-]/gu, '') - .trim() - .slice(0, 40) + .trim(), 40) .trim(); if (tag && !GENERIC_TAGS.has(tag) && !tags.includes(tag)) tags.push(tag); } @@ -137,11 +147,10 @@ const stripQuotes = (s) => String(s).replace(/^["'“”‘’]+|["'“”‘’ // written differently. function cleanName(value, currentName) { if (typeof value !== 'string') return null; - const name = stripQuotes(value) + const name = clip(stripQuotes(value) .replace(/\.(stl|3mf|obj|fbx|step|stp|3dm|gcode|zip|7z|rar)$/i, '') .replace(/\s+/g, ' ') - .trim() - .slice(0, 80) + .trim(), 80) .trim(); return name && comparable(name) !== comparable(currentName) ? name : null; } @@ -151,7 +160,7 @@ function normalizeCollections(value) { const list = Array.isArray(value) ? value : typeof value === 'string' ? value.split(',') : []; const names = []; for (const raw of list) { - const name = stripQuotes(raw).replace(/\s+/g, ' ').trim().slice(0, 60).trim(); + const name = clip(stripQuotes(raw).replace(/\s+/g, ' ').trim(), 60).trim(); if (name && !names.some((n) => comparable(n) === comparable(name))) names.push(name); } return names.slice(0, 3); @@ -171,7 +180,7 @@ function parseReply(content, options = {}) { } if (!data || typeof data !== 'object') throw new ReplyError('The AI reply wasn’t the JSON asked for'); const tags = describe ? normalizeTags(data.tags) : []; - const description = describe && typeof data.description === 'string' ? data.description.trim().slice(0, 600) : ''; + const description = describe && typeof data.description === 'string' ? clip(data.description.trim(), 600) : ''; if (describe && !tags.length && !description) throw new ReplyError('The AI reply had no tags or description'); return { tags, diff --git a/test/ai.test.js b/test/ai.test.js index 4a59435..d4705f1 100644 --- a/test/ai.test.js +++ b/test/ai.test.js @@ -146,6 +146,23 @@ test('a reply that isn’t JSON is asked again once, and both answers count towa } }); +test('text sent to the AI never holds half an emoji', () => { + // The panda starts at unit 799, so a plain cut at 800 would split it. + const description = `${'a'.repeat(799)}🐼 red panda`; + const [, user] = ai.buildMessages({ name: 'Red panda', files: [], tags: [], description }); + assert.ok(user.content.isWellFormed()); + assert.ok(!user.content.includes('\uFFFD'), 'clipped before the emoji, not replaced'); + assert.match(user.content, /Description so far: a{799}$/m); + + // Text that's already broken in the library is made safe before it's sent. + const [, broken] = ai.buildMessages({ name: 'Bad \uD83D name', files: [], tags: ['ok'], description: '' }, { knownTags: ['tag\uDC3C'] }); + assert.ok(broken.content.isWellFormed()); + + const reply = JSON.stringify({ name: `x${'🐼'.repeat(50)}`, tags: ['panda'], description: `x${'🐼'.repeat(400)}`, collections: [`x${'🐼'.repeat(40)}`] }); + const parsed = ai.parseReply(reply, { currentName: 'Red panda', collections: true }); + assert.ok(parsed.name.isWellFormed() && parsed.description.isWellFormed() && parsed.collections[0].isWellFormed()); +}); + test('a rejected key is fatal for a run; a bad reply is not', async () => { const provider = await fakeProvider([() => [401, { error: { message: 'Authentication Fails' } }]]); try {