Merge sturdier AI replies with clearer errors

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

lib/ai.js

@@ -113,10 +113,16 @@ function buildMessages(model, {
       ? `\nExisting collections in this library: ${collectionNames.join(', ')}`
       : '\nThis library has no collections yet.';
   }
+  // An example object helps JSON mode (DeepSeek can otherwise send back an empty reply).
+  const example = {};
+  if (describe) Object.assign(example, { tags: ['wall hook', 'headphone holder'], description: 'A wall hook for hanging headphones.' });
+  if (suggestName) example.name = 'Headphone Wall Hook';
+  if (suggestCollections) example.collections = ['Desk Organizers'];
   const system = BASE_PROMPT
     + (describe ? DESCRIBE_PROMPT : '')
     + (suggestName ? NAME_PROMPT : '')
-    + (suggestCollections ? COLLECTIONS_PROMPT : '');
+    + (suggestCollections ? COLLECTIONS_PROMPT : '')
+    + `\nThe format, shown for a different model: ${JSON.stringify(example)}`;
   // Stored names and descriptions may already hold a broken emoji; send a stand-in instead.
   text = text.toWellFormed();
   const content = image
@@ -166,19 +172,54 @@ function normalizeCollections(value) {
   return names.slice(0, 3);
 }
 
+// The JSON object in a reply, allowing for code fences, notes around it, or braces
+// in a note after it; null when there isn't one.
+function extractJson(content) {
+  const text = String(content || '').replace(/```(?:json)?/gi, '').trim();
+  const candidates = [text];
+  const first = text.indexOf('{');
+  const last = text.lastIndexOf('}');
+  if (first >= 0 && last > first) candidates.push(text.slice(first, last + 1));
+  if (first >= 0) {
+    // The first complete {...}, reading past braces inside strings.
+    let depth = 0;
+    let inString = false;
+    let escaped = false;
+    for (let i = first; i < text.length; i++) {
+      const ch = text[i];
+      if (inString) {
+        if (escaped) escaped = false;
+        else if (ch === '\\') escaped = true;
+        else if (ch === '"') inString = false;
+      } else if (ch === '"') inString = true;
+      else if (ch === '{') depth += 1;
+      else if (ch === '}' && --depth === 0) { candidates.push(text.slice(first, i + 1)); break; }
+    }
+  }
+  for (const candidate of candidates) {
+    try {
+      const data = JSON.parse(candidate);
+      if (data && typeof data === 'object' && !Array.isArray(data)) return data;
+    } catch { /* try the next */ }
+  }
+  return null;
+}
+
+// Says what came back instead of the JSON, so a failure can be understood.
+function badReplyMessage(content, finishReason) {
+  const text = String(content || '').replace(/\s+/g, ' ').trim();
+  if (!text) return 'The AI reply was empty, not the JSON asked for';
+  const cutOff = finishReason === 'length' ? ' (it was cut off)' : '';
+  return `The AI reply wasn’t the JSON asked for${cutOff}. It began: “${clip(text, 120)}${text.length > 120 ? '…' : ''}”`;
+}
+
 // The model's reply -> { tags, description, name, collections }. Tolerates code
 // fences or prose around the JSON. Only the fields asked for are read: name when
 // { currentName } is given, collections when { collections: true }.
 function parseReply(content, options = {}) {
   const { describe = true, collections = false } = options;
-  const text = String(content || '');
-  const start = text.indexOf('{');
-  const end = text.lastIndexOf('}');
-  let data = null;
-  if (start >= 0 && end > start) {
-    try { data = JSON.parse(text.slice(start, end + 1)); } catch { /* handled below */ }
-  }
-  if (!data || typeof data !== 'object') throw new ReplyError('The AI reply wasn’t the JSON asked for');
+  const data = extractJson(content);
+  if (!data) throw new ReplyError(badReplyMessage(content, options.finishReason));
   const tags = describe ? normalizeTags(data.tags) : [];
   const description = describe && typeof data.description === 'string' ? clip(data.description.trim(), 600) : '';
   if (describe && !tags.length && !description) throw new ReplyError('The AI reply had no tags or description');
@@ -244,16 +285,20 @@ async function chat(config, body, { timeoutMs = 90000 } = {}) {
 // Asks about one model: tags and a description, and optionally a clearer name and
 // collection ideas. Adapts to what the model supports: if it refuses JSON mode or
 // images, the request is retried without them (imageRejected tells the caller to
-// stop sending images). A reply that isn't the JSON asked for is asked again once.
-// Rate limits and server errors are retried with backoff.
+// stop sending images). A reply that isn't the JSON asked for is asked again, up
+// to twice: with more room if it was cut off, and the last time without JSON mode,
+// which can send back empty replies. Rate limits and server errors are retried
+// with backoff. Errors carry the tokens and cost spent on the failed attempts.
 async function describeModel(config, model, {
   image = null, knownTags = [], describe = true, suggestName = false, suggestCollections = false, collectionNames = [],
 } = {}) {
   let useImage = Boolean(image);
   let useJson = true;
   let imageRejected = false;
-  let replyRetried = false;
+  let replyRetries = 0;
+  let maxTokens = 1500;
   const spent = { prompt_tokens: 0, completion_tokens: 0, cost: null };
+  const withSpend = (err) => Object.assign(err, { usage: usageOf(spent), cost: costOf(spent, config.prices) });
   const want = { describe, collections: suggestCollections, ...(suggestName ? { currentName: model.name } : {}) };
 
   for (let attempt = 0; ; attempt++) {
@@ -263,20 +308,23 @@ async function describeModel(config, model, {
         image: useImage ? image : null, knownTags, describe, suggestName, suggestCollections, collectionNames,
       }),
       temperature: 0.2,
-      max_tokens: 800,
+      max_tokens: maxTokens,
     };
     if (useJson) body.response_format = { type: 'json_object' };
     if (config.provider === 'openrouter') body.usage = { include: true };
     try {
       const data = await chat(config, body);
       addUsage(spent, data.usage);
+      const choice = data.choices[0] || {};
       let reply;
       try {
-        reply = parseReply((data.choices[0].message || {}).content, want);
+        reply = parseReply((choice.message || {}).content, { ...want, finishReason: choice.finish_reason });
       } catch (err) {
-        // Usually a one-off: ask once more before giving up on this model.
-        if (err instanceof ReplyError && !replyRetried) { replyRetried = true; continue; }
-        throw err;
+        if (!(err instanceof ReplyError) || replyRetries >= 2) throw withSpend(err);
+        replyRetries += 1;
+        if (choice.finish_reason === 'length') maxTokens = Math.min(maxTokens * 2, 4000); // cut off: give it room
+        else if (replyRetries === 2) useJson = false;
+        continue;
       }
       return {
         ...reply,
@@ -284,23 +332,23 @@ async function describeModel(config, model, {
         cost: costOf(spent, config.prices),
         imageUsed: useImage,
         imageRejected,
-        replyRetried,
+        replyRetried: replyRetries > 0,
       };
     } catch (err) {
-      if (!(err instanceof AiError) || attempt >= 5) throw err;
+      if (!(err instanceof AiError) || attempt >= 5) throw withSpend(err);
       if (err.status === 400 || err.status === 415 || err.status === 422) {
         if (useImage && /image|vision|multimodal|modalit|content.?part|content type/i.test(err.message)) {
           useImage = false; imageRejected = true; continue;
         }
         if (useJson) { useJson = false; continue; }
         if (useImage) { useImage = false; imageRejected = true; continue; }
-        throw err;
+        throw withSpend(err);
       }
       if (err.status === 429 || err.status >= 500 || err.kind === 'timeout') {
         await sleep(Math.min(20000, 1500 * 2 ** attempt));
         continue;
       }
-      throw err;
+      throw withSpend(err);
     }
   }
 }

lib/aiRuns.js

@@ -128,6 +128,13 @@ async function tagOne(runId, modelId, config, context) {
       [runId, result.usage.prompt, result.usage.completion, result.cost || 0]);
     return { ok: true, imageRejected: result.imageRejected, suggested: suggested.length > 0 };
   } catch (err) {
+    // Failed attempts still cost tokens.
+    if (err.usage && (err.usage.prompt || err.usage.completion)) {
+      await db.query(
+        `UPDATE ai_runs SET prompt_tokens = prompt_tokens + $2, completion_tokens = completion_tokens + $3,
+           cost_usd = cost_usd + $4 WHERE id = $1`,
+        [runId, err.usage.prompt, err.usage.completion, err.cost || 0]);
+    }
     const fatal = ai.fatalMessage(err);
     if (fatal) return { ok: false, fatal }; // left pending, so resuming retries it
     const message = err.message.slice(0, 300);

test/ai.test.js

@@ -121,7 +121,7 @@ test('collection ideas are asked for with the library’s collections, and tidie
   assert.deepEqual(ai.parseReply('{"tags": ["dragon"], "collections": ["Dragons"]}').collections, []);
 });
 
-test('a reply that isn’t JSON is asked again once, and both answers count towards cost', async () => {
+test('a reply that isn’t JSON is asked again, and every answer counts towards cost', async () => {
   const provider = await fakeProvider([
     () => reply('Sure! Tags: dragon, toy'),
     () => reply('{"tags": ["dragon"], "description": "A dragon."}'),
@@ -137,15 +137,57 @@ test('a reply that isn’t JSON is asked again once, and both answers count towa
     await provider.close();
   }
 
-  const stubborn = await fakeProvider([() => reply('no JSON here')]);
+  // Empty replies in JSON mode: the last try goes without JSON mode.
+  const empty = await fakeProvider([() => reply(''), () => reply(''), () => reply('{"tags": ["dragon"], "description": "A dragon."}')]);
   try {
-    await assert.rejects(ai.describeModel(stubborn.config, MODEL), /JSON/);
-    assert.equal(stubborn.requests.length, 2);
+    const result = await ai.describeModel(empty.config, MODEL);
+    assert.deepEqual(result.tags, ['dragon']);
+    assert.ok(empty.requests[1].body.response_format);
+    assert.equal(empty.requests[2].body.response_format, undefined);
+  } finally {
+    await empty.close();
+  }
+
+  // Still nothing usable: the error says what came back and carries the spend.
+  const stubborn = await fakeProvider([() => reply('')]);
+  try {
+    const err = await ai.describeModel(stubborn.config, MODEL).catch((e) => e);
+    assert.match(err.message, /reply was empty, not the JSON asked for/);
+    assert.equal(stubborn.requests.length, 3);
+    assert.deepEqual(err.usage, { prompt: 3000, completion: 300 });
   } finally {
     await stubborn.close();
   }
 });
 
+test('a reply that was cut off is asked again with more room', async () => {
+  const cutOff = [200, { choices: [{ message: { content: '{"tags": ["red panda", "fl' }, finish_reason: 'length' }], usage: { prompt_tokens: 1000, completion_tokens: 100 } }];
+  const provider = await fakeProvider([() => cutOff, () => reply('{"tags": ["red panda"], "description": "A flexi red panda."}')]);
+  try {
+    const result = await ai.describeModel(provider.config, MODEL);
+    assert.deepEqual(result.tags, ['red panda']);
+    assert.equal(provider.requests[0].body.max_tokens, 1500);
+    assert.equal(provider.requests[1].body.max_tokens, 3000);
+  } finally {
+    await provider.close();
+  }
+  assert.throws(() => ai.parseReply('{"tags": ["red', { finishReason: 'length' }),
+    /cut off\)\. It began: “\{"tags": \["red”/);
+});
+
+test('JSON is found inside fences, notes and braces after it', () => {
+  assert.deepEqual(ai.parseReply('```json\n{"tags": ["owl"], "description": "An owl {bookend}."}\n```').tags, ['owl']);
+  assert.equal(ai.parseReply('Here you go: {"tags": ["owl"], "description": "An owl."} Note: fields were {tags, description}.').description, 'An owl.');
+  assert.throws(() => ai.parseReply('I cannot help with that'), /It began: “I cannot help with that”/);
+});
+
+test('the prompt shows the reply format for just the fields asked for', () => {
+  const plain = ai.buildMessages(MODEL)[0].content;
+  assert.match(plain, /The format, shown for a different model: \{"tags":\["wall hook","headphone holder"\],"description":"A wall hook for hanging headphones\."\}/);
+  const collectionsOnly = ai.buildMessages(MODEL, { describe: false, suggestCollections: true })[0].content;
+  assert.match(collectionsOnly, /\{"collections":\["Desk Organizers"\]\}/);
+});
+
 test('text sent to the AI never holds half an emoji', () => {
   // The panda starts at unit 799, so a plain cut at 800 would split it.
   const description = `${'a'.repeat(799)}🐼 red panda`;