// AI tagging client (lib/ai.js), against a fake OpenAI-compatible server. const test = require('node:test'); const assert = require('node:assert/strict'); const http = require('http'); const ai = require('../lib/ai'); // Starts a server that answers each request with the next handler's result. async function fakeProvider(handlers) { const requests = []; const server = http.createServer((req, res) => { let body = ''; req.on('data', (c) => { body += c; }); req.on('end', () => { const parsed = JSON.parse(body); requests.push({ path: req.url, auth: req.headers.authorization, body: parsed }); const [status, payload] = handlers[Math.min(requests.length, handlers.length) - 1](parsed); res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(payload)); }); }); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); const config = ai.configFrom({ ai_provider: 'custom', ai_base_url: `http://127.0.0.1:${server.address().port}/v1/`, ai_api_key: 'sk-test', ai_model: 'test-model', ai_price_input: '0.3', ai_price_output: '1.2', }); return { config, requests, close: () => new Promise((resolve) => server.close(resolve)) }; } const reply = (content, usage = { prompt_tokens: 1000, completion_tokens: 100 }) => [200, { choices: [{ message: { content } }], usage }]; const MODEL = { name: 'Flexi Dragon', files: [{ originalName: 'dragon_PLA.3mf', bbox: { x: 120, y: 40, z: 30 } }], tags: ['toy'] }; test('presets fill in DeepSeek by default, and your own values win', () => { const deepseek = ai.configFrom({ ai_api_key: 'k' }); assert.equal(deepseek.provider, 'deepseek'); assert.equal(deepseek.baseUrl, 'https://api.deepseek.com'); assert.equal(deepseek.model, 'deepseek-flash'); assert.equal(ai.missingConfig(deepseek), null); assert.match(ai.missingConfig(ai.configFrom({})), /API key/); const openrouter = ai.configFrom({ ai_provider: 'openrouter', ai_api_key: 'k' }); assert.match(ai.missingConfig(openrouter), /model/); assert.equal(openrouter.prices.input, null); // A local server needs no key. assert.equal(ai.missingConfig(ai.configFrom({ ai_provider: 'custom', ai_base_url: 'http://localhost:11434/v1', ai_model: 'llava' })), null); }); test('the prompt carries the model details and, when given, its image', () => { const [system, user] = ai.buildMessages(MODEL, { image: 'data:image/png;base64,AAAA', knownTags: ['dragon', 'toy'] }); assert.equal(system.role, 'system'); const text = user.content[0].text; assert.match(text, /Name: Flexi Dragon/); assert.match(text, /Files: dragon_PLA\.3mf/); assert.match(text, /Size: 120 × 40 × 30 mm/); assert.match(text, /Tags already used in this library: dragon, toy/); assert.deepEqual(user.content[1], { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } }); assert.equal(typeof ai.buildMessages(MODEL)[1].content, 'string'); }); test('replies are read leniently and tags tidied', () => { const r = ai.parseReply('Sure!\n```json\n{"tags": ["#Dragon", "3D print", "Print_In Place", "dragon", "a", "b", "c", "d", "e", "f"], "description": " A flexible dragon toy. "}\n```'); assert.deepEqual(r.tags, ['dragon', 'print in place', 'a', 'b', 'c', 'd', 'e', 'f']); assert.equal(r.description, 'A flexible dragon toy.'); assert.deepEqual(ai.parseReply('{"tags": "hook, wall mount", "description": ""}').tags, ['hook', 'wall mount']); assert.throws(() => ai.parseReply('I cannot help with that'), /JSON/); assert.throws(() => ai.parseReply('{"tags": [], "description": ""}'), /no tags/); }); test('a clearer name is suggested only when asked, and only when it changes', () => { assert.match(ai.buildMessages(MODEL, { suggestName: true })[0].content, /"name"/); assert.doesNotMatch(ai.buildMessages(MODEL)[0].content, /"name"/); const reply = '{"name": "“Articulated Dragon Pair.3mf”", "tags": ["dragon"], "description": "Two dragons."}'; assert.equal(ai.parseReply(reply, { currentName: 'Dra g _v4' }).name, 'Articulated Dragon Pair'); assert.equal(ai.parseReply('{"name": "flexi dragon!", "tags": ["dragon"]}', { currentName: 'Flexi Dragon' }).name, null); assert.equal(ai.parseReply(reply).name, null); // not asked for }); test('cost comes from the provider when reported, else from your prices', () => { assert.equal(ai.costOf({ prompt_tokens: 1e6, completion_tokens: 1e6, cost: 0.0123 }, { input: 1, output: 1 }), 0.0123); assert.equal(ai.costOf({ prompt_tokens: 2e6, completion_tokens: 1e6 }, { input: 0.3, output: 1.2 }), 1.8); assert.equal(ai.costOf({ prompt_tokens: 10, completion_tokens: 10 }, { input: null, output: null }), null); }); test('a model that refuses JSON mode and images is retried without them', async () => { const provider = await fakeProvider([ () => [400, { error: { message: 'response_format is not supported by this model' } }], () => [400, { error: { message: 'This model does not support image input' } }], () => reply('{"tags": ["dragon", "articulated"], "description": "A flexi dragon."}'), ]); try { const result = await ai.describeModel(provider.config, MODEL, { image: 'data:image/png;base64,AAAA' }); assert.deepEqual(result.tags, ['dragon', 'articulated']); assert.equal(result.imageRejected, true); assert.equal(result.imageUsed, false); assert.deepEqual(result.usage, { prompt: 1000, completion: 100 }); assert.ok(Math.abs(result.cost - 0.00042) < 1e-9); assert.equal(provider.requests.length, 3); assert.equal(provider.requests[0].path, '/v1/chat/completions'); assert.equal(provider.requests[0].auth, 'Bearer sk-test'); assert.ok(provider.requests[0].body.response_format); assert.equal(provider.requests[1].body.response_format, undefined); assert.equal(typeof provider.requests[2].body.messages[1].content, 'string'); // no image part } finally { await provider.close(); } }); test('collection ideas are asked for with the library’s collections, and tidied', () => { const [system, user] = ai.buildMessages(MODEL, { describe: false, suggestCollections: true, collectionNames: ['Bookends', 'Halloween'] }); assert.match(system.content, /"collections"/); assert.doesNotMatch(system.content, /"tags"/); assert.match(user.content, /Existing collections in this library: Bookends, Halloween/); assert.match(ai.buildMessages(MODEL, { suggestCollections: true })[1].content, /no collections yet/); const reply = '{"collections": ["bookends", " Halloween Decor ", "Book-ends", "Dragons", "Toys"]}'; const r = ai.parseReply(reply, { describe: false, collections: true }); assert.deepEqual(r.collections, ['bookends', 'Halloween Decor', 'Dragons']); assert.deepEqual(r.tags, []); // not asked for, so an answer without tags is fine assert.deepEqual(ai.parseReply('{"tags": ["dragon"], "collections": ["Dragons"]}').collections, []); }); test('a reply that isn’t JSON is asked again, and every answer counts towards cost', async () => { const provider = await fakeProvider([ () => reply('Sure! Tags: dragon, toy'), () => reply('{"tags": ["dragon"], "description": "A dragon."}'), ]); try { const result = await ai.describeModel(provider.config, MODEL); assert.deepEqual(result.tags, ['dragon']); assert.equal(result.replyRetried, true); assert.deepEqual(result.usage, { prompt: 2000, completion: 200 }); assert.ok(Math.abs(result.cost - 0.00084) < 1e-9); assert.equal(provider.requests.length, 2); } finally { await provider.close(); } // Empty replies in JSON mode: the last try goes without JSON mode. const empty = await fakeProvider([() => reply(''), () => reply(''), () => reply('{"tags": ["dragon"], "description": "A dragon."}')]); try { const result = await ai.describeModel(empty.config, MODEL); assert.deepEqual(result.tags, ['dragon']); assert.ok(empty.requests[1].body.response_format); assert.equal(empty.requests[2].body.response_format, undefined); } finally { await empty.close(); } // Still nothing usable: the error says what came back and carries the spend. const stubborn = await fakeProvider([() => reply('')]); try { const err = await ai.describeModel(stubborn.config, MODEL).catch((e) => e); assert.match(err.message, /reply was empty, not the JSON asked for/); assert.equal(stubborn.requests.length, 3); assert.deepEqual(err.usage, { prompt: 3000, completion: 300 }); } finally { await stubborn.close(); } }); test('a reply that was cut off is asked again with more room', async () => { const cutOff = [200, { choices: [{ message: { content: '{"tags": ["red panda", "fl' }, finish_reason: 'length' }], usage: { prompt_tokens: 1000, completion_tokens: 100 } }]; const provider = await fakeProvider([() => cutOff, () => reply('{"tags": ["red panda"], "description": "A flexi red panda."}')]); try { const result = await ai.describeModel(provider.config, MODEL); assert.deepEqual(result.tags, ['red panda']); assert.equal(provider.requests[0].body.max_tokens, 1500); assert.equal(provider.requests[1].body.max_tokens, 3000); } finally { await provider.close(); } assert.throws(() => ai.parseReply('{"tags": ["red', { finishReason: 'length' }), /cut off\)\. It began: “\{"tags": \["red”/); }); test('JSON is found inside fences, notes and braces after it', () => { assert.deepEqual(ai.parseReply('```json\n{"tags": ["owl"], "description": "An owl {bookend}."}\n```').tags, ['owl']); assert.equal(ai.parseReply('Here you go: {"tags": ["owl"], "description": "An owl."} Note: fields were {tags, description}.').description, 'An owl.'); assert.throws(() => ai.parseReply('I cannot help with that'), /It began: “I cannot help with that”/); }); test('the prompt shows the reply format for just the fields asked for', () => { const plain = ai.buildMessages(MODEL)[0].content; assert.match(plain, /The format, shown for a different model: \{"tags":\["wall hook","headphone holder"\],"description":"A wall hook for hanging headphones\."\}/); const collectionsOnly = ai.buildMessages(MODEL, { describe: false, suggestCollections: true })[0].content; assert.match(collectionsOnly, /\{"collections":\["Desk Organizers"\]\}/); }); test('text sent to the AI never holds half an emoji', () => { // The panda starts at unit 799, so a plain cut at 800 would split it. const description = `${'a'.repeat(799)}🐼 red panda`; const [, user] = ai.buildMessages({ name: 'Red panda', files: [], tags: [], description }); assert.ok(user.content.isWellFormed()); assert.ok(!user.content.includes('\uFFFD'), 'clipped before the emoji, not replaced'); assert.match(user.content, /Description so far: a{799}$/m); // Text that's already broken in the library is made safe before it's sent. const [, broken] = ai.buildMessages({ name: 'Bad \uD83D name', files: [], tags: ['ok'], description: '' }, { knownTags: ['tag\uDC3C'] }); assert.ok(broken.content.isWellFormed()); const reply = JSON.stringify({ name: `x${'🐼'.repeat(50)}`, tags: ['panda'], description: `x${'🐼'.repeat(400)}`, collections: [`x${'🐼'.repeat(40)}`] }); const parsed = ai.parseReply(reply, { currentName: 'Red panda', collections: true }); assert.ok(parsed.name.isWellFormed() && parsed.description.isWellFormed() && parsed.collections[0].isWellFormed()); }); test('a rejected key is fatal for a run; a bad reply is not', async () => { const provider = await fakeProvider([() => [401, { error: { message: 'Authentication Fails' } }]]); try { const err = await ai.describeModel(provider.config, MODEL).catch((e) => e); assert.match(ai.fatalMessage(err), /API key was rejected/); } finally { await provider.close(); } assert.equal(ai.fatalMessage(new Error('The AI reply wasn’t the JSON asked for')), null); assert.match(ai.fatalMessage(new ai.AiError('nope', 402)), /out of credit/); });