printhive/test/ai.test.js

// AI tagging client (lib/ai.js), against a fake OpenAI-compatible server.
const test = require('node:test');
const assert = require('node:assert/strict');
const http = require('http');
const ai = require('../lib/ai');

// Starts a server that answers each request with the next handler's result.
async function fakeProvider(handlers) {
  const requests = [];
  const server = http.createServer((req, res) => {
    let body = '';
    req.on('data', (c) => { body += c; });
    req.on('end', () => {
      const parsed = JSON.parse(body);
      requests.push({ path: req.url, auth: req.headers.authorization, body: parsed });
      const [status, payload] = handlers[Math.min(requests.length, handlers.length) - 1](parsed);
      res.writeHead(status, { 'Content-Type': 'application/json' });
      res.end(JSON.stringify(payload));
    });
  });
  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
  const config = ai.configFrom({
    ai_provider: 'custom', ai_base_url: `http://127.0.0.1:${server.address().port}/v1/`, ai_api_key: 'sk-test',
    ai_model: 'test-model', ai_price_input: '0.3', ai_price_output: '1.2',
  });
  return { config, requests, close: () => new Promise((resolve) => server.close(resolve)) };
}

const reply = (content, usage = { prompt_tokens: 1000, completion_tokens: 100 }) =>
  [200, { choices: [{ message: { content } }], usage }];

const MODEL = { name: 'Flexi Dragon', files: [{ originalName: 'dragon_PLA.3mf', bbox: { x: 120, y: 40, z: 30 } }], tags: ['toy'] };

test('presets fill in DeepSeek by default, and your own values win', () => {
  const deepseek = ai.configFrom({ ai_api_key: 'k' });
  assert.equal(deepseek.provider, 'deepseek');
  assert.equal(deepseek.baseUrl, 'https://api.deepseek.com');
  assert.equal(deepseek.model, 'deepseek-flash');
  assert.equal(ai.missingConfig(deepseek), null);
  assert.match(ai.missingConfig(ai.configFrom({})), /API key/);

  const openrouter = ai.configFrom({ ai_provider: 'openrouter', ai_api_key: 'k' });
  assert.match(ai.missingConfig(openrouter), /model/);
  assert.equal(openrouter.prices.input, null);
  // A local server needs no key.
  assert.equal(ai.missingConfig(ai.configFrom({ ai_provider: 'custom', ai_base_url: 'http://localhost:11434/v1', ai_model: 'llava' })), null);
});

test('the prompt carries the model details and, when given, its image', () => {
  const [system, user] = ai.buildMessages(MODEL, { image: 'data:image/png;base64,AAAA', knownTags: ['dragon', 'toy'] });
  assert.equal(system.role, 'system');
  const text = user.content[0].text;
  assert.match(text, /Name: Flexi Dragon/);
  assert.match(text, /Files: dragon_PLA\.3mf/);
  assert.match(text, /Size: 120 × 40 × 30 mm/);
  assert.match(text, /Tags already used in this library: dragon, toy/);
  assert.deepEqual(user.content[1], { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } });
  assert.equal(typeof ai.buildMessages(MODEL)[1].content, 'string');
});

test('replies are read leniently and tags tidied', () => {
  const r = ai.parseReply('Sure!\n```json\n{"tags": ["#Dragon", "3D print", "Print_In Place", "dragon", "a", "b", "c", "d", "e", "f"], "description": " A flexible dragon toy. "}\n```');
  assert.deepEqual(r.tags, ['dragon', 'print in place', 'a', 'b', 'c', 'd', 'e', 'f']);
  assert.equal(r.description, 'A flexible dragon toy.');
  assert.deepEqual(ai.parseReply('{"tags": "hook, wall mount", "description": ""}').tags, ['hook', 'wall mount']);
  assert.throws(() => ai.parseReply('I cannot help with that'), /JSON/);
  assert.throws(() => ai.parseReply('{"tags": [], "description": ""}'), /no tags/);
});

test('a clearer name is suggested only when asked, and only when it changes', () => {
  assert.match(ai.buildMessages(MODEL, { suggestName: true })[0].content, /"name"/);
  assert.doesNotMatch(ai.buildMessages(MODEL)[0].content, /"name"/);
  const reply = '{"name": "“Articulated Dragon Pair.3mf”", "tags": ["dragon"], "description": "Two dragons."}';
  assert.equal(ai.parseReply(reply, { currentName: 'Dra g _v4' }).name, 'Articulated Dragon Pair');
  assert.equal(ai.parseReply('{"name": "flexi  dragon!", "tags": ["dragon"]}', { currentName: 'Flexi Dragon' }).name, null);
  assert.equal(ai.parseReply(reply).name, null); // not asked for
});

test('cost comes from the provider when reported, else from your prices', () => {
  assert.equal(ai.costOf({ prompt_tokens: 1e6, completion_tokens: 1e6, cost: 0.0123 }, { input: 1, output: 1 }), 0.0123);
  assert.equal(ai.costOf({ prompt_tokens: 2e6, completion_tokens: 1e6 }, { input: 0.3, output: 1.2 }), 1.8);
  assert.equal(ai.costOf({ prompt_tokens: 10, completion_tokens: 10 }, { input: null, output: null }), null);
});

test('a model that refuses JSON mode and images is retried without them', async () => {
  const provider = await fakeProvider([
    () => [400, { error: { message: 'response_format is not supported by this model' } }],
    () => [400, { error: { message: 'This model does not support image input' } }],
    () => reply('{"tags": ["dragon", "articulated"], "description": "A flexi dragon."}'),
  ]);
  try {
    const result = await ai.describeModel(provider.config, MODEL, { image: 'data:image/png;base64,AAAA' });
    assert.deepEqual(result.tags, ['dragon', 'articulated']);
    assert.equal(result.imageRejected, true);
    assert.equal(result.imageUsed, false);
    assert.deepEqual(result.usage, { prompt: 1000, completion: 100 });
    assert.ok(Math.abs(result.cost - 0.00042) < 1e-9);

    assert.equal(provider.requests.length, 3);
    assert.equal(provider.requests[0].path, '/v1/chat/completions');
    assert.equal(provider.requests[0].auth, 'Bearer sk-test');
    assert.ok(provider.requests[0].body.response_format);
    assert.equal(provider.requests[1].body.response_format, undefined);
    assert.equal(typeof provider.requests[2].body.messages[1].content, 'string'); // no image part
  } finally {
    await provider.close();
  }
});

test('collection ideas are asked for with the library’s collections, and tidied', () => {
  const [system, user] = ai.buildMessages(MODEL, { describe: false, suggestCollections: true, collectionNames: ['Bookends', 'Halloween'] });
  assert.match(system.content, /"collections"/);
  assert.doesNotMatch(system.content, /"tags"/);
  assert.match(user.content, /Existing collections in this library: Bookends, Halloween/);
  assert.match(ai.buildMessages(MODEL, { suggestCollections: true })[1].content, /no collections yet/);

  const reply = '{"collections": ["bookends", " Halloween  Decor ", "Book-ends", "Dragons", "Toys"]}';
  const r = ai.parseReply(reply, { describe: false, collections: true });
  assert.deepEqual(r.collections, ['bookends', 'Halloween Decor', 'Dragons']);
  assert.deepEqual(r.tags, []); // not asked for, so an answer without tags is fine
  assert.deepEqual(ai.parseReply('{"tags": ["dragon"], "collections": ["Dragons"]}').collections, []);
});

test('a reply that isn’t JSON is asked again, and every answer counts towards cost', async () => {
  const provider = await fakeProvider([
    () => reply('Sure! Tags: dragon, toy'),
    () => reply('{"tags": ["dragon"], "description": "A dragon."}'),
  ]);
  try {
    const result = await ai.describeModel(provider.config, MODEL);
    assert.deepEqual(result.tags, ['dragon']);
    assert.equal(result.replyRetried, true);
    assert.deepEqual(result.usage, { prompt: 2000, completion: 200 });
    assert.ok(Math.abs(result.cost - 0.00084) < 1e-9);
    assert.equal(provider.requests.length, 2);
  } finally {
    await provider.close();
  }

  // Empty replies in JSON mode: the last try goes without JSON mode.
  const empty = await fakeProvider([() => reply(''), () => reply(''), () => reply('{"tags": ["dragon"], "description": "A dragon."}')]);
  try {
    const result = await ai.describeModel(empty.config, MODEL);
    assert.deepEqual(result.tags, ['dragon']);
    assert.ok(empty.requests[1].body.response_format);
    assert.equal(empty.requests[2].body.response_format, undefined);
  } finally {
    await empty.close();
  }

  // Still nothing usable: the error says what came back and carries the spend.
  const stubborn = await fakeProvider([() => reply('')]);
  try {
    const err = await ai.describeModel(stubborn.config, MODEL).catch((e) => e);
    assert.match(err.message, /reply was empty, not the JSON asked for/);
    assert.equal(stubborn.requests.length, 3);
    assert.deepEqual(err.usage, { prompt: 3000, completion: 300 });
  } finally {
    await stubborn.close();
  }
});

test('a reply that was cut off is asked again with more room', async () => {
  const cutOff = [200, { choices: [{ message: { content: '{"tags": ["red panda", "fl' }, finish_reason: 'length' }], usage: { prompt_tokens: 1000, completion_tokens: 100 } }];
  const provider = await fakeProvider([() => cutOff, () => reply('{"tags": ["red panda"], "description": "A flexi red panda."}')]);
  try {
    const result = await ai.describeModel(provider.config, MODEL);
    assert.deepEqual(result.tags, ['red panda']);
    assert.equal(provider.requests[0].body.max_tokens, 1500);
    assert.equal(provider.requests[1].body.max_tokens, 3000);
  } finally {
    await provider.close();
  }
  assert.throws(() => ai.parseReply('{"tags": ["red', { finishReason: 'length' }),
    /cut off\)\. It began: “\{"tags": \["red”/);
});

test('JSON is found inside fences, notes and braces after it', () => {
  assert.deepEqual(ai.parseReply('```json\n{"tags": ["owl"], "description": "An owl {bookend}."}\n```').tags, ['owl']);
  assert.equal(ai.parseReply('Here you go: {"tags": ["owl"], "description": "An owl."} Note: fields were {tags, description}.').description, 'An owl.');
  assert.throws(() => ai.parseReply('I cannot help with that'), /It began: “I cannot help with that”/);
});

test('the prompt shows the reply format for just the fields asked for', () => {
  const plain = ai.buildMessages(MODEL)[0].content;
  assert.match(plain, /The format, shown for a different model: \{"tags":\["wall hook","headphone holder"\],"description":"A wall hook for hanging headphones\."\}/);
  const collectionsOnly = ai.buildMessages(MODEL, { describe: false, suggestCollections: true })[0].content;
  assert.match(collectionsOnly, /\{"collections":\["Desk Organizers"\]\}/);
});

test('text sent to the AI never holds half an emoji', () => {
  // The panda starts at unit 799, so a plain cut at 800 would split it.
  const description = `${'a'.repeat(799)}🐼 red panda`;
  const [, user] = ai.buildMessages({ name: 'Red panda', files: [], tags: [], description });
  assert.ok(user.content.isWellFormed());
  assert.ok(!user.content.includes('\uFFFD'), 'clipped before the emoji, not replaced');
  assert.match(user.content, /Description so far: a{799}$/m);

  // Text that's already broken in the library is made safe before it's sent.
  const [, broken] = ai.buildMessages({ name: 'Bad \uD83D name', files: [], tags: ['ok'], description: '' }, { knownTags: ['tag\uDC3C'] });
  assert.ok(broken.content.isWellFormed());

  const reply = JSON.stringify({ name: `x${'🐼'.repeat(50)}`, tags: ['panda'], description: `x${'🐼'.repeat(400)}`, collections: [`x${'🐼'.repeat(40)}`] });
  const parsed = ai.parseReply(reply, { currentName: 'Red panda', collections: true });
  assert.ok(parsed.name.isWellFormed() && parsed.description.isWellFormed() && parsed.collections[0].isWellFormed());
});

test('a rejected key is fatal for a run; a bad reply is not', async () => {
  const provider = await fakeProvider([() => [401, { error: { message: 'Authentication Fails' } }]]);
  try {
    const err = await ai.describeModel(provider.config, MODEL).catch((e) => e);
    assert.match(ai.fatalMessage(err), /API key was rejected/);
  } finally {
    await provider.close();
  }
  assert.equal(ai.fatalMessage(new Error('The AI reply wasn’t the JSON asked for')), null);
  assert.match(ai.fatalMessage(new ai.AiError('nope', 402)), /out of credit/);
});