printhive/test/ai.test.js

// AI tagging client (lib/ai.js), against a fake OpenAI-compatible server.
const test = require('node:test');
const assert = require('node:assert/strict');
const http = require('http');
const ai = require('../lib/ai');

// Starts a server that answers each request with the next handler's result.
async function fakeProvider(handlers) {
  const requests = [];
  const server = http.createServer((req, res) => {
    let body = '';
    req.on('data', (c) => { body += c; });
    req.on('end', () => {
      const parsed = JSON.parse(body);
      requests.push({ path: req.url, auth: req.headers.authorization, body: parsed });
      const [status, payload] = handlers[Math.min(requests.length, handlers.length) - 1](parsed);
      res.writeHead(status, { 'Content-Type': 'application/json' });
      res.end(JSON.stringify(payload));
    });
  });
  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
  const config = ai.configFrom({
    ai_provider: 'custom', ai_base_url: `http://127.0.0.1:${server.address().port}/v1/`, ai_api_key: 'sk-test',
    ai_model: 'test-model', ai_price_input: '0.3', ai_price_output: '1.2',
  });
  return { config, requests, close: () => new Promise((resolve) => server.close(resolve)) };
}

const reply = (content, usage = { prompt_tokens: 1000, completion_tokens: 100 }) =>
  [200, { choices: [{ message: { content } }], usage }];

const MODEL = { name: 'Flexi Dragon', files: [{ originalName: 'dragon_PLA.3mf', bbox: { x: 120, y: 40, z: 30 } }], tags: ['toy'] };

test('presets fill in DeepSeek by default, and your own values win', () => {
  const deepseek = ai.configFrom({ ai_api_key: 'k' });
  assert.equal(deepseek.provider, 'deepseek');
  assert.equal(deepseek.baseUrl, 'https://api.deepseek.com');
  assert.equal(deepseek.model, 'deepseek-flash');
  assert.equal(ai.missingConfig(deepseek), null);
  assert.match(ai.missingConfig(ai.configFrom({})), /API key/);

  const openrouter = ai.configFrom({ ai_provider: 'openrouter', ai_api_key: 'k' });
  assert.match(ai.missingConfig(openrouter), /model/);
  assert.equal(openrouter.prices.input, null);
  // A local server needs no key.
  assert.equal(ai.missingConfig(ai.configFrom({ ai_provider: 'custom', ai_base_url: 'http://localhost:11434/v1', ai_model: 'llava' })), null);
});

test('the prompt carries the model details and, when given, its image', () => {
  const [system, user] = ai.buildMessages(MODEL, { image: 'data:image/png;base64,AAAA', knownTags: ['dragon', 'toy'] });
  assert.equal(system.role, 'system');
  const text = user.content[0].text;
  assert.match(text, /Name: Flexi Dragon/);
  assert.match(text, /Files: dragon_PLA\.3mf/);
  assert.match(text, /Size: 120 × 40 × 30 mm/);
  assert.match(text, /Tags already used in this library: dragon, toy/);
  assert.deepEqual(user.content[1], { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } });
  assert.equal(typeof ai.buildMessages(MODEL)[1].content, 'string');
});

test('replies are read leniently and tags tidied', () => {
  const r = ai.parseReply('Sure!\n```json\n{"tags": ["#Dragon", "3D print", "Print_In Place", "dragon", "a", "b", "c", "d", "e", "f"], "description": " A flexible dragon toy. "}\n```');
  assert.deepEqual(r.tags, ['dragon', 'print in place', 'a', 'b', 'c', 'd', 'e', 'f']);
  assert.equal(r.description, 'A flexible dragon toy.');
  assert.deepEqual(ai.parseReply('{"tags": "hook, wall mount", "description": ""}').tags, ['hook', 'wall mount']);
  assert.throws(() => ai.parseReply('I cannot help with that'), /JSON/);
  assert.throws(() => ai.parseReply('{"tags": [], "description": ""}'), /no tags/);
});

test('a clearer name is suggested only when asked, and only when it changes', () => {
  assert.match(ai.buildMessages(MODEL, { suggestName: true })[0].content, /"name"/);
  assert.doesNotMatch(ai.buildMessages(MODEL)[0].content, /"name"/);
  const reply = '{"name": "“Articulated Dragon Pair.3mf”", "tags": ["dragon"], "description": "Two dragons."}';
  assert.equal(ai.parseReply(reply, { currentName: 'Dra g _v4' }).name, 'Articulated Dragon Pair');
  assert.equal(ai.parseReply('{"name": "flexi  dragon!", "tags": ["dragon"]}', { currentName: 'Flexi Dragon' }).name, null);
  assert.equal(ai.parseReply(reply).name, null); // not asked for
});

test('cost comes from the provider when reported, else from your prices', () => {
  assert.equal(ai.costOf({ prompt_tokens: 1e6, completion_tokens: 1e6, cost: 0.0123 }, { input: 1, output: 1 }), 0.0123);
  assert.equal(ai.costOf({ prompt_tokens: 2e6, completion_tokens: 1e6 }, { input: 0.3, output: 1.2 }), 1.8);
  assert.equal(ai.costOf({ prompt_tokens: 10, completion_tokens: 10 }, { input: null, output: null }), null);
});

test('a model that refuses JSON mode and images is retried without them', async () => {
  const provider = await fakeProvider([
    () => [400, { error: { message: 'response_format is not supported by this model' } }],
    () => [400, { error: { message: 'This model does not support image input' } }],
    () => reply('{"tags": ["dragon", "articulated"], "description": "A flexi dragon."}'),
  ]);
  try {
    const result = await ai.describeModel(provider.config, MODEL, { image: 'data:image/png;base64,AAAA' });
    assert.deepEqual(result.tags, ['dragon', 'articulated']);
    assert.equal(result.imageRejected, true);
    assert.equal(result.imageUsed, false);
    assert.deepEqual(result.usage, { prompt: 1000, completion: 100 });
    assert.ok(Math.abs(result.cost - 0.00042) < 1e-9);

    assert.equal(provider.requests.length, 3);
    assert.equal(provider.requests[0].path, '/v1/chat/completions');
    assert.equal(provider.requests[0].auth, 'Bearer sk-test');
    assert.ok(provider.requests[0].body.response_format);
    assert.equal(provider.requests[1].body.response_format, undefined);
    assert.equal(typeof provider.requests[2].body.messages[1].content, 'string'); // no image part
  } finally {
    await provider.close();
  }
});

test('collection ideas are asked for with the library’s collections, and tidied', () => {
  const [system, user] = ai.buildMessages(MODEL, { describe: false, suggestCollections: true, collectionNames: ['Bookends', 'Halloween'] });
  assert.match(system.content, /"collections"/);
  assert.doesNotMatch(system.content, /"tags"/);
  assert.match(user.content, /Existing collections in this library: Bookends, Halloween/);
  assert.match(ai.buildMessages(MODEL, { suggestCollections: true })[1].content, /no collections yet/);

  const reply = '{"collections": ["bookends", " Halloween  Decor ", "Book-ends", "Dragons", "Toys"]}';
  const r = ai.parseReply(reply, { describe: false, collections: true });
  assert.deepEqual(r.collections, ['bookends', 'Halloween Decor', 'Dragons']);
  assert.deepEqual(r.tags, []); // not asked for, so an answer without tags is fine
  assert.deepEqual(ai.parseReply('{"tags": ["dragon"], "collections": ["Dragons"]}').collections, []);
});

test('a reply that isn’t JSON is asked again once, and both answers count towards cost', async () => {
  const provider = await fakeProvider([
    () => reply('Sure! Tags: dragon, toy'),
    () => reply('{"tags": ["dragon"], "description": "A dragon."}'),
  ]);
  try {
    const result = await ai.describeModel(provider.config, MODEL);
    assert.deepEqual(result.tags, ['dragon']);
    assert.equal(result.replyRetried, true);
    assert.deepEqual(result.usage, { prompt: 2000, completion: 200 });
    assert.ok(Math.abs(result.cost - 0.00084) < 1e-9);
    assert.equal(provider.requests.length, 2);
  } finally {
    await provider.close();
  }

  const stubborn = await fakeProvider([() => reply('no JSON here')]);
  try {
    await assert.rejects(ai.describeModel(stubborn.config, MODEL), /JSON/);
    assert.equal(stubborn.requests.length, 2);
  } finally {
    await stubborn.close();
  }
});

test('a rejected key is fatal for a run; a bad reply is not', async () => {
  const provider = await fakeProvider([() => [401, { error: { message: 'Authentication Fails' } }]]);
  try {
    const err = await ai.describeModel(provider.config, MODEL).catch((e) => e);
    assert.match(ai.fatalMessage(err), /API key was rejected/);
  } finally {
    await provider.close();
  }
  assert.equal(ai.fatalMessage(new Error('The AI reply wasn’t the JSON asked for')), null);
  assert.match(ai.fatalMessage(new ai.AiError('nope', 402)), /out of credit/);
});