printhive/lib/store.js
dlawler489 0f49fe8c1a AI tags and descriptions, search by meaning, and automatic tags
- AI tagging through any OpenAI-compatible API: DeepSeek by default, or
  OpenRouter, OpenAI, or a local server (Ollama, LM Studio). Settings → AI
  with presets, prices and "Save & test connection"; the key never leaves
  the server except to the provider.
- Runs over untagged models, all models, or one model from its page: live
  progress, tokens and cost, stop/resume (retrying failures), an optional
  spending limit, and paused rather than resumed after a restart. Adapts to
  models without JSON mode or image input; stops on a rejected key or no
  credit. Tags are only added; descriptions filled only where empty unless
  asked.
- Search by meaning: local bge-small-en-v1.5 embeddings (transformers.js),
  indexed in the background whenever a model's text changes; Words/Meaning
  switch on the search box. Stored as REAL[] in Postgres, no pgvector needed.
- Automatic tags: strict whole-word rules (part types, mechanisms, materials,
  techniques) over names, file names and archive contents, plus your own
  word -> tag rules; on/off (optionally removing its tags) and re-apply.
- Tags remember where they came from (user / rule / ai); edits keep that,
  rule and AI passes never touch your own tags, and the model page marks them.
- Docker: ships the embedding model and only Linux onnxruntime binaries.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-15 11:12:58 +10:00

542 lines
22 KiB
JavaScript

const db = require('./db');
const { MODEL_FORMATS, ARCHIVE_FORMATS } = require('./fileProcessor');
// ---- shared SELECT that hydrates a model with tags, projects, files, designer ----
const MODEL_SELECT = `
SELECT m.id, m.name, m.description, m.source_site, m.source_url, m.source_id,
m.license, m.commercial_use, m.thumbnail_path, m.print_settings, m.date_added, m.date_modified,
m.notes, m.deleted_at, m.ai_tagged_at,
d.id AS designer_id, d.name AS designer_name, d.profile_url AS designer_url,
COALESCE(t.tags, '[]'::json) AS tags,
COALESCE(t.tag_sources, '{}'::json) AS tag_sources,
COALESCE(p.projects, '[]'::json) AS projects,
COALESCE(f.files, '[]'::json) AS files
FROM models m
LEFT JOIN designers d ON d.id = m.designer_id
LEFT JOIN LATERAL (
SELECT json_agg(tg.name ORDER BY tg.name) AS tags, json_object_agg(tg.name, mt.source) AS tag_sources
FROM model_tags mt JOIN tags tg ON tg.id = mt.tag_id
WHERE mt.model_id = m.id
) t ON true
LEFT JOIN LATERAL (
SELECT json_agg(json_build_object('id', pr.id, 'name', pr.name) ORDER BY pr.name) AS projects
FROM model_projects mp JOIN projects pr ON pr.id = mp.project_id
WHERE mp.model_id = m.id
) p ON true
LEFT JOIN LATERAL (
SELECT json_agg(json_build_object(
'id', fl.id, 'originalName', fl.original_name, 'fileName', fl.file_name,
'format', fl.format, 'size', fl.size, 'volume', fl.volume,
'bbox', fl.bbox, 'facets', fl.facets, 'archiveStatus', fl.archive_status,
'entryCount', (SELECT COUNT(*)::int FROM archive_entries ae WHERE ae.file_id = fl.id)) ORDER BY fl.id) AS files
FROM files fl WHERE fl.model_id = m.id
) f ON true
`;
function shapeModel(row) {
if (!row) return null;
const { designer_id, designer_name, designer_url, ...rest } = row;
return {
...rest,
designer: designer_id ? { id: designer_id, name: designer_name, url: designer_url } : null,
};
}
// ---------------------------------------------------------------- models
// The library, or with { trash: true } the models in the trash.
async function getAllModels({ trash = false } = {}) {
const { rows } = await db.query(trash
? `${MODEL_SELECT} WHERE m.deleted_at IS NOT NULL ORDER BY m.deleted_at DESC`
: `${MODEL_SELECT} WHERE m.deleted_at IS NULL ORDER BY m.date_added DESC`);
return rows.map(shapeModel);
}
async function getModel(id) {
const { rows } = await db.query(`${MODEL_SELECT} WHERE m.id = $1`, [id]);
return shapeModel(rows[0]);
}
async function getModelsByIds(ids) {
if (!ids.length) return [];
const { rows } = await db.query(`${MODEL_SELECT} WHERE m.id = ANY($1::int[])`, [ids]);
return rows.map(shapeModel);
}
async function findModelBySource(site, sourceId) {
const { rows } = await db.query(
`${MODEL_SELECT} WHERE m.source_site = $1 AND m.source_id = $2 AND m.deleted_at IS NULL`, [site, sourceId]);
return shapeModel(rows[0]);
}
// data = { name, description, source_site, source_url, source_id, license,
// designer:{name,source_site,profile_url}, thumbnail_path, print_settings,
// tags:[string], projects:[id], files:[{...}] }
async function createModel(data) {
return db.transaction(async (client) => {
let designerId = null;
if (data.designer && data.designer.name) {
designerId = await upsertDesigner(client, data.designer);
}
const { rows } = await client.query(
`INSERT INTO models (name, description, source_site, source_url, source_id,
license, commercial_use, designer_id, thumbnail_path, print_settings)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING id`,
[data.name, data.description || '', data.source_site || null, data.source_url || null,
data.source_id || null, data.license || null, data.commercial_use ?? null, designerId,
data.thumbnail_path || null, data.print_settings || {}]
);
const modelId = rows[0].id;
await setTags(client, modelId, data.tags || []);
await setProjects(client, modelId, data.projects || []);
for (const file of data.files || []) {
await insertFile(client, modelId, file);
}
return modelId;
}).then(getModel);
}
async function updateModel(id, updates) {
await db.transaction(async (client) => {
const fields = [];
const values = [];
let i = 1;
for (const col of ['name', 'description', 'notes', 'license', 'commercial_use', 'thumbnail_path']) {
if (updates[col] !== undefined) { fields.push(`${col} = $${i++}`); values.push(updates[col]); }
}
if (updates.print_settings !== undefined) {
fields.push(`print_settings = $${i++}`); values.push(updates.print_settings);
}
if (fields.length) {
fields.push(`date_modified = now()`);
values.push(id);
await client.query(`UPDATE models SET ${fields.join(', ')} WHERE id = $${i}`, values);
}
if (updates.tags !== undefined) await setTags(client, id, updates.tags);
if (updates.projects !== undefined) await setProjects(client, id, updates.projects);
});
return getModel(id);
}
// Returns the on-disk file rows so the caller can unlink them.
async function deleteModel(id) {
const { rows } = await db.query('SELECT id, file_path, glb_path FROM files WHERE model_id = $1', [id]);
const entries = await db.query(
'SELECT e.id FROM archive_entries e JOIN files f ON f.id = e.file_id WHERE f.model_id = $1', [id]);
const model = await db.query('SELECT thumbnail_path FROM models WHERE id = $1', [id]);
const result = await db.query('DELETE FROM models WHERE id = $1 RETURNING id', [id]);
if (result.rowCount === 0) return null;
return {
fileIds: rows.map((r) => r.id),
entryIds: entries.rows.map((e) => e.id),
filePaths: rows.map((r) => r.file_path),
glbPaths: [
...rows.map((r) => r.glb_path).filter(Boolean),
...entries.rows.map((e) => `glb/entry-${e.id}.glb`),
],
thumbnailPath: model.rows[0] && model.rows[0].thumbnail_path,
};
}
// ---------------------------------------------------------------- files / archive entries
async function getFile(id) {
const { rows } = await db.query('SELECT * FROM files WHERE id = $1', [id]);
return rows[0] || null;
}
async function getArchiveEntries(fileId) {
const { rows } = await db.query(
'SELECT id, path, format, size FROM archive_entries WHERE file_id = $1 ORDER BY path', [fileId]);
return rows.map((r) => ({ ...r, size: Number(r.size) }));
}
// An entry plus the archive it lives in ({ id, file_path, format }).
async function getArchiveEntry(id) {
const { rows } = await db.query(
`SELECT e.id, e.file_id, e.path, e.format, e.size, f.file_path, f.format AS archive_format
FROM archive_entries e JOIN files f ON f.id = e.file_id WHERE e.id = $1`, [id]);
const r = rows[0];
if (!r) return null;
return {
id: r.id, file_id: r.file_id, path: r.path, format: r.format, size: Number(r.size),
archive: { id: r.file_id, file_path: r.file_path, format: r.archive_format },
};
}
async function replaceArchiveEntries(fileId, entries) {
await db.transaction(async (client) => {
await client.query('DELETE FROM archive_entries WHERE file_id = $1', [fileId]);
if (!entries.length) return;
await client.query(
`INSERT INTO archive_entries (file_id, path, format, size)
SELECT $1::int, * FROM unnest($2::text[], $3::text[], $4::bigint[])
ON CONFLICT (file_id, path) DO NOTHING`,
[fileId, entries.map((e) => e.path), entries.map((e) => e.format), entries.map((e) => e.size)]);
});
}
// ---------------------------------------------------------------- trash
async function trashModels(ids) {
const { rowCount } = await db.query(
'UPDATE models SET deleted_at = now() WHERE id = ANY($1::int[]) AND deleted_at IS NULL', [ids]);
return rowCount;
}
async function restoreModel(id) {
const { rowCount } = await db.query(
'UPDATE models SET deleted_at = NULL WHERE id = $1 AND deleted_at IS NOT NULL', [id]);
return rowCount > 0;
}
async function trashedModelIds() {
const { rows } = await db.query('SELECT id FROM models WHERE deleted_at IS NOT NULL');
return rows.map((r) => r.id);
}
async function countTrashed() {
const { rows } = await db.query('SELECT COUNT(*)::int AS n FROM models WHERE deleted_at IS NOT NULL');
return rows[0].n;
}
// ---------------------------------------------------------------- duplicates
// Files of non-trashed models whose content hash or geometry fingerprint also
// appears in another non-trashed model — the input to the duplicate finder.
// Ids are prefixed ('f12' for a file, 'e7' for an archive entry) so both kinds
// can be matched together. Pictures and documents never count.
async function duplicateCandidateFiles() {
const { rows } = await db.query(`
WITH live AS (
SELECT 'f' || f.id AS id, f.model_id, f.sha256, f.geometry_hash
FROM files f JOIN models m ON m.id = f.model_id
WHERE m.deleted_at IS NULL AND f.size > 0 AND f.format = ANY($1)
UNION ALL
SELECT 'e' || e.id, f.model_id, e.sha256, e.geometry_hash
FROM archive_entries e JOIN files f ON f.id = e.file_id JOIN models m ON m.id = f.model_id
WHERE m.deleted_at IS NULL AND e.size > 0 AND e.format = ANY($2)
)
SELECT * FROM live
WHERE sha256 IN (SELECT sha256 FROM live WHERE sha256 IS NOT NULL
GROUP BY sha256 HAVING COUNT(DISTINCT model_id) > 1)
OR geometry_hash IN (SELECT geometry_hash FROM live WHERE geometry_hash IS NOT NULL
GROUP BY geometry_hash HAVING COUNT(DISTINCT model_id) > 1)`,
[[...MODEL_FORMATS, ...ARCHIVE_FORMATS], MODEL_FORMATS]);
return rows;
}
// How many duplicate-matchable items (model files, archives, models inside
// archives) each model holds: { modelId: n }.
async function matchableCounts(modelIds) {
if (!modelIds.length) return {};
const { rows } = await db.query(`
SELECT m.id,
(SELECT COUNT(*) FROM files f WHERE f.model_id = m.id AND f.format = ANY($2))
+ (SELECT COUNT(*) FROM archive_entries e JOIN files f ON f.id = e.file_id
WHERE f.model_id = m.id AND e.format = ANY($3)) AS n
FROM models m WHERE m.id = ANY($1::int[])`,
[modelIds, [...MODEL_FORMATS, ...ARCHIVE_FORMATS], MODEL_FORMATS]);
return Object.fromEntries(rows.map((r) => [r.id, Number(r.n)]));
}
// Keep one model of a duplicate group: fold the others' tags, collections and
// notes into it, then move the others to the trash.
async function mergeAndTrash(keepId, otherIds) {
await db.transaction(async (client) => {
await client.query(
`INSERT INTO model_tags (model_id, tag_id)
SELECT $1::int, tag_id FROM model_tags WHERE model_id = ANY($2::int[])
ON CONFLICT DO NOTHING`, [keepId, otherIds]);
await client.query(
`INSERT INTO model_projects (model_id, project_id)
SELECT $1::int, project_id FROM model_projects WHERE model_id = ANY($2::int[])
ON CONFLICT DO NOTHING`, [keepId, otherIds]);
const { rows } = await client.query(
'SELECT notes FROM models WHERE id = $1 OR id = ANY($2::int[]) ORDER BY id = $1 DESC, id',
[keepId, otherIds]);
const notes = [...new Set(rows.map((r) => r.notes.trim()).filter(Boolean))].join('\n\n');
await client.query('UPDATE models SET notes = $2, date_modified = now() WHERE id = $1', [keepId, notes]);
await client.query('UPDATE models SET deleted_at = now() WHERE id = ANY($1::int[])', [otherIds]);
});
}
async function dismissDuplicateGroup(signature) {
await db.query(
'INSERT INTO duplicate_dismissals (signature) VALUES ($1) ON CONFLICT DO NOTHING', [signature]);
}
async function dismissedDuplicateSignatures() {
const { rows } = await db.query('SELECT signature FROM duplicate_dismissals');
return rows.map((r) => r.signature);
}
async function addFiles(modelId, files) {
await db.transaction(async (client) => {
for (const file of files) await insertFile(client, modelId, file);
// New files may give the background worker something to make a thumbnail from.
await client.query('UPDATE models SET thumbnail_status = NULL WHERE id = $1', [modelId]);
});
return getModel(modelId);
}
// Record a 3MF's GLB conversion + computed geometry on the file row.
async function updateFileConversion(id, { glb_path, volume, bbox }) {
await db.query(
`UPDATE files SET glb_path = $2,
volume = COALESCE($3, volume),
bbox = COALESCE($4, bbox)
WHERE id = $1`,
[id, glb_path, volume ?? null, bbox ?? null]);
}
async function insertFile(client, modelId, file) {
await client.query(
`INSERT INTO files (model_id, file_name, original_name, file_path, format, size, volume, bbox, facets)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
[modelId, file.fileName, file.originalName, file.filePath, file.format,
file.size || 0, file.volume ?? null, file.bbox ?? null, file.facets ?? null]
);
}
// ---------------------------------------------------------------- tags
async function tagId(client, name) {
const { rows } = await client.query(
`INSERT INTO tags (name) VALUES ($1)
ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id`, [name]);
return rows[0].id;
}
// Sets a model's tags from an edit. A tag that stays keeps where it came from
// (rule / ai); new ones are yours.
async function setTags(client, modelId, tagNames) {
const { rows: before } = await client.query('SELECT tag_id, source FROM model_tags WHERE model_id = $1', [modelId]);
const sourceOf = new Map(before.map((r) => [r.tag_id, r.source]));
await client.query('DELETE FROM model_tags WHERE model_id = $1', [modelId]);
for (const raw of tagNames) {
const name = String(raw).trim();
if (!name) continue;
const id = await tagId(client, name);
await client.query(
'INSERT INTO model_tags (model_id, tag_id, source) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING',
[modelId, id, sourceOf.get(id) || 'user']);
}
}
// Automatic tagging: makes the model's rule tags exactly `tagNames`. A tag the
// model already has from you or the AI is left as it is.
async function replaceRuleTags(modelId, tagNames) {
await db.transaction(async (client) => {
const ids = [];
for (const name of tagNames) ids.push(await tagId(client, name));
await client.query(
`DELETE FROM model_tags WHERE model_id = $1 AND source = 'rule' AND NOT (tag_id = ANY($2::int[]))`, [modelId, ids]);
for (const id of ids) {
await client.query(`INSERT INTO model_tags (model_id, tag_id, source) VALUES ($1,$2,'rule') ON CONFLICT DO NOTHING`,
[modelId, id]);
}
});
}
async function removeTagsFrom(source) {
const { rowCount } = await db.query('DELETE FROM model_tags WHERE source = $1', [source]);
return rowCount;
}
// AI tagging: adds the suggested tags (never removes any) and fills in the
// description if it's empty, or replaces it when asked to.
async function applyAiResult(modelId, { tags, description, overwriteDescription }) {
await db.transaction(async (client) => {
for (const name of tags) {
const id = await tagId(client, name);
await client.query(`INSERT INTO model_tags (model_id, tag_id, source) VALUES ($1,$2,'ai') ON CONFLICT DO NOTHING`,
[modelId, id]);
}
await client.query(
`UPDATE models SET ai_tagged_at = now(),
description = CASE WHEN $2::text <> '' AND ($3::boolean OR btrim(description) = '') THEN $2 ELSE description END
WHERE id = $1`, [modelId, description || '', Boolean(overwriteDescription)]);
});
}
// The library's most used tags, so AI suggestions reuse the same words.
async function topTags(limit = 150) {
const { rows } = await db.query(
`SELECT t.name FROM tags t JOIN model_tags mt ON mt.tag_id = t.id
GROUP BY t.name ORDER BY COUNT(*) DESC, t.name LIMIT $1`, [limit]);
return rows.map((r) => r.name);
}
async function getAllTags() {
const { rows } = await db.query('SELECT name FROM tags ORDER BY name');
return rows.map((r) => r.name);
}
// ---------------------------------------------------------------- projects
async function setProjects(client, modelId, projectIds) {
await client.query('DELETE FROM model_projects WHERE model_id = $1', [modelId]);
for (const pid of projectIds) {
await client.query(
'INSERT INTO model_projects (model_id, project_id) VALUES ($1,$2) ON CONFLICT DO NOTHING',
[modelId, pid]);
}
}
async function getAllProjects() {
const { rows } = await db.query(`
SELECT p.id, p.name, p.description, p.date_created, p.date_modified,
COUNT(m.id)::int AS model_count
FROM projects p
LEFT JOIN model_projects mp ON mp.project_id = p.id
LEFT JOIN models m ON m.id = mp.model_id AND m.deleted_at IS NULL
GROUP BY p.id ORDER BY p.name`);
return rows;
}
async function getProject(id) {
const { rows } = await db.query('SELECT * FROM projects WHERE id = $1', [id]);
return rows[0] || null;
}
async function createProject({ name, description }) {
const { rows } = await db.query(
'INSERT INTO projects (name, description) VALUES ($1,$2) RETURNING *',
[name, description || '']);
return rows[0];
}
async function updateProject(id, { name, description }) {
const { rows } = await db.query(
`UPDATE projects SET name = COALESCE($2, name),
description = COALESCE($3, description), date_modified = now()
WHERE id = $1 RETURNING *`, [id, name ?? null, description ?? null]);
return rows[0] || null;
}
async function deleteProject(id) {
const { rowCount } = await db.query('DELETE FROM projects WHERE id = $1', [id]);
return rowCount > 0;
}
// ---------------------------------------------------------------- license rules
async function getLicenseRule(name) {
const { rows } = await db.query('SELECT name, commercial FROM license_rules WHERE name = $1', [name]);
return rows[0] || null;
}
async function upsertLicenseRule(name, commercial) {
await db.query(
`INSERT INTO license_rules (name, commercial) VALUES ($1, $2)
ON CONFLICT (name) DO UPDATE SET commercial = EXCLUDED.commercial`,
[name, commercial ?? null]);
}
async function listLicenseRules() {
const { rows } = await db.query(`
SELECT lr.name, lr.commercial, COUNT(m.id)::int AS model_count
FROM license_rules lr
LEFT JOIN models m ON m.license = lr.name
GROUP BY lr.name, lr.commercial
ORDER BY lr.name`);
return rows;
}
async function distinctModelLicenses() {
const { rows } = await db.query("SELECT DISTINCT license FROM models WHERE license IS NOT NULL AND license <> ''");
return rows.map((r) => r.license);
}
// Push a rule's value onto every model that carries that license.
async function applyLicenseRuleToModels(name, commercial) {
await db.query('UPDATE models SET commercial_use = $2 WHERE license = $1', [name, commercial ?? null]);
}
// Align every model's commercial_use to its license rule (license = source of truth).
async function reconcileModelsToRules() {
await db.query(`
UPDATE models m SET commercial_use = lr.commercial
FROM license_rules lr
WHERE m.license = lr.name AND m.commercial_use IS DISTINCT FROM lr.commercial`);
}
// ---------------------------------------------------------------- users / auth
async function countUsers() {
const { rows } = await db.query('SELECT COUNT(*)::int AS n FROM users');
return rows[0].n;
}
async function getUserByEmail(email) {
const { rows } = await db.query('SELECT * FROM users WHERE lower(email) = lower($1)', [email]);
return rows[0] || null;
}
async function getUserById(id) {
const { rows } = await db.query('SELECT id, email, created_at FROM users WHERE id = $1', [id]);
return rows[0] || null;
}
async function createUser(email, passwordHash) {
const { rows } = await db.query(
'INSERT INTO users (email, password_hash) VALUES ($1, $2) RETURNING id, email, created_at',
[email.trim(), passwordHash]);
return rows[0];
}
async function updateUserPassword(id, passwordHash) {
await db.query('UPDATE users SET password_hash = $2 WHERE id = $1', [id, passwordHash]);
}
async function createPasswordReset(userId, tokenHash, expiresAt) {
await db.query(
'INSERT INTO password_resets (user_id, token_hash, expires_at) VALUES ($1,$2,$3)',
[userId, tokenHash, expiresAt]);
}
async function getValidReset(tokenHash) {
const { rows } = await db.query(
`SELECT * FROM password_resets
WHERE token_hash = $1 AND used = false AND expires_at > now()`, [tokenHash]);
return rows[0] || null;
}
async function markResetUsed(id) {
await db.query('UPDATE password_resets SET used = true WHERE id = $1', [id]);
}
// ---------------------------------------------------------------- settings
async function getSetting(key) {
const { rows } = await db.query('SELECT value FROM settings WHERE key = $1', [key]);
return rows.length ? rows[0].value : null;
}
// value of '' or null clears the setting.
async function setSetting(key, value) {
if (value === null || value === '') {
await db.query('DELETE FROM settings WHERE key = $1', [key]);
return;
}
await db.query(
`INSERT INTO settings (key, value, updated_at) VALUES ($1, $2, now())
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()`,
[key, value]);
}
// ---------------------------------------------------------------- designers
async function upsertDesigner(client, { name, source_site, profile_url }) {
const { rows } = await client.query(
`INSERT INTO designers (name, source_site, profile_url) VALUES ($1,$2,$3)
ON CONFLICT (name, source_site) DO UPDATE SET profile_url = EXCLUDED.profile_url
RETURNING id`, [name, source_site || null, profile_url || null]);
return rows[0].id;
}
module.exports = {
getAllModels, getModel, getModelsByIds, findModelBySource, createModel, updateModel, deleteModel, addFiles,
updateFileConversion,
trashModels, restoreModel, trashedModelIds, countTrashed,
duplicateCandidateFiles, matchableCounts, mergeAndTrash, dismissDuplicateGroup, dismissedDuplicateSignatures,
getFile, getArchiveEntries, getArchiveEntry, replaceArchiveEntries,
getAllTags, replaceRuleTags, removeTagsFrom, applyAiResult, topTags,
getAllProjects, getProject, createProject, updateProject, deleteProject,
getSetting, setSetting,
countUsers, getUserByEmail, getUserById, createUser, updateUserPassword,
createPasswordReset, getValidReset, markResetUsed,
getLicenseRule, upsertLicenseRule, listLicenseRules, distinctModelLicenses, applyLicenseRuleToModels,
reconcileModelsToRules,
};