0f49fe8c1a
- AI tagging through any OpenAI-compatible API: DeepSeek by default, or OpenRouter, OpenAI, or a local server (Ollama, LM Studio). Settings → AI with presets, prices and "Save & test connection"; the key never leaves the server except to the provider. - Runs over untagged models, all models, or one model from its page: live progress, tokens and cost, stop/resume (retrying failures), an optional spending limit, and paused rather than resumed after a restart. Adapts to models without JSON mode or image input; stops on a rejected key or no credit. Tags are only added; descriptions filled only where empty unless asked. - Search by meaning: local bge-small-en-v1.5 embeddings (transformers.js), indexed in the background whenever a model's text changes; Words/Meaning switch on the search box. Stored as REAL[] in Postgres, no pgvector needed. - Automatic tags: strict whole-word rules (part types, mechanisms, materials, techniques) over names, file names and archive contents, plus your own word -> tag rules; on/off (optionally removing its tags) and re-apply. - Tags remember where they came from (user / rule / ai); edits keep that, rule and AI passes never touch your own tags, and the model page marks them. - Docker: ships the embedding model and only Linux onnxruntime binaries. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
46 lines
1.6 KiB
JavaScript
46 lines
1.6 KiB
JavaScript
// The local text-embedding model behind semantic search: bge-small-en-v1.5
|
|
// (quantized, ~34 MB) run with transformers.js. Nothing is sent anywhere; the model
|
|
// is downloaded once into MODELS_DIR (the Docker image ships with it).
|
|
const path = require('path');
|
|
|
|
const MODEL = 'Xenova/bge-small-en-v1.5';
|
|
const DIMENSIONS = 384;
|
|
const CACHE_DIR = process.env.MODELS_DIR || path.join(__dirname, '..', 'models');
|
|
// bge models match short queries against passages best with this instruction.
|
|
const QUERY_PREFIX = 'Represent this sentence for searching relevant passages: ';
|
|
const IDLE_MS = 10 * 60 * 1000;
|
|
|
|
let pipelinePromise = null;
|
|
let idleTimer = null;
|
|
|
|
// Loaded on first use and released after a while unused (it holds ~150 MB).
|
|
function load() {
|
|
if (!pipelinePromise) {
|
|
pipelinePromise = (async () => {
|
|
const { pipeline, env } = await import('@huggingface/transformers');
|
|
env.cacheDir = CACHE_DIR;
|
|
return pipeline('feature-extraction', MODEL, { dtype: 'q8' });
|
|
})();
|
|
pipelinePromise.catch(() => { pipelinePromise = null; });
|
|
}
|
|
clearTimeout(idleTimer);
|
|
idleTimer = setTimeout(release, IDLE_MS);
|
|
idleTimer.unref();
|
|
return pipelinePromise;
|
|
}
|
|
|
|
async function release() {
|
|
const loaded = pipelinePromise;
|
|
pipelinePromise = null;
|
|
try { await (await loaded).dispose(); } catch { /* already gone */ }
|
|
}
|
|
|
|
// texts -> unit-length vectors (arrays of numbers).
|
|
async function embed(texts) {
|
|
if (!texts.length) return [];
|
|
const extractor = await load();
|
|
const output = await extractor(texts, { pooling: 'cls', normalize: true });
|
|
return output.tolist();
|
|
}
|
|
|
|
module.exports = { MODEL, DIMENSIONS, QUERY_PREFIX, load, embed };
|