// Background worker that fingerprints stored files for the duplicate finder: a // SHA-256 of the bytes for every file, plus a geometry fingerprint for mesh // formats so the same model still matches after being renamed, re-exported // (ASCII/binary STL) or converted (STL -> 3MF). const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const db = require('./db'); const { PYTHON, runScript } = require('./python'); const GEOMETRY_FORMATS = ['stl', 'obj', '3mf']; const BATCH = 10; let pythonAvailable = true; let running = false; let rerun = false; function sha256File(filePath) { return new Promise((resolve, reject) => { const hash = crypto.createHash('sha256'); fs.createReadStream(filePath) .on('error', reject) .on('data', (chunk) => hash.update(chunk)) .on('end', () => resolve(hash.digest('hex'))); }); } // n rounded to `digits` significant figures, as a stable string. function sig(n, digits = 4) { if (!Number.isFinite(n) || n === 0) return '0'; return String(Number(n.toPrecision(digits))); } // Hash of rounded shape measurements. Position and axis-aligned rotation don't // change it, and rounding absorbs float noise between export formats. function geometryKey(fp) { if (!fp || !fp.faces) return null; const extents = (fp.extents || []).map((e) => Math.abs(e)).sort((a, b) => a - b).map((e) => e.toFixed(1)); const raw = `${fp.faces}|${sig(fp.area)}|${sig(Math.abs(fp.volume || 0))}|${extents.join('x')}`; return crypto.createHash('sha1').update(raw).digest('hex'); } // Runs the trimesh sidecar over some files. Resolves Map(absPath -> {fp}|{error}); // output from a run that timed out or crashed part-way is still returned. async function measure(paths) { const { err, stdout } = await runScript('fingerprint.py', paths, { timeout: 60000 * (paths.length + 1), isBackground: true }); if (err && err.code === 'ENOENT') throw err; const results = new Map(); for (const line of stdout.split('\n')) { if (!line.trim()) continue; try { const r = JSON.parse(line); results.set(r.path, r); } catch { /* ignore non-JSON noise */ } } return results; } async function fingerprintGeometry(rows) { const abs = (row) => path.resolve(row.file_path); // 3MF projects can be big, so each gets its own run; that keeps a queued // preview from waiting behind a whole batch of them. const chunks = [rows.filter((r) => r.format !== '3mf'), ...rows.filter((r) => r.format === '3mf').map((r) => [r])]; for (const chunk of chunks.filter((c) => c.length)) { const results = await measure(chunk.map(abs)); for (const row of chunk) { let r = results.get(abs(row)); // A batch that died part-way: retry the missing file on its own. if (!r && chunk.length > 1) r = (await measure([abs(row)])).get(abs(row)); const key = r && r.fp ? geometryKey(r.fp) : null; await db.query('UPDATE files SET geometry_hash = $2, geometry_status = $3 WHERE id = $1', [row.id, key, key ? 'ok' : 'error']); } } } async function run() { for (;;) { const { rows } = await db.query( `SELECT id, file_path, format, fingerprinted_at FROM files WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean) ORDER BY id LIMIT $2`, [pythonAvailable, BATCH]); if (rows.length === 0) return; const geometry = []; for (const row of rows) { const abs = path.resolve(row.file_path); try { if (!fs.existsSync(abs)) { await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'missing' WHERE id = $1`, [row.id]); continue; } if (!row.fingerprinted_at) { await db.query('UPDATE files SET sha256 = $2, fingerprinted_at = now() WHERE id = $1', [row.id, await sha256File(abs)]); } if (!GEOMETRY_FORMATS.includes(row.format)) { await db.query(`UPDATE files SET geometry_status = 'na' WHERE id = $1`, [row.id]); } else if (pythonAvailable) { geometry.push(row); } } catch (err) { console.warn(`Fingerprint failed for file ${row.id}:`, err.message); await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [row.id]); } } if (geometry.length) { try { await fingerprintGeometry(geometry); } catch (err) { if (err.code !== 'ENOENT') throw err; pythonAvailable = false; console.warn(`Geometry fingerprints disabled: no Python at ${PYTHON} (content hashes still run).`); } } } } // Start (or queue another pass of) the worker. Safe to call often. function kick() { if (running) { rerun = true; return; } running = true; run() .catch((err) => console.error('Fingerprinting stopped:', err.message)) .finally(() => { running = false; if (rerun) { rerun = false; kick(); } }); } async function pendingCount() { const { rows } = await db.query( `SELECT COUNT(*)::int AS n FROM files WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean)`, [pythonAvailable]); return rows[0].n; } module.exports = { kick, pendingCount, geometryKey, sha256File };