printhive/lib/fingerprint.js

// Background worker that indexes and fingerprints stored files for the
// duplicate finder: a SHA-256 of the bytes, plus a geometry fingerprint for mesh
// formats so the same model still matches after being renamed, re-exported
// (ASCII/binary STL) or converted (STL -> 3MF). Archives are listed, and the
// models inside them are fingerprinted too.

const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const db = require('./db');
const store = require('./store');
const archives = require('./archives');
const thumbnails = require('./thumbnails');
const autotag = require('./autotag');
const embeddings = require('./embeddings');
const { MODEL_FORMATS, ARCHIVE_FORMATS, formatOf } = require('./fileProcessor');
const { PYTHON, runScript } = require('./python');

const GEOMETRY_FORMATS = ['stl', 'obj', '3mf'];
const BATCH = 10;
const TABLES = ['files', 'archive_entries'];

let pythonAvailable = true;
let running = false;
let rerun = false;

function sha256File(filePath) {
  return new Promise((resolve, reject) => {
    const hash = crypto.createHash('sha256');
    fs.createReadStream(filePath)
      .on('error', reject)
      .on('data', (chunk) => hash.update(chunk))
      .on('end', () => resolve(hash.digest('hex')));
  });
}

// n rounded to `digits` significant figures, as a stable string.
function sig(n, digits = 4) {
  if (!Number.isFinite(n) || n === 0) return '0';
  return String(Number(n.toPrecision(digits)));
}

// Hash of rounded shape measurements. Position and axis-aligned rotation don't
// change it, and rounding absorbs float noise between export formats.
function geometryKey(fp) {
  if (!fp || !fp.faces) return null;
  const extents = (fp.extents || []).map((e) => Math.abs(e)).sort((a, b) => a - b).map((e) => e.toFixed(1));
  const raw = `${fp.faces}|${sig(fp.area)}|${sig(Math.abs(fp.volume || 0))}|${extents.join('x')}`;
  return crypto.createHash('sha1').update(raw).digest('hex');
}

// Runs the trimesh sidecar over some files. Resolves Map(absPath -> {fp}|{error});
// output from a run that timed out or crashed part-way is still returned.
async function measure(paths) {
  const { err, stdout } = await runScript('fingerprint.py', paths,
    { timeout: 60000 * (paths.length + 1), isBackground: true });
  if (err && err.code === 'ENOENT') throw err;
  const results = new Map();
  for (const line of stdout.split('\n')) {
    if (!line.trim()) continue;
    try {
      const r = JSON.parse(line);
      results.set(r.path, r);
    } catch { /* ignore non-JSON noise */ }
  }
  return results;
}

// rows: [{ id, format, abs }] from `table` ('files' or 'archive_entries').
async function fingerprintGeometry(rows, table) {
  if (!TABLES.includes(table)) throw new Error(`Unknown table ${table}`);
  // 3MF projects can be big, so each gets its own run; that keeps a queued
  // preview from waiting behind a whole batch of them.
  const chunks = [rows.filter((r) => r.format !== '3mf'), ...rows.filter((r) => r.format === '3mf').map((r) => [r])];
  for (const chunk of chunks.filter((c) => c.length)) {
    const results = await measure(chunk.map((r) => r.abs));
    for (const row of chunk) {
      let r = results.get(row.abs);
      // A batch that died part-way: retry the missing file on its own.
      if (!r && chunk.length > 1) r = (await measure([row.abs])).get(row.abs);
      const key = r && r.fp ? geometryKey(r.fp) : null;
      await db.query(`UPDATE ${table} SET geometry_hash = $2, geometry_status = $3 WHERE id = $1`,
        [row.id, key, key ? 'ok' : 'error']);
    }
  }
}

// Runs fn; if Python turns out to be missing, geometry fingerprints are switched
// off for this process (content hashes still run).
async function withPython(fn) {
  try {
    await fn();
  } catch (err) {
    if (err.code !== 'ENOENT') throw err;
    pythonAvailable = false;
    console.warn(`Geometry fingerprints disabled: no Python at ${PYTHON} (content hashes still run).`);
  }
}

async function fingerprintFiles() {
  const { rows } = await db.query(
    `SELECT id, file_path, format, fingerprinted_at FROM files
     WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean)
     ORDER BY id LIMIT $2`, [pythonAvailable, BATCH]);
  if (rows.length === 0) return false;

  const geometry = [];
  for (const row of rows) {
    const abs = path.resolve(row.file_path);
    try {
      if (!fs.existsSync(abs)) {
        await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'missing' WHERE id = $1`, [row.id]);
        continue;
      }
      if (!row.fingerprinted_at) {
        await db.query('UPDATE files SET sha256 = $2, fingerprinted_at = now() WHERE id = $1',
          [row.id, await sha256File(abs)]);
      }
      if (!GEOMETRY_FORMATS.includes(row.format)) {
        await db.query(`UPDATE files SET geometry_status = 'na' WHERE id = $1`, [row.id]);
      } else if (pythonAvailable) {
        geometry.push({ id: row.id, format: row.format, abs });
      }
    } catch (err) {
      console.warn(`Fingerprint failed for file ${row.id}:`, err.message);
      await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [row.id]);
    }
  }
  if (geometry.length) await withPython(() => fingerprintGeometry(geometry, 'files'));
  return true;
}

async function indexArchives() {
  const { rows } = await db.query(
    `SELECT id, file_path, format FROM files
     WHERE format = ANY($1) AND archive_status IS NULL ORDER BY id LIMIT 5`, [ARCHIVE_FORMATS]);
  for (const file of rows) {
    try {
      const entries = await archives.listEntries(file);
      await store.replaceArchiveEntries(file.id,
        entries.map((e) => ({ path: e.path, size: e.size, format: formatOf(e.path) })));
      await db.query(`UPDATE files SET archive_status = 'ok', archive_error = NULL WHERE id = $1`, [file.id]);
    } catch (err) {
      console.warn(`Could not read archive (file ${file.id}):`, err.message);
      await db.query(`UPDATE files SET archive_status = 'error', archive_error = $2 WHERE id = $1`,
        [file.id, err.message.slice(0, 300)]);
    }
  }
  return rows.length > 0;
}

const ENTRY_PENDING = 'e.fingerprinted_at IS NULL OR (e.geometry_status IS NULL AND $1::boolean)';

// Fingerprints the pending entries of one archive, extracting its models in a
// single pass (cheaper than one extraction per entry, especially for solid 7z).
async function fingerprintEntries() {
  const { rows: [file] } = await db.query(
    `SELECT f.id, f.file_path, f.format FROM files f
     WHERE EXISTS (SELECT 1 FROM archive_entries e WHERE e.file_id = f.id AND (${ENTRY_PENDING}))
     ORDER BY f.id LIMIT 1`, [pythonAvailable]);
  if (!file) return false;
  const { rows: entries } = await db.query(
    `SELECT e.id, e.path, e.format, e.fingerprinted_at FROM archive_entries e
     WHERE e.file_id = $2 AND (${ENTRY_PENDING}) ORDER BY e.id LIMIT 500`, [pythonAvailable, file.id]);

  // Pictures, documents and nested archives are listed but not matched as duplicates.
  const others = entries.filter((e) => !MODEL_FORMATS.includes(e.format));
  if (others.length) {
    await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'na' WHERE id = ANY($1::int[])`,
      [others.map((e) => e.id)]);
  }
  const models = entries.filter((e) => MODEL_FORMATS.includes(e.format));
  if (!models.length) return true;

  const tmp = archives.tempDir();
  try {
    let extracted = new Map();
    try {
      extracted = await archives.extractMany(file, models.map((e) => e.path), tmp);
    } catch (err) {
      console.warn(`Could not extract from archive (file ${file.id}):`, err.message);
    }
    const geometry = [];
    for (const entry of models) {
      const abs = extracted.get(entry.path);
      try {
        if (!abs) {
          await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [entry.id]);
          continue;
        }
        if (!entry.fingerprinted_at) {
          await db.query('UPDATE archive_entries SET sha256 = $2, fingerprinted_at = now() WHERE id = $1',
            [entry.id, await sha256File(abs)]);
        }
        if (!GEOMETRY_FORMATS.includes(entry.format)) {
          await db.query(`UPDATE archive_entries SET geometry_status = 'na' WHERE id = $1`, [entry.id]);
        } else if (pythonAvailable) {
          geometry.push({ id: entry.id, format: entry.format, abs });
        }
      } catch (err) {
        console.warn(`Fingerprint failed for archive entry ${entry.id}:`, err.message);
        await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [entry.id]);
      }
    }
    if (geometry.length) await withPython(() => fingerprintGeometry(geometry, 'archive_entries'));
  } finally {
    fs.rmSync(tmp, { recursive: true, force: true });
  }
  return true;
}

async function run() {
  for (;;) {
    // Thumbnails, tags and search come after indexing so they can use what's inside
    // newly indexed archives.
    const worked = [await fingerprintFiles(), await indexArchives(), await fingerprintEntries(),
      await thumbnails.generate(), await autotag.run(), await embeddings.run()];
    if (!worked.some(Boolean)) return;
  }
}

// Start (or queue another pass of) the worker. Safe to call often.
function kick() {
  if (running) { rerun = true; return; }
  running = true;
  run()
    .catch((err) => console.error('Fingerprinting stopped:', err.message))
    .finally(() => {
      running = false;
      if (rerun) { rerun = false; kick(); }
    });
}

async function pendingCount() {
  const { rows } = await db.query(
    `SELECT (SELECT COUNT(*) FROM files
             WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean))
          + (SELECT COUNT(*) FROM files WHERE format = ANY($2) AND archive_status IS NULL)
          + (SELECT COUNT(*) FROM archive_entries e WHERE ${ENTRY_PENDING}) AS n`,
    [pythonAvailable, ARCHIVE_FORMATS]);
  return Number(rows[0].n);
}

module.exports = { kick, pendingCount, geometryKey, sha256File };