// Groups models that hold the same files. Two models are linked when they share // a file with the same content hash (exact copy) or the same geometry // fingerprint (same shape: renamed, re-exported, converted — or mirrored, so // those groups are flagged for review). Links are transitive, so a pack // downloaded twice collapses into one group. // files: [{ id, model_id, sha256, geometry_hash }] // Returns [{ modelIds: [sorted], exact: boolean, matchedFiles: { modelId: count } }] function groupDuplicates(files) { const parent = new Map(); const find = (x) => { while (parent.get(x) !== x) { parent.set(x, parent.get(parent.get(x))); x = parent.get(x); } return x; }; for (const f of files) parent.set(f.model_id, f.model_id); const matched = new Map(); // modelId -> Set(fileId) const inexact = []; // a model in each shape-only link const buckets = (key) => { const map = new Map(); for (const f of files) { if (!f[key]) continue; if (!map.has(f[key])) map.set(f[key], []); map.get(f[key]).push(f); } return map.values(); }; const link = (bucket, exact) => { const modelIds = [...new Set(bucket.map((f) => f.model_id))]; if (modelIds.length < 2) return; for (const f of bucket) { if (!matched.has(f.model_id)) matched.set(f.model_id, new Set()); matched.get(f.model_id).add(f.id); } for (const id of modelIds.slice(1)) parent.set(find(id), find(modelIds[0])); if (!exact) inexact.push(modelIds[0]); }; for (const bucket of buckets('sha256')) link(bucket, true); for (const bucket of buckets('geometry_hash')) { // Same shape but different bytes somewhere in the bucket = not a plain copy. link(bucket, new Set(bucket.map((f) => f.sha256)).size === 1); } const groups = new Map(); for (const id of matched.keys()) { const root = find(id); if (!groups.has(root)) groups.set(root, []); groups.get(root).push(id); } const inexactRoots = new Set(inexact.map(find)); return [...groups.entries()] .filter(([, ids]) => ids.length > 1) .map(([root, ids]) => ({ modelIds: ids.sort((a, b) => a - b), exact: !inexactRoots.has(root), matchedFiles: Object.fromEntries(ids.map((id) => [id, matched.get(id).size])), })); } // Stable id for a group of models (used to remember "not duplicates"). function signature(modelIds) { return [...new Set(modelIds.map(Number))].sort((a, b) => a - b).join(','); } function keepScore(m) { return (m.source_url ? 4 : 0) + (m.license ? 2 : 0) + (m.notes ? 2 : 0) + (m.description ? 1 : 0) + Math.min((m.tags || []).length, 3) + Math.min((m.projects || []).length, 3) + (m.files || []).length; } // Which model of a group to keep: the most complete one (source, license, notes, // tags, collections, most files), then the one added first. function recommendKeep(models) { return [...models].sort((a, b) => keepScore(b) - keepScore(a) || new Date(a.date_added) - new Date(b.date_added) || a.id - b.id)[0].id; } module.exports = { groupDuplicates, signature, recommendKeep };