00a4b4ba07
- Fingerprint every file in the background: SHA-256 for all formats plus a trimesh geometry fingerprint for STL/OBJ/3MF, so renamed, re-exported or converted copies are still matched. Backfills existing files on startup. - Duplicates view groups models that share files (exact copies vs same shape), recommends which to keep, merges tags/collections/notes into the kept model and trashes the rest; groups can be dismissed. - Deleting now moves models to a restorable trash; files are only removed from disk on permanent delete or when the trash is emptied. - Per-model notes with autosave. - Search across names, notes, tags, files and designers; filter by format, source and license; include/exclude tags with any/all matching; sort and group by source or format. Preferences persist per browser. - Add node:test unit tests for duplicate grouping and fingerprints. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
86 lines
3.1 KiB
JavaScript
86 lines
3.1 KiB
JavaScript
// Groups models that hold the same files. Two models are linked when they share
|
|
// a file with the same content hash (exact copy) or the same geometry
|
|
// fingerprint (same shape: renamed, re-exported, converted — or mirrored, so
|
|
// those groups are flagged for review). Links are transitive, so a pack
|
|
// downloaded twice collapses into one group.
|
|
|
|
// files: [{ id, model_id, sha256, geometry_hash }]
|
|
// Returns [{ modelIds: [sorted], exact: boolean, matchedFiles: { modelId: count } }]
|
|
function groupDuplicates(files) {
|
|
const parent = new Map();
|
|
const find = (x) => {
|
|
while (parent.get(x) !== x) {
|
|
parent.set(x, parent.get(parent.get(x)));
|
|
x = parent.get(x);
|
|
}
|
|
return x;
|
|
};
|
|
for (const f of files) parent.set(f.model_id, f.model_id);
|
|
|
|
const matched = new Map(); // modelId -> Set(fileId)
|
|
const inexact = []; // a model in each shape-only link
|
|
|
|
const buckets = (key) => {
|
|
const map = new Map();
|
|
for (const f of files) {
|
|
if (!f[key]) continue;
|
|
if (!map.has(f[key])) map.set(f[key], []);
|
|
map.get(f[key]).push(f);
|
|
}
|
|
return map.values();
|
|
};
|
|
|
|
const link = (bucket, exact) => {
|
|
const modelIds = [...new Set(bucket.map((f) => f.model_id))];
|
|
if (modelIds.length < 2) return;
|
|
for (const f of bucket) {
|
|
if (!matched.has(f.model_id)) matched.set(f.model_id, new Set());
|
|
matched.get(f.model_id).add(f.id);
|
|
}
|
|
for (const id of modelIds.slice(1)) parent.set(find(id), find(modelIds[0]));
|
|
if (!exact) inexact.push(modelIds[0]);
|
|
};
|
|
|
|
for (const bucket of buckets('sha256')) link(bucket, true);
|
|
for (const bucket of buckets('geometry_hash')) {
|
|
// Same shape but different bytes somewhere in the bucket = not a plain copy.
|
|
link(bucket, new Set(bucket.map((f) => f.sha256)).size === 1);
|
|
}
|
|
|
|
const groups = new Map();
|
|
for (const id of matched.keys()) {
|
|
const root = find(id);
|
|
if (!groups.has(root)) groups.set(root, []);
|
|
groups.get(root).push(id);
|
|
}
|
|
const inexactRoots = new Set(inexact.map(find));
|
|
|
|
return [...groups.entries()]
|
|
.filter(([, ids]) => ids.length > 1)
|
|
.map(([root, ids]) => ({
|
|
modelIds: ids.sort((a, b) => a - b),
|
|
exact: !inexactRoots.has(root),
|
|
matchedFiles: Object.fromEntries(ids.map((id) => [id, matched.get(id).size])),
|
|
}));
|
|
}
|
|
|
|
// Stable id for a group of models (used to remember "not duplicates").
|
|
function signature(modelIds) {
|
|
return [...new Set(modelIds.map(Number))].sort((a, b) => a - b).join(',');
|
|
}
|
|
|
|
function keepScore(m) {
|
|
return (m.source_url ? 4 : 0) + (m.license ? 2 : 0) + (m.notes ? 2 : 0) + (m.description ? 1 : 0)
|
|
+ Math.min((m.tags || []).length, 3) + Math.min((m.projects || []).length, 3) + (m.files || []).length;
|
|
}
|
|
|
|
// Which model of a group to keep: the most complete one (source, license, notes,
|
|
// tags, collections, most files), then the one added first.
|
|
function recommendKeep(models) {
|
|
return [...models].sort((a, b) =>
|
|
keepScore(b) - keepScore(a)
|
|
|| new Date(a.date_added) - new Date(b.date_added)
|
|
|| a.id - b.id)[0].id;
|
|
}
|
|
|
|
module.exports = { groupDuplicates, signature, recommendKeep };
|