Browse archives, preview companion files, and view FBX/STEP/3DM/OBJ+MTL

Archives
- ZIP, 7Z, RAR and TAR/.tar.gz uploads are indexed in the background into
  archive_entries (7-Zip compiled to WASM in a worker thread; node-tar for
  tar). macOS/Windows clutter and paths that could escape the extraction
  folder are skipped.
- Models inside archives are extracted once per archive to hash and
  geometry-fingerprint them, so duplicates are found inside packs too.
  Pictures, PDFs and readmes are listed but never matched.
- New routes list an archive's entries and serve/convert a single entry,
  extracted on first use and cached under extracted/.

Viewer
- Model detail lists what's inside each archive with view/download links;
  opening an entry adds a viewer tab.
- New 3D formats: FBX (with sibling textures), OBJ with its MTL and
  textures, STEP/STP via occt-import-js, and Rhino 3DM via rhino3dm — the
  WASM readers are served from node_modules, loaded on first use.
- Pictures, PDFs and text files (jpg/png/gif/webp/bmp/svg, pdf, txt/md)
  can be uploaded and preview in the viewer; a model without an embedded
  3MF preview uses its first picture as the card thumbnail.
- Inline file responses send nosniff, and SVGs are sandboxed.
- GLB conversions write to a temp file and rename, so a half-written
  preview is never served; a later tab can't be overdrawn by a slow load.
- Better 3MF card image: the project thumbnail or plate 1 render, not a
  slicer pick map.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dlawler489 2026-09-15 07:36:47 +10:00
parent 502eefdec2
commit d4b7e29348
19 changed files with 1045 additions and 157 deletions

View file

@ -14,3 +14,4 @@ npm-debug.log*
Dockerfile Dockerfile
.dockerignore .dockerignore
docker-compose.yml docker-compose.yml
extracted

1
.gitignore vendored
View file

@ -9,3 +9,4 @@ thumbnails/
glb/ glb/
tools/venv/ tools/venv/
__pycache__/ __pycache__/
extracted/

View file

@ -124,6 +124,27 @@ ALTER TABLE models ADD COLUMN IF NOT EXISTS notes TEXT NOT NULL DEFAULT '';
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ; ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
CREATE INDEX IF NOT EXISTS idx_models_deleted ON models(deleted_at); CREATE INDEX IF NOT EXISTS idx_models_deleted ON models(deleted_at);
-- Archive files (zip/7z/rar/tar) are indexed in the background:
-- null = not yet, ok | error (with archive_error).
ALTER TABLE files ADD COLUMN IF NOT EXISTS archive_status TEXT;
ALTER TABLE files ADD COLUMN IF NOT EXISTS archive_error TEXT;
-- Files inside archives, so they can be browsed, previewed and matched as duplicates.
CREATE TABLE IF NOT EXISTS archive_entries (
id SERIAL PRIMARY KEY,
file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
path TEXT NOT NULL, -- path inside the archive, as the archive stores it
format TEXT NOT NULL,
size BIGINT NOT NULL DEFAULT 0,
sha256 TEXT,
geometry_hash TEXT,
geometry_status TEXT, -- same states as files.geometry_status
fingerprinted_at TIMESTAMPTZ,
UNIQUE (file_id, path)
);
CREATE INDEX IF NOT EXISTS idx_entries_sha256 ON archive_entries(sha256);
CREATE INDEX IF NOT EXISTS idx_entries_geometry ON archive_entries(geometry_hash);
-- Duplicate groups marked "not duplicates" (signature = sorted model ids). -- Duplicate groups marked "not duplicates" (signature = sorted model ids).
CREATE TABLE IF NOT EXISTS duplicate_dismissals ( CREATE TABLE IF NOT EXISTS duplicate_dismissals (
signature TEXT PRIMARY KEY, signature TEXT PRIMARY KEY,

82
lib/archiveWorker.js Normal file
View file

@ -0,0 +1,82 @@
// Worker thread running 7-Zip (compiled to WASM) for ZIP/7Z/RAR archives.
// 7-Zip calls are synchronous, so they run here to keep the server responsive.
const { parentPort } = require('worker_threads');
const SevenZip = require('7z-wasm');
let sevenZip = null;
const output = [];
// The host filesystem is mounted at /host inside the WASM filesystem.
const host = (p) => `/host${p}`;
async function instance() {
if (!sevenZip) {
sevenZip = await SevenZip({ print: (l) => output.push(l), printErr: (l) => output.push(l) });
sevenZip.FS.mkdir('/host');
sevenZip.FS.mount(sevenZip.NODEFS, { root: '/' }, '/host');
}
return sevenZip;
}
async function run(args) {
const sz = await instance();
output.length = 0;
let code;
try {
code = sz.callMain(args);
} catch (err) {
code = err && typeof err.status === 'number' ? err.status : 2;
}
return { code, lines: output.slice() };
}
function errorText(lines) {
const line = lines.find((l) => /error|wrong password|can ?not open|unsupported|headers/i.test(l));
return line ? line.trim() : null;
}
// Parses `7zz l -slt` into [{ path, size }] for files (folders are dropped).
function parseListing(lines) {
const start = lines.findIndex((l) => l.startsWith('----------'));
if (start === -1) return null;
const entries = [];
let cur = {};
const flush = () => {
const isFolder = cur.Folder === '+' || (cur.Attributes || '').startsWith('D');
if (cur.Path !== undefined && !isFolder) entries.push({ path: cur.Path, size: Number(cur.Size) || 0 });
cur = {};
};
for (const line of lines.slice(start + 1)) {
if (!line.trim()) { flush(); continue; }
const i = line.indexOf(' = ');
if (i > 0) cur[line.slice(0, i)] = line.slice(i + 3);
}
flush();
return entries;
}
async function list(archive) {
// A dummy password makes encrypted archives fail fast instead of prompting.
const { code, lines } = await run(['l', '-slt', '-pprinthive', host(archive)]);
const entries = parseListing(lines);
if (code !== 0 || !entries) throw new Error(errorText(lines) || `7-Zip could not read the archive (exit ${code})`);
return entries;
}
// Extracts the named entries (keeping their folders) under destDir. Callers
// check which files actually arrived.
async function extract(archive, entries, destDir) {
const { code, lines } = await run(
['x', host(archive), `-o${host(destDir)}`, '-y', '-spd', '-bsp0', '-pprinthive', '--', ...entries]);
return { code, error: code === 0 ? null : errorText(lines) };
}
parentPort.on('message', async ({ id, op, archive, entries, destDir }) => {
try {
const result = op === 'list' ? await list(archive) : await extract(archive, entries, destDir);
parentPort.postMessage({ id, result });
} catch (err) {
parentPort.postMessage({ id, error: err.message });
}
});

133
lib/archives.js Normal file
View file

@ -0,0 +1,133 @@
// Lists and extracts files inside archives. ZIP/7Z/RAR go through 7-Zip (WASM,
// in a worker thread); TAR and .tar.gz go through node-tar. Extracted entries
// are cached under EXTRACT_DIR/<file id>/ and can always be re-extracted.
const fs = require('fs');
const path = require('path');
const { Worker } = require('worker_threads');
const tar = require('tar');
const EXTRACT_DIR = path.resolve(process.env.EXTRACT_DIR || './extracted');
const TAR_FILE_TYPES = ['File', 'OldFile', 'ContiguousFile'];
let worker = null;
let nextId = 1;
const pending = new Map();
function failPending(err) {
for (const p of pending.values()) p.reject(err);
pending.clear();
worker = null;
}
// The worker keeps the process alive only while a request is outstanding, so
// an idle worker never stops Node from exiting.
function sevenZip(message) {
if (!worker) {
worker = new Worker(path.join(__dirname, 'archiveWorker.js'));
worker.on('message', ({ id, result, error }) => {
const p = pending.get(id);
pending.delete(id);
if (pending.size === 0 && worker) worker.unref();
if (p) error ? p.reject(new Error(error)) : p.resolve(result);
});
worker.on('error', failPending);
worker.on('exit', () => failPending(new Error('Archive worker stopped')));
}
return new Promise((resolve, reject) => {
const id = nextId++;
pending.set(id, { resolve, reject });
worker.ref();
worker.postMessage({ id, ...message });
});
}
const isTar = (format) => format === 'tar' || format === 'tgz';
const clean = (p) => p.replace(/^\.\//, '');
// Clutter archivers add (macOS resource forks, Finder/Explorer files).
function isJunk(entryPath) {
const parts = clean(entryPath).split('/');
const name = parts[parts.length - 1];
return parts.includes('__MACOSX') || name.startsWith('._') || name === '.DS_Store' || name === 'Thumbs.db';
}
// Paths that could land outside the extraction folder are never used.
function isSafe(entryPath) {
return Boolean(entryPath) && !path.isAbsolute(entryPath) && !entryPath.split(/[\\/]/).includes('..');
}
// file: { file_path, format }. Returns [{ path, size }] of real files.
async function listEntries(file) {
const abs = path.resolve(file.file_path);
let raw;
if (isTar(file.format)) {
raw = [];
await tar.t({
file: abs,
onReadEntry: (e) => { if (TAR_FILE_TYPES.includes(e.type)) raw.push({ path: e.path, size: e.size }); },
});
} else {
raw = await sevenZip({ op: 'list', archive: abs });
}
return raw.filter((e) => isSafe(e.path) && !isJunk(e.path));
}
function tempDir() {
fs.mkdirSync(EXTRACT_DIR, { recursive: true });
return fs.mkdtempSync(path.join(EXTRACT_DIR, '.tmp-'));
}
// Extracts entries into destDir. Returns Map(entry path -> absolute path) for
// the ones that came out; archive-level failures throw.
async function extractMany(file, entryPaths, destDir) {
const abs = path.resolve(file.file_path);
const wanted = new Set(entryPaths.filter(isSafe));
if (isTar(file.format)) {
await tar.x({ file: abs, cwd: destDir, filter: (p) => wanted.has(p) });
} else {
const { code, error } = await sevenZip({ op: 'extract', archive: abs, entries: [...wanted], destDir });
if (code !== 0 && error) console.warn(`7-Zip reported a problem extracting from ${abs}: ${error}`);
}
const out = new Map();
for (const p of wanted) {
const target = path.resolve(destDir, p);
if (target.startsWith(destDir + path.sep) && fs.existsSync(target)) out.set(p, target);
}
return out;
}
function entryCachePath(fileId, entry) {
return path.join(EXTRACT_DIR, String(fileId), `${entry.id}${path.extname(entry.path).toLowerCase()}`);
}
const inflight = new Map();
// file: { id, file_path, format }; entry: { id, path }. Returns the absolute
// path of a cached copy, extracting it on first use.
async function extractEntry(file, entry) {
const dest = entryCachePath(file.id, entry);
if (fs.existsSync(dest)) return dest;
if (!inflight.has(dest)) {
const job = (async () => {
fs.mkdirSync(path.dirname(dest), { recursive: true });
const tmp = tempDir();
try {
const got = (await extractMany(file, [entry.path], tmp)).get(entry.path);
if (!got) throw new Error(`Could not extract ${clean(entry.path)} from the archive`);
fs.renameSync(got, dest);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
})().finally(() => inflight.delete(dest));
inflight.set(dest, job);
}
await inflight.get(dest);
return dest;
}
function removeExtracted(fileId) {
fs.rmSync(path.join(EXTRACT_DIR, String(fileId)), { recursive: true, force: true });
}
module.exports = { listEntries, extractMany, extractEntry, removeExtracted, tempDir, isJunk };

View file

@ -5,18 +5,25 @@ const AdmZip = require('adm-zip');
const THUMBNAIL_DIR = './thumbnails'; const THUMBNAIL_DIR = './thumbnails';
const MODEL_FORMATS = ['stl', '3mf', 'obj', 'step', 'stp', 'gcode', 'gco', 'zip']; const MODEL_FORMATS = ['stl', '3mf', 'obj', 'fbx', 'step', 'stp', '3dm', 'gcode', 'gco'];
const ARCHIVE_FORMATS = ['zip', '7z', 'rar', 'tar', 'tgz'];
const IMAGE_FORMATS = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'svg'];
// Kept with a model and viewable, but never matched as duplicates.
const COMPANION_FORMATS = [...IMAGE_FORMATS, 'pdf', 'txt', 'md', 'mtl'];
const ALLOWED_FORMATS = [...MODEL_FORMATS, ...ARCHIVE_FORMATS, ...COMPANION_FORMATS];
function ensureThumbnailDir() { function ensureThumbnailDir() {
if (!fs.existsSync(THUMBNAIL_DIR)) fs.mkdirSync(THUMBNAIL_DIR, { recursive: true }); if (!fs.existsSync(THUMBNAIL_DIR)) fs.mkdirSync(THUMBNAIL_DIR, { recursive: true });
} }
function formatOf(filename) { function formatOf(filename) {
return path.extname(filename).slice(1).toLowerCase(); const lower = filename.toLowerCase();
if (lower.endsWith('.tar.gz')) return 'tgz';
return path.extname(lower).slice(1);
} }
function isAllowed(filename) { function isAllowed(filename) {
return MODEL_FORMATS.includes(formatOf(filename)); return ALLOWED_FORMATS.includes(formatOf(filename));
} }
// Geometry metadata. Only STL is computed natively today; others return nulls // Geometry metadata. Only STL is computed natively today; others return nulls
@ -37,16 +44,28 @@ function extractMetadata(filePath, format) {
} }
} }
// Which embedded PNG makes the best card image: the project thumbnail, then the
// first plate render. Slicer pick maps, top views and small/unlit variants last.
function thumbnailRank(name) {
if (name === 'metadata/thumbnail.png') return 0;
if (/metadata\/plate_1\.png$/.test(name)) return 1;
if (/metadata\/plate_\d+\.png$/.test(name)) return 2;
if (/(pick_|top_|no_light|_small)/.test(name)) return 9;
return 5;
}
// A .3mf is a zip; slicers embed a PNG preview under /Metadata/. If present, // A .3mf is a zip; slicers embed a PNG preview under /Metadata/. If present,
// pull it out as the model thumbnail. Returns a path under thumbnails/ or null. // pull it out as the model thumbnail. Returns a path under thumbnails/ or null.
function extract3mfThumbnail(filePath, baseName) { function extract3mfThumbnail(filePath, baseName) {
try { try {
ensureThumbnailDir(); ensureThumbnailDir();
const zip = new AdmZip(filePath); const zip = new AdmZip(filePath);
const entry = zip.getEntries().find((e) => { const entry = zip.getEntries()
const n = e.entryName.toLowerCase(); .filter((e) => {
return n.endsWith('.png') && (n.includes('thumbnail') || n.includes('/metadata/')); const n = e.entryName.toLowerCase();
}); return n.endsWith('.png') && (n.includes('thumbnail') || n.includes('metadata/'));
})
.sort((a, b) => thumbnailRank(a.entryName.toLowerCase()) - thumbnailRank(b.entryName.toLowerCase()))[0];
if (!entry) return null; if (!entry) return null;
const out = path.join(THUMBNAIL_DIR, `${baseName}.png`); const out = path.join(THUMBNAIL_DIR, `${baseName}.png`);
fs.writeFileSync(out, entry.getData()); fs.writeFileSync(out, entry.getData());
@ -57,4 +76,21 @@ function extract3mfThumbnail(filePath, baseName) {
} }
} }
module.exports = { MODEL_FORMATS, formatOf, isAllowed, extractMetadata, extract3mfThumbnail }; // Copy a picture that came with a model into thumbnails/ (a copy, so replacing
// the thumbnail later never deletes the model's own file).
function copyAsThumbnail(filePath, baseName) {
try {
ensureThumbnailDir();
const out = path.join(THUMBNAIL_DIR, `${baseName}-image${path.extname(filePath).toLowerCase()}`);
fs.copyFileSync(filePath, out);
return out;
} catch (err) {
console.error('Image thumbnail copy failed:', err.message);
return null;
}
}
module.exports = {
MODEL_FORMATS, ARCHIVE_FORMATS, IMAGE_FORMATS, COMPANION_FORMATS, ALLOWED_FORMATS,
formatOf, isAllowed, extractMetadata, extract3mfThumbnail, copyAsThumbnail,
};

View file

@ -1,16 +1,21 @@
// Background worker that fingerprints stored files for the duplicate finder: a // Background worker that indexes and fingerprints stored files for the
// SHA-256 of the bytes for every file, plus a geometry fingerprint for mesh // duplicate finder: a SHA-256 of the bytes, plus a geometry fingerprint for mesh
// formats so the same model still matches after being renamed, re-exported // formats so the same model still matches after being renamed, re-exported
// (ASCII/binary STL) or converted (STL -> 3MF). // (ASCII/binary STL) or converted (STL -> 3MF). Archives are listed, and the
// models inside them are fingerprinted too.
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
const crypto = require('crypto'); const crypto = require('crypto');
const db = require('./db'); const db = require('./db');
const store = require('./store');
const archives = require('./archives');
const { MODEL_FORMATS, ARCHIVE_FORMATS, formatOf } = require('./fileProcessor');
const { PYTHON, runScript } = require('./python'); const { PYTHON, runScript } = require('./python');
const GEOMETRY_FORMATS = ['stl', 'obj', '3mf']; const GEOMETRY_FORMATS = ['stl', 'obj', '3mf'];
const BATCH = 10; const BATCH = 10;
const TABLES = ['files', 'archive_entries'];
let pythonAvailable = true; let pythonAvailable = true;
let running = false; let running = false;
@ -58,64 +63,153 @@ async function measure(paths) {
return results; return results;
} }
async function fingerprintGeometry(rows) { // rows: [{ id, format, abs }] from `table` ('files' or 'archive_entries').
const abs = (row) => path.resolve(row.file_path); async function fingerprintGeometry(rows, table) {
if (!TABLES.includes(table)) throw new Error(`Unknown table ${table}`);
// 3MF projects can be big, so each gets its own run; that keeps a queued // 3MF projects can be big, so each gets its own run; that keeps a queued
// preview from waiting behind a whole batch of them. // preview from waiting behind a whole batch of them.
const chunks = [rows.filter((r) => r.format !== '3mf'), ...rows.filter((r) => r.format === '3mf').map((r) => [r])]; const chunks = [rows.filter((r) => r.format !== '3mf'), ...rows.filter((r) => r.format === '3mf').map((r) => [r])];
for (const chunk of chunks.filter((c) => c.length)) { for (const chunk of chunks.filter((c) => c.length)) {
const results = await measure(chunk.map(abs)); const results = await measure(chunk.map((r) => r.abs));
for (const row of chunk) { for (const row of chunk) {
let r = results.get(abs(row)); let r = results.get(row.abs);
// A batch that died part-way: retry the missing file on its own. // A batch that died part-way: retry the missing file on its own.
if (!r && chunk.length > 1) r = (await measure([abs(row)])).get(abs(row)); if (!r && chunk.length > 1) r = (await measure([row.abs])).get(row.abs);
const key = r && r.fp ? geometryKey(r.fp) : null; const key = r && r.fp ? geometryKey(r.fp) : null;
await db.query('UPDATE files SET geometry_hash = $2, geometry_status = $3 WHERE id = $1', await db.query(`UPDATE ${table} SET geometry_hash = $2, geometry_status = $3 WHERE id = $1`,
[row.id, key, key ? 'ok' : 'error']); [row.id, key, key ? 'ok' : 'error']);
} }
} }
} }
async function run() { // Runs fn; if Python turns out to be missing, geometry fingerprints are switched
for (;;) { // off for this process (content hashes still run).
const { rows } = await db.query( async function withPython(fn) {
`SELECT id, file_path, format, fingerprinted_at FROM files try {
WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean) await fn();
ORDER BY id LIMIT $2`, [pythonAvailable, BATCH]); } catch (err) {
if (rows.length === 0) return; if (err.code !== 'ENOENT') throw err;
pythonAvailable = false;
console.warn(`Geometry fingerprints disabled: no Python at ${PYTHON} (content hashes still run).`);
}
}
async function fingerprintFiles() {
const { rows } = await db.query(
`SELECT id, file_path, format, fingerprinted_at FROM files
WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean)
ORDER BY id LIMIT $2`, [pythonAvailable, BATCH]);
if (rows.length === 0) return false;
const geometry = [];
for (const row of rows) {
const abs = path.resolve(row.file_path);
try {
if (!fs.existsSync(abs)) {
await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'missing' WHERE id = $1`, [row.id]);
continue;
}
if (!row.fingerprinted_at) {
await db.query('UPDATE files SET sha256 = $2, fingerprinted_at = now() WHERE id = $1',
[row.id, await sha256File(abs)]);
}
if (!GEOMETRY_FORMATS.includes(row.format)) {
await db.query(`UPDATE files SET geometry_status = 'na' WHERE id = $1`, [row.id]);
} else if (pythonAvailable) {
geometry.push({ id: row.id, format: row.format, abs });
}
} catch (err) {
console.warn(`Fingerprint failed for file ${row.id}:`, err.message);
await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [row.id]);
}
}
if (geometry.length) await withPython(() => fingerprintGeometry(geometry, 'files'));
return true;
}
async function indexArchives() {
const { rows } = await db.query(
`SELECT id, file_path, format FROM files
WHERE format = ANY($1) AND archive_status IS NULL ORDER BY id LIMIT 5`, [ARCHIVE_FORMATS]);
for (const file of rows) {
try {
const entries = await archives.listEntries(file);
await store.replaceArchiveEntries(file.id,
entries.map((e) => ({ path: e.path, size: e.size, format: formatOf(e.path) })));
await db.query(`UPDATE files SET archive_status = 'ok', archive_error = NULL WHERE id = $1`, [file.id]);
} catch (err) {
console.warn(`Could not read archive (file ${file.id}):`, err.message);
await db.query(`UPDATE files SET archive_status = 'error', archive_error = $2 WHERE id = $1`,
[file.id, err.message.slice(0, 300)]);
}
}
return rows.length > 0;
}
const ENTRY_PENDING = 'e.fingerprinted_at IS NULL OR (e.geometry_status IS NULL AND $1::boolean)';
// Fingerprints the pending entries of one archive, extracting its models in a
// single pass (cheaper than one extraction per entry, especially for solid 7z).
async function fingerprintEntries() {
const { rows: [file] } = await db.query(
`SELECT f.id, f.file_path, f.format FROM files f
WHERE EXISTS (SELECT 1 FROM archive_entries e WHERE e.file_id = f.id AND (${ENTRY_PENDING}))
ORDER BY f.id LIMIT 1`, [pythonAvailable]);
if (!file) return false;
const { rows: entries } = await db.query(
`SELECT e.id, e.path, e.format, e.fingerprinted_at FROM archive_entries e
WHERE e.file_id = $2 AND (${ENTRY_PENDING}) ORDER BY e.id LIMIT 500`, [pythonAvailable, file.id]);
// Pictures, documents and nested archives are listed but not matched as duplicates.
const others = entries.filter((e) => !MODEL_FORMATS.includes(e.format));
if (others.length) {
await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'na' WHERE id = ANY($1::int[])`,
[others.map((e) => e.id)]);
}
const models = entries.filter((e) => MODEL_FORMATS.includes(e.format));
if (!models.length) return true;
const tmp = archives.tempDir();
try {
let extracted = new Map();
try {
extracted = await archives.extractMany(file, models.map((e) => e.path), tmp);
} catch (err) {
console.warn(`Could not extract from archive (file ${file.id}):`, err.message);
}
const geometry = []; const geometry = [];
for (const row of rows) { for (const entry of models) {
const abs = path.resolve(row.file_path); const abs = extracted.get(entry.path);
try { try {
if (!fs.existsSync(abs)) { if (!abs) {
await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'missing' WHERE id = $1`, [row.id]); await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [entry.id]);
continue; continue;
} }
if (!row.fingerprinted_at) { if (!entry.fingerprinted_at) {
await db.query('UPDATE files SET sha256 = $2, fingerprinted_at = now() WHERE id = $1', await db.query('UPDATE archive_entries SET sha256 = $2, fingerprinted_at = now() WHERE id = $1',
[row.id, await sha256File(abs)]); [entry.id, await sha256File(abs)]);
} }
if (!GEOMETRY_FORMATS.includes(row.format)) { if (!GEOMETRY_FORMATS.includes(entry.format)) {
await db.query(`UPDATE files SET geometry_status = 'na' WHERE id = $1`, [row.id]); await db.query(`UPDATE archive_entries SET geometry_status = 'na' WHERE id = $1`, [entry.id]);
} else if (pythonAvailable) { } else if (pythonAvailable) {
geometry.push(row); geometry.push({ id: entry.id, format: entry.format, abs });
} }
} catch (err) { } catch (err) {
console.warn(`Fingerprint failed for file ${row.id}:`, err.message); console.warn(`Fingerprint failed for archive entry ${entry.id}:`, err.message);
await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [row.id]); await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [entry.id]);
} }
} }
if (geometry.length) await withPython(() => fingerprintGeometry(geometry, 'archive_entries'));
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
return true;
}
if (geometry.length) { async function run() {
try { for (;;) {
await fingerprintGeometry(geometry); const worked = [await fingerprintFiles(), await indexArchives(), await fingerprintEntries()];
} catch (err) { if (!worked.some(Boolean)) return;
if (err.code !== 'ENOENT') throw err;
pythonAvailable = false;
console.warn(`Geometry fingerprints disabled: no Python at ${PYTHON} (content hashes still run).`);
}
}
} }
} }
@ -133,9 +227,12 @@ function kick() {
async function pendingCount() { async function pendingCount() {
const { rows } = await db.query( const { rows } = await db.query(
`SELECT COUNT(*)::int AS n FROM files `SELECT (SELECT COUNT(*) FROM files
WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean)`, [pythonAvailable]); WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean))
return rows[0].n; + (SELECT COUNT(*) FROM files WHERE format = ANY($2) AND archive_status IS NULL)
+ (SELECT COUNT(*) FROM archive_entries e WHERE ${ENTRY_PENDING}) AS n`,
[pythonAvailable, ARCHIVE_FORMATS]);
return Number(rows[0].n);
} }
module.exports = { kick, pendingCount, geometryKey, sha256File }; module.exports = { kick, pendingCount, geometryKey, sha256File };

View file

@ -1,4 +1,5 @@
const db = require('./db'); const db = require('./db');
const { MODEL_FORMATS, ARCHIVE_FORMATS } = require('./fileProcessor');
// ---- shared SELECT that hydrates a model with tags, projects, files, designer ---- // ---- shared SELECT that hydrates a model with tags, projects, files, designer ----
const MODEL_SELECT = ` const MODEL_SELECT = `
@ -25,7 +26,8 @@ const MODEL_SELECT = `
SELECT json_agg(json_build_object( SELECT json_agg(json_build_object(
'id', fl.id, 'originalName', fl.original_name, 'fileName', fl.file_name, 'id', fl.id, 'originalName', fl.original_name, 'fileName', fl.file_name,
'format', fl.format, 'size', fl.size, 'volume', fl.volume, 'format', fl.format, 'size', fl.size, 'volume', fl.volume,
'bbox', fl.bbox, 'facets', fl.facets) ORDER BY fl.id) AS files 'bbox', fl.bbox, 'facets', fl.facets, 'archiveStatus', fl.archive_status,
'entryCount', (SELECT COUNT(*)::int FROM archive_entries ae WHERE ae.file_id = fl.id)) ORDER BY fl.id) AS files
FROM files fl WHERE fl.model_id = m.id FROM files fl WHERE fl.model_id = m.id
) f ON true ) f ON true
`; `;
@ -116,17 +118,60 @@ async function updateModel(id, updates) {
// Returns the on-disk file rows so the caller can unlink them. // Returns the on-disk file rows so the caller can unlink them.
async function deleteModel(id) { async function deleteModel(id) {
const { rows } = await db.query('SELECT file_path, glb_path FROM files WHERE model_id = $1', [id]); const { rows } = await db.query('SELECT id, file_path, glb_path FROM files WHERE model_id = $1', [id]);
const entries = await db.query(
'SELECT e.id FROM archive_entries e JOIN files f ON f.id = e.file_id WHERE f.model_id = $1', [id]);
const model = await db.query('SELECT thumbnail_path FROM models WHERE id = $1', [id]); const model = await db.query('SELECT thumbnail_path FROM models WHERE id = $1', [id]);
const result = await db.query('DELETE FROM models WHERE id = $1 RETURNING id', [id]); const result = await db.query('DELETE FROM models WHERE id = $1 RETURNING id', [id]);
if (result.rowCount === 0) return null; if (result.rowCount === 0) return null;
return { return {
fileIds: rows.map((r) => r.id),
filePaths: rows.map((r) => r.file_path), filePaths: rows.map((r) => r.file_path),
glbPaths: rows.map((r) => r.glb_path).filter(Boolean), glbPaths: [
...rows.map((r) => r.glb_path).filter(Boolean),
...entries.rows.map((e) => `glb/entry-${e.id}.glb`),
],
thumbnailPath: model.rows[0] && model.rows[0].thumbnail_path, thumbnailPath: model.rows[0] && model.rows[0].thumbnail_path,
}; };
} }
// ---------------------------------------------------------------- files / archive entries
async function getFile(id) {
const { rows } = await db.query('SELECT * FROM files WHERE id = $1', [id]);
return rows[0] || null;
}
async function getArchiveEntries(fileId) {
const { rows } = await db.query(
'SELECT id, path, format, size FROM archive_entries WHERE file_id = $1 ORDER BY path', [fileId]);
return rows.map((r) => ({ ...r, size: Number(r.size) }));
}
// An entry plus the archive it lives in ({ id, file_path, format }).
async function getArchiveEntry(id) {
const { rows } = await db.query(
`SELECT e.id, e.file_id, e.path, e.format, e.size, f.file_path, f.format AS archive_format
FROM archive_entries e JOIN files f ON f.id = e.file_id WHERE e.id = $1`, [id]);
const r = rows[0];
if (!r) return null;
return {
id: r.id, file_id: r.file_id, path: r.path, format: r.format, size: Number(r.size),
archive: { id: r.file_id, file_path: r.file_path, format: r.archive_format },
};
}
async function replaceArchiveEntries(fileId, entries) {
await db.transaction(async (client) => {
await client.query('DELETE FROM archive_entries WHERE file_id = $1', [fileId]);
if (!entries.length) return;
await client.query(
`INSERT INTO archive_entries (file_id, path, format, size)
SELECT $1::int, * FROM unnest($2::text[], $3::text[], $4::bigint[])
ON CONFLICT (file_id, path) DO NOTHING`,
[fileId, entries.map((e) => e.path), entries.map((e) => e.format), entries.map((e) => e.size)]);
});
}
// ---------------------------------------------------------------- trash // ---------------------------------------------------------------- trash
async function trashModels(ids) { async function trashModels(ids) {
const { rowCount } = await db.query( const { rowCount } = await db.query(
@ -153,21 +198,42 @@ async function countTrashed() {
// ---------------------------------------------------------------- duplicates // ---------------------------------------------------------------- duplicates
// Files of non-trashed models whose content hash or geometry fingerprint also // Files of non-trashed models whose content hash or geometry fingerprint also
// appears in another non-trashed model — the input to the duplicate finder. // appears in another non-trashed model — the input to the duplicate finder.
// Ids are prefixed ('f12' for a file, 'e7' for an archive entry) so both kinds
// can be matched together. Pictures and documents never count.
async function duplicateCandidateFiles() { async function duplicateCandidateFiles() {
const { rows } = await db.query(` const { rows } = await db.query(`
WITH live AS ( WITH live AS (
SELECT f.id, f.model_id, f.sha256, f.geometry_hash SELECT 'f' || f.id AS id, f.model_id, f.sha256, f.geometry_hash
FROM files f JOIN models m ON m.id = f.model_id FROM files f JOIN models m ON m.id = f.model_id
WHERE m.deleted_at IS NULL AND f.size > 0 WHERE m.deleted_at IS NULL AND f.size > 0 AND f.format = ANY($1)
UNION ALL
SELECT 'e' || e.id, f.model_id, e.sha256, e.geometry_hash
FROM archive_entries e JOIN files f ON f.id = e.file_id JOIN models m ON m.id = f.model_id
WHERE m.deleted_at IS NULL AND e.size > 0 AND e.format = ANY($2)
) )
SELECT * FROM live SELECT * FROM live
WHERE sha256 IN (SELECT sha256 FROM live WHERE sha256 IS NOT NULL WHERE sha256 IN (SELECT sha256 FROM live WHERE sha256 IS NOT NULL
GROUP BY sha256 HAVING COUNT(DISTINCT model_id) > 1) GROUP BY sha256 HAVING COUNT(DISTINCT model_id) > 1)
OR geometry_hash IN (SELECT geometry_hash FROM live WHERE geometry_hash IS NOT NULL OR geometry_hash IN (SELECT geometry_hash FROM live WHERE geometry_hash IS NOT NULL
GROUP BY geometry_hash HAVING COUNT(DISTINCT model_id) > 1)`); GROUP BY geometry_hash HAVING COUNT(DISTINCT model_id) > 1)`,
[[...MODEL_FORMATS, ...ARCHIVE_FORMATS], MODEL_FORMATS]);
return rows; return rows;
} }
// How many duplicate-matchable items (model files, archives, models inside
// archives) each model holds: { modelId: n }.
async function matchableCounts(modelIds) {
if (!modelIds.length) return {};
const { rows } = await db.query(`
SELECT m.id,
(SELECT COUNT(*) FROM files f WHERE f.model_id = m.id AND f.format = ANY($2))
+ (SELECT COUNT(*) FROM archive_entries e JOIN files f ON f.id = e.file_id
WHERE f.model_id = m.id AND e.format = ANY($3)) AS n
FROM models m WHERE m.id = ANY($1::int[])`,
[modelIds, [...MODEL_FORMATS, ...ARCHIVE_FORMATS], MODEL_FORMATS]);
return Object.fromEntries(rows.map((r) => [r.id, Number(r.n)]));
}
// Keep one model of a duplicate group: fold the others' tags, collections and // Keep one model of a duplicate group: fold the others' tags, collections and
// notes into it, then move the others to the trash. // notes into it, then move the others to the trash.
async function mergeAndTrash(keepId, otherIds) { async function mergeAndTrash(keepId, otherIds) {
@ -407,7 +473,8 @@ module.exports = {
getAllModels, getModel, getModelsByIds, findModelBySource, createModel, updateModel, deleteModel, addFiles, getAllModels, getModel, getModelsByIds, findModelBySource, createModel, updateModel, deleteModel, addFiles,
updateFileConversion, updateFileConversion,
trashModels, restoreModel, trashedModelIds, countTrashed, trashModels, restoreModel, trashedModelIds, countTrashed,
duplicateCandidateFiles, mergeAndTrash, dismissDuplicateGroup, dismissedDuplicateSignatures, duplicateCandidateFiles, matchableCounts, mergeAndTrash, dismissDuplicateGroup, dismissedDuplicateSignatures,
getFile, getArchiveEntries, getArchiveEntry, replaceArchiveEntries,
getAllTags, getAllTags,
getAllProjects, getProject, createProject, updateProject, deleteProject, getAllProjects, getProject, createProject, updateProject, deleteProject,
getSetting, setSetting, getSetting, setSetting,

109
package-lock.json generated
View file

@ -9,6 +9,7 @@
"version": "1.0.0", "version": "1.0.0",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"7z-wasm": "^1.2.0",
"adm-zip": "^0.5.17", "adm-zip": "^0.5.17",
"bcryptjs": "^3.0.3", "bcryptjs": "^3.0.3",
"cookie-parser": "^1.4.7", "cookie-parser": "^1.4.7",
@ -18,12 +19,42 @@
"multer": "^1.4.5-lts.1", "multer": "^1.4.5-lts.1",
"node-stl": "^0.7.0", "node-stl": "^0.7.0",
"nodemailer": "^9.0.1", "nodemailer": "^9.0.1",
"pg": "^8.22.0" "occt-import-js": "^0.0.23",
"pg": "^8.22.0",
"rhino3dm": "^8.32.2",
"tar": "^7.5.22"
}, },
"devDependencies": { "devDependencies": {
"nodemon": "^3.0.2" "nodemon": "^3.0.2"
} }
}, },
"node_modules/@isaacs/fs-minipass": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
"integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==",
"license": "ISC",
"dependencies": {
"minipass": "^7.0.4"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/7z-wasm": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/7z-wasm/-/7z-wasm-1.2.0.tgz",
"integrity": "sha512-AZjtWXFleAe5nQ27JYISs70cICjO0zZr4LU45I9pXKcrhG0qiEWgq3Vjwr9JHHihFLuqgc7uSHYVn/z3vXCjdw==",
"license": "SEE LICENSE IN License.txt",
"dependencies": {
"readline-sync": "^1.4.10"
},
"bin": {
"7z-wasm": "bin/cli"
},
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/accepts": { "node_modules/accepts": {
"version": "1.3.8", "version": "1.3.8",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@ -240,6 +271,15 @@
"fsevents": "~2.3.2" "fsevents": "~2.3.2"
} }
}, },
"node_modules/chownr": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
"integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
"license": "BlueOak-1.0.0",
"engines": {
"node": ">=18"
}
},
"node_modules/concat-stream": { "node_modules/concat-stream": {
"version": "1.6.2", "version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
@ -945,6 +985,27 @@
"url": "https://github.com/sponsors/ljharb" "url": "https://github.com/sponsors/ljharb"
} }
}, },
"node_modules/minipass": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz",
"integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==",
"license": "BlueOak-1.0.0",
"engines": {
"node": ">=16 || 14 >=14.17"
}
},
"node_modules/minizlib": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz",
"integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==",
"license": "MIT",
"dependencies": {
"minipass": "^7.1.2"
},
"engines": {
"node": ">= 18"
}
},
"node_modules/mkdirp": { "node_modules/mkdirp": {
"version": "0.5.6", "version": "0.5.6",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
@ -1091,6 +1152,12 @@
"url": "https://github.com/sponsors/ljharb" "url": "https://github.com/sponsors/ljharb"
} }
}, },
"node_modules/occt-import-js": {
"version": "0.0.23",
"resolved": "https://registry.npmjs.org/occt-import-js/-/occt-import-js-0.0.23.tgz",
"integrity": "sha512-RFfYQXYFX5C1mB1Aywm0ShcUKzXOr/VzTnlzhBSDJOR6YCAPt1HYCzeXWg1vwwjn/cUxwqRNhhtf1dlewoZYCQ==",
"license": "LGPL-2.1"
},
"node_modules/on-finished": { "node_modules/on-finished": {
"version": "2.4.1", "version": "2.4.1",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
@ -1358,6 +1425,21 @@
"node": ">=8.10.0" "node": ">=8.10.0"
} }
}, },
"node_modules/readline-sync": {
"version": "1.4.10",
"resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz",
"integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==",
"license": "MIT",
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/rhino3dm": {
"version": "8.32.2",
"resolved": "https://registry.npmjs.org/rhino3dm/-/rhino3dm-8.32.2.tgz",
"integrity": "sha512-Q3/5iHIKJeoE5doXrRy7C1vJer4F8uMYe8K6B8VQHqhGj5BCS1L3EkqNpKyUTERoHyKh7rMmiiZpw9iEUkCnPQ==",
"license": "MIT"
},
"node_modules/safe-buffer": { "node_modules/safe-buffer": {
"version": "5.2.1", "version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@ -1586,6 +1668,22 @@
"node": ">=4" "node": ">=4"
} }
}, },
"node_modules/tar": {
"version": "7.5.22",
"resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz",
"integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==",
"license": "BlueOak-1.0.0",
"dependencies": {
"@isaacs/fs-minipass": "^4.0.0",
"chownr": "^3.0.0",
"minipass": "^7.1.2",
"minizlib": "^3.1.0",
"yallist": "^5.0.0"
},
"engines": {
"node": ">=18"
}
},
"node_modules/to-regex-range": { "node_modules/to-regex-range": {
"version": "5.0.1", "version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
@ -1685,6 +1783,15 @@
"engines": { "engines": {
"node": ">=0.4" "node": ">=0.4"
} }
},
"node_modules/yallist": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz",
"integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==",
"license": "BlueOak-1.0.0",
"engines": {
"node": ">=18"
}
} }
} }
} }

View file

@ -17,6 +17,7 @@
"author": "", "author": "",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"7z-wasm": "^1.2.0",
"adm-zip": "^0.5.17", "adm-zip": "^0.5.17",
"bcryptjs": "^3.0.3", "bcryptjs": "^3.0.3",
"cookie-parser": "^1.4.7", "cookie-parser": "^1.4.7",
@ -26,7 +27,10 @@
"multer": "^1.4.5-lts.1", "multer": "^1.4.5-lts.1",
"node-stl": "^0.7.0", "node-stl": "^0.7.0",
"nodemailer": "^9.0.1", "nodemailer": "^9.0.1",
"pg": "^8.22.0" "occt-import-js": "^0.0.23",
"pg": "^8.22.0",
"rhino3dm": "^8.32.2",
"tar": "^7.5.22"
}, },
"devDependencies": { "devDependencies": {
"nodemon": "^3.0.2" "nodemon": "^3.0.2"

View file

@ -189,8 +189,8 @@ async function generateMissingThumbnails() {
if (!file) continue; if (!file) continue;
thumbAttempted.add(m.id); thumbAttempted.add(m.id);
try { try {
const src = viewerSourceFor(file.id, file.format); const src = viewerSourceFor({ kind: 'file', id: file.id, format: file.format });
const image = await window.ModelViewer.snapshot(src.url, src.format, 512); const image = await window.ModelViewer.snapshot(src.url, src.format, 512, { resolve: resolverFor(m) });
if (!image) continue; if (!image) continue;
const res = await fetch(`/api/models/${m.id}/thumbnail`, { const res = await fetch(`/api/models/${m.id}/thumbnail`, {
method: 'POST', method: 'POST',
@ -484,7 +484,7 @@ function duplicateGroupHtml(g) {
function duplicateCardHtml(g, m) { function duplicateCardHtml(g, m) {
const recommended = m.id === g.recommendedId; const recommended = m.id === g.recommendedId;
const unique = m.files.length - m.matchedFiles; const unique = m.matchableFiles - m.matchedFiles;
return ` return `
<div class="dup-card${recommended ? ' recommended' : ''}" data-model-id="${m.id}"> <div class="dup-card${recommended ? ' recommended' : ''}" data-model-id="${m.id}">
<div class="file-thumbnail">${thumbHtml(m)}</div> <div class="file-thumbnail">${thumbHtml(m)}</div>
@ -492,7 +492,7 @@ function duplicateCardHtml(g, m) {
${recommended ? '<span class="rec-badge">Recommended keep</span>' : ''} ${recommended ? '<span class="rec-badge">Recommended keep</span>' : ''}
<div class="file-name" title="${escapeHtml(m.name)}">${escapeHtml(m.name)}</div> <div class="file-name" title="${escapeHtml(m.name)}">${escapeHtml(m.name)}</div>
<div class="file-meta">${escapeHtml(sourceLabel(sourceKey(m)))} • ${formatDate(m.date_added)}</div> <div class="file-meta">${escapeHtml(sourceLabel(sourceKey(m)))} • ${formatDate(m.date_added)}</div>
<div class="file-meta">${m.matchedFiles} of ${m.files.length} files match${unique ? ` • <span class="warn">${unique} unique</span>` : ''}</div> <div class="file-meta">${m.matchedFiles} of ${m.matchableFiles} files match${unique ? ` • <span class="warn">${unique} unique</span>` : ''}</div>
<button class="btn btn-primary btn-sm" data-action="keep">Keep this one</button> <button class="btn btn-primary btn-sm" data-action="keep">Keep this one</button>
</div> </div>
</div>`; </div>`;
@ -505,7 +505,7 @@ async function handleKeepDuplicate(signature, keepId) {
const others = group.models.filter((m) => m.id !== keepId); const others = group.models.filter((m) => m.id !== keepId);
let message = `Keep "${keep.name}" and move ${others.length} other model${others.length === 1 ? '' : 's'} to the trash?\n\n` let message = `Keep "${keep.name}" and move ${others.length} other model${others.length === 1 ? '' : 's'} to the trash?\n\n`
+ 'Their tags, collections and notes are merged into the one you keep.'; + 'Their tags, collections and notes are merged into the one you keep.';
const withUnique = others.filter((m) => m.files.length > m.matchedFiles); const withUnique = others.filter((m) => m.matchableFiles > m.matchedFiles);
if (withUnique.length) { if (withUnique.length) {
message += `\n\nHeads up: ${withUnique.map((m) => `"${m.name}"`).join(', ')} also contain files that aren't duplicated — they go to the trash too.`; message += `\n\nHeads up: ${withUnique.map((m) => `"${m.name}"`).join(', ')} also contain files that aren't duplicated — they go to the trash too.`;
} }
@ -700,6 +700,22 @@ function setupEventListeners() {
}); });
document.getElementById('detailNotes').addEventListener('input', queueNotesSave); document.getElementById('detailNotes').addEventListener('input', queueNotesSave);
document.getElementById('detailNotes').addEventListener('blur', flushNotes); document.getElementById('detailNotes').addEventListener('blur', flushNotes);
document.getElementById('detailFileTabs').addEventListener('click', (e) => {
const tab = e.target.closest('.file-tab');
const item = tab && detailItems.find((i) => itemKey(i) === tab.dataset.key);
if (!item) return;
renderViewerTabs(itemKey(item));
loadViewer(item);
});
document.getElementById('detailFileInfo').addEventListener('click', (e) => {
const btn = e.target.closest('[data-view-entry]');
if (!btn) return;
const archiveId = Number(btn.dataset.archive);
const entry = (archiveEntryCache.get(archiveId) || []).find((x) => String(x.id) === btn.dataset.viewEntry);
if (!entry) return;
openViewerItem({ kind: 'entry', id: entry.id, format: entry.format, name: entry.path.split('/').pop(), archiveId });
document.getElementById('detailViewer').scrollIntoView({ block: 'nearest', behavior: 'smooth' });
});
setupTagAutocomplete(document.getElementById('fileTags')); setupTagAutocomplete(document.getElementById('fileTags'));
setupTagAutocomplete(document.getElementById('editTags')); setupTagAutocomplete(document.getElementById('editTags'));
@ -924,38 +940,98 @@ function openModelDetail(modelId) {
<div class="file-row"> <div class="file-row">
<span>${escapeHtml(f.originalName)} <em>(${formatFileSize(f.size)}${f.volume ? `, ${Number(f.volume).toFixed(2)} cm³` : ''})</em></span> <span>${escapeHtml(f.originalName)} <em>(${formatFileSize(f.size)}${f.volume ? `, ${Number(f.volume).toFixed(2)} cm³` : ''})</em></span>
<a href="/api/files/${f.id}/raw" download>download</a> <a href="/api/files/${f.id}/raw" download>download</a>
</div>`).join(''); </div>
${ARCHIVE_FORMATS.includes(f.format) ? `<div class="entry-list" data-entries-for="${f.id}"><div class="entry-head">Reading archive…</div></div>` : ''}`).join('');
model.files.filter((f) => ARCHIVE_FORMATS.includes(f.format)).forEach((f) => loadArchiveEntries(model.id, f.id));
} }
// Viewer tabs // Viewer tabs: 3D files first, then pictures/PDFs/text (entries inside
const renderable = model.files.filter((f) => window.ModelViewer.RENDERABLE.includes(f.format)); // archives get a tab when opened from the file list).
const tabs = document.getElementById('detailFileTabs'); const { RENDERABLE, DOCUMENT_FORMATS } = window.ModelViewer;
tabs.innerHTML = renderable.map((f, i) => ` detailItems = [
<button class="file-tab${i === 0 ? ' active' : ''}" data-file-id="${f.id}" data-format="${f.format}">${escapeHtml(f.originalName)}</button>`).join(''); ...model.files.filter((f) => RENDERABLE.includes(f.format)),
tabs.querySelectorAll('.file-tab').forEach((tab) => { ...model.files.filter((f) => DOCUMENT_FORMATS.includes(f.format) && f.format !== 'mtl'),
tab.addEventListener('click', () => { ].map((f) => ({ kind: 'file', id: f.id, format: f.format, name: f.originalName }));
tabs.querySelectorAll('.file-tab').forEach((t) => t.classList.remove('active')); renderViewerTabs(detailItems.length ? itemKey(detailItems[0]) : null);
tab.classList.add('active');
loadViewer(tab.dataset.fileId, tab.dataset.format);
});
});
fileDetailModal.classList.add('active'); fileDetailModal.classList.add('active');
currentDetailFile = null; currentDetailFile = null;
if (renderable.length) { if (detailItems.length) {
loadViewer(renderable[0].id, renderable[0].format); loadViewer(detailItems[0]);
} else { } else {
document.getElementById('regenThumbBtn').style.display = 'none'; document.getElementById('regenThumbBtn').style.display = 'none';
window.ModelViewer.render(document.getElementById('detailViewer'), '', 'none', { fallbackImage: currentDetailThumb }); window.ModelViewer.render(document.getElementById('detailViewer'), '', 'none', { fallbackImage: currentDetailThumb });
} }
} }
// Maps a file to what the viewer should load: 3MF goes through the server's // ---------------------------------------------------------------- viewer items
// GLB conversion; everything else is served raw. // An item is something the viewer can show: { kind: 'file' | 'entry', id, format, name, archiveId? }.
function viewerSourceFor(fileId, format) { const ARCHIVE_FORMATS = ['zip', '7z', 'rar', 'tar', 'tgz'];
return format === '3mf' let detailItems = [];
? { url: `/api/files/${fileId}/glb`, format: 'glb' } const archiveEntryCache = new Map(); // archive file id -> entries
: { url: `/api/files/${fileId}/raw`, format }; const itemKey = (item) => `${item.kind}:${item.id}`;
function renderViewerTabs(activeKey) {
document.getElementById('detailFileTabs').innerHTML = detailItems.map((item) => `
<button class="file-tab${itemKey(item) === activeKey ? ' active' : ''}" data-key="${itemKey(item)}">${escapeHtml(item.name)}</button>`).join('');
}
function openViewerItem(item) {
if (!detailItems.some((i) => itemKey(i) === itemKey(item))) detailItems.push(item);
renderViewerTabs(itemKey(item));
loadViewer(item);
}
// What the viewer should load: 3MF goes through the server's GLB conversion,
// documents are served inline, everything else raw.
function viewerSourceFor(item) {
const base = `/api/${item.kind === 'entry' ? 'entries' : 'files'}/${item.id}`;
if (item.format === '3mf') return { url: `${base}/glb`, format: 'glb' };
if (window.ModelViewer.DOCUMENT_FORMATS.includes(item.format)) return { url: `${base}/raw?inline=1`, format: item.format };
return { url: `${base}/raw`, format: item.format };
}
// Finds files shipped alongside a model by name — OBJ materials, FBX/MTL
// textures. Entries of the same archive win, then the model's own files.
function resolverFor(model, archiveEntries = null) {
const byName = new Map();
const key = (name) => String(name).split(/[\\/]/).pop().toLowerCase();
const add = (name, url) => { if (!byName.has(key(name))) byName.set(key(name), url); };
(archiveEntries || []).forEach((e) => add(e.path, `/api/entries/${e.id}/raw`));
((model && model.files) || []).forEach((f) => add(f.originalName, `/api/files/${f.id}/raw`));
return (name) => byName.get(key(name)) || null;
}
async function loadArchiveEntries(modelId, fileId) {
const container = () => (fileDetailModal.dataset.currentModelId === String(modelId)
? document.querySelector(`[data-entries-for="${fileId}"]`) : null);
try {
const res = await fetch(`/api/files/${fileId}/entries`);
const data = await res.json();
if (!res.ok) throw new Error(data.error);
if (!container()) return;
if (data.status === 'pending') {
setTimeout(() => { if (container()) loadArchiveEntries(modelId, fileId); }, 2000);
return;
}
if (data.status === 'error') {
container().innerHTML = `<div class="entry-head">Couldn't read this archive: ${escapeHtml(data.error || 'unknown error')}</div>`;
return;
}
archiveEntryCache.set(fileId, data.entries);
const { PREVIEWABLE } = window.ModelViewer;
container().innerHTML = `<div class="entry-head">${data.entries.length} file${data.entries.length === 1 ? '' : 's'} inside</div>`
+ data.entries.map((e) => `
<div class="entry-row">
<span title="${escapeHtml(e.path)}">${escapeHtml(e.path.replace(/^\.\//, ''))} <em>${formatFileSize(e.size)}</em></span>
<span class="row-actions">
${PREVIEWABLE.includes(e.format) && e.format !== 'mtl' ? `<button type="button" class="link-btn" data-view-entry="${e.id}" data-archive="${fileId}">view</button>` : ''}
<a href="/api/entries/${e.id}/raw" download>download</a>
</span>
</div>`).join('');
} catch (err) {
if (container()) container().innerHTML = `<div class="entry-head">Couldn't list this archive: ${escapeHtml(err.message)}</div>`;
}
} }
function renderDetailLicense(model) { function renderDetailLicense(model) {
@ -1068,11 +1144,15 @@ async function handleRefreshMetadata() {
} }
} }
function loadViewer(fileId, format) { function loadViewer(item) {
const src = viewerSourceFor(fileId, format); const src = viewerSourceFor(item);
currentDetailFile = { id: fileId, viewerUrl: src.url, viewerFormat: src.format }; const model = findModel(fileDetailModal.dataset.currentModelId);
document.getElementById('regenThumbBtn').style.display = ''; const resolve = resolverFor(model, item.kind === 'entry' ? archiveEntryCache.get(item.archiveId) : null);
window.ModelViewer.render(document.getElementById('detailViewer'), src.url, src.format, { fallbackImage: currentDetailThumb }); const is3d = window.ModelViewer.MODEL_FORMATS.includes(src.format);
currentDetailFile = is3d ? { viewerUrl: src.url, viewerFormat: src.format, resolve } : null;
document.getElementById('regenThumbBtn').style.display = is3d ? '' : 'none';
window.ModelViewer.render(document.getElementById('detailViewer'), src.url, src.format,
{ fallbackImage: currentDetailThumb, resolve });
} }
function updateCardThumbnail(modelId, thumbnailUrl) { function updateCardThumbnail(modelId, thumbnailUrl) {
@ -1089,7 +1169,8 @@ async function handleRegenThumbnail() {
btn.disabled = true; btn.disabled = true;
btn.textContent = 'Rendering…'; btn.textContent = 'Rendering…';
try { try {
const image = await window.ModelViewer.snapshot(currentDetailFile.viewerUrl, currentDetailFile.viewerFormat, 512); const image = await window.ModelViewer.snapshot(currentDetailFile.viewerUrl, currentDetailFile.viewerFormat, 512,
{ resolve: currentDetailFile.resolve });
const res = await fetch(`/api/models/${modelId}/thumbnail`, { const res = await fetch(`/api/models/${modelId}/thumbnail`, {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },

View file

@ -329,8 +329,8 @@
<div class="modal-body"> <div class="modal-body">
<form id="uploadForm"> <form id="uploadForm">
<div class="form-group"> <div class="form-group">
<label for="fileInput">Files * <span class="form-hint-inline">(stl, 3mf, obj, step, gcode, zip — multiple allowed)</span></label> <label for="fileInput">Files * <span class="form-hint-inline">(models, archives, pictures, PDFs &amp; readmes — multiple allowed)</span></label>
<input type="file" id="fileInput" accept=".stl,.3mf,.obj,.step,.stp,.gcode,.gco,.zip" multiple required> <input type="file" id="fileInput" accept=".stl,.3mf,.obj,.fbx,.step,.stp,.3dm,.gcode,.gco,.zip,.7z,.rar,.tar,.tgz,.gz,.jpg,.jpeg,.png,.gif,.webp,.bmp,.svg,.pdf,.txt,.md,.mtl" multiple required>
</div> </div>
<div class="form-group"> <div class="form-group">
@ -448,7 +448,7 @@
<button type="button" class="btn btn-secondary btn-sm" id="downloadFilesBtn" style="margin-top:10px; display:none"></button> <button type="button" class="btn btn-secondary btn-sm" id="downloadFilesBtn" style="margin-top:10px; display:none"></button>
</div> </div>
<input type="file" id="addFilesInput" multiple style="display:none" <input type="file" id="addFilesInput" multiple style="display:none"
accept=".stl,.3mf,.obj,.step,.stp,.gcode,.gco,.zip"> accept=".stl,.3mf,.obj,.fbx,.step,.stp,.3dm,.gcode,.gco,.zip,.7z,.rar,.tar,.tgz,.gz,.jpg,.jpeg,.png,.gif,.webp,.bmp,.svg,.pdf,.txt,.md,.mtl">
<div class="form-actions" id="detailActions"> <div class="form-actions" id="detailActions">
<button class="btn btn-danger" id="deleteFileBtn">Move to trash</button> <button class="btn btn-danger" id="deleteFileBtn">Move to trash</button>
<button class="btn btn-secondary" id="addFilesBtn">Add files</button> <button class="btn btn-secondary" id="addFilesBtn">Add files</button>

View file

@ -214,6 +214,20 @@ label { display: block; font-size: 13px; font-weight: 500; color: var(--text-mut
.viewer-fallback img { max-width: 100%; max-height: 320px; object-fit: contain; border-radius: var(--radius-sm); } .viewer-fallback img { max-width: 100%; max-height: 320px; object-fit: contain; border-radius: var(--radius-sm); }
.viewer-fallback span { color: var(--text-dim); font-size: 12px; text-align: center; } .viewer-fallback span { color: var(--text-dim); font-size: 12px; text-align: center; }
.viewer-image { max-width: 100%; max-height: 100%; object-fit: contain; }
.viewer-pdf, .viewer-text { align-self: stretch; width: 100%; height: 100%; border: 0; border-radius: var(--radius); }
.viewer-pdf { background: #fff; }
.viewer-text { margin: 0; padding: 14px 16px; overflow: auto; font-family: var(--mono); font-size: 12px; line-height: 1.55; color: var(--text); white-space: pre-wrap; word-break: break-word; text-align: left; }
.info-section .entry-list { margin: 2px 0 8px 10px; padding-left: 12px; border-left: 1px solid var(--border); max-height: 260px; overflow-y: auto; }
.info-section .entry-head { font-size: 12px; color: var(--text-dim); margin: 6px 0 2px; }
.info-section .entry-row { display: flex; justify-content: space-between; align-items: center; gap: 10px; padding: 3px 0; font-size: 13px; color: var(--text-muted); }
.entry-row > span:first-child { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; }
.entry-row em { color: var(--text-dim); font-style: normal; font-family: var(--mono); font-size: 11px; margin-left: 4px; }
.row-actions { display: flex; gap: 12px; flex-shrink: 0; align-items: center; }
.link-btn { background: none; border: none; color: var(--accent); font: inherit; cursor: pointer; padding: 0; }
.link-btn:hover { text-decoration: underline; }
.file-tabs { display: flex; flex-wrap: wrap; gap: 6px; } .file-tabs { display: flex; flex-wrap: wrap; gap: 6px; }
.file-tab { border: 1px solid var(--border); background: var(--surface-2); color: var(--text-muted); padding: 5px 11px; border-radius: var(--radius-sm); font-size: 12px; cursor: pointer; font-family: inherit; } .file-tab { border: 1px solid var(--border); background: var(--surface-2); color: var(--text-muted); padding: 5px 11px; border-radius: var(--radius-sm); font-size: 12px; cursor: pointer; font-family: inherit; }
.file-tab:hover { color: var(--text); } .file-tab:hover { color: var(--text); }

View file

@ -1,16 +1,27 @@
// In-browser 3D preview for STL / 3MF / OBJ files, loaded as an ES module. // In-browser preview for 3D models (STL, OBJ + MTL, 3MF via GLB, FBX, STEP, 3DM)
// and for the pictures, PDFs and text files that come with them.
import * as THREE from 'three'; import * as THREE from 'three';
import { OrbitControls } from 'three/addons/controls/OrbitControls.js'; import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
import { STLLoader } from 'three/addons/loaders/STLLoader.js'; import { STLLoader } from 'three/addons/loaders/STLLoader.js';
import { OBJLoader } from 'three/addons/loaders/OBJLoader.js'; import { OBJLoader } from 'three/addons/loaders/OBJLoader.js';
import { ThreeMFLoader } from 'three/addons/loaders/3MFLoader.js'; import { MTLLoader } from 'three/addons/loaders/MTLLoader.js';
import { FBXLoader } from 'three/addons/loaders/FBXLoader.js';
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
import { Rhino3dmLoader } from 'three/addons/loaders/3DMLoader.js';
// Formats the 3D renderer draws directly (3MF arrives as a server-converted GLB).
const MODEL_FORMATS = ['stl', 'obj', 'glb', 'fbx', '3dm', 'step', 'stp'];
const IMAGE_FORMATS = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'svg'];
const TEXT_FORMATS = ['txt', 'md', 'mtl'];
const DOCUMENT_FORMATS = [...IMAGE_FORMATS, 'pdf', ...TEXT_FORMATS];
// What the app offers a preview for.
const RENDERABLE = ['stl', '3mf', 'obj', 'fbx', '3dm', 'step', 'stp'];
const PREVIEWABLE = [...RENDERABLE, ...DOCUMENT_FORMATS];
// Formats authored Z-up; turned to sit upright in three.js's Y-up world.
const Z_UP = ['stl', 'glb', '3dm', 'step', 'stp'];
// Formats the renderer can draw directly. 3MF is rendered via server-converted
// GLB (see RENDERABLE for what the app treats as previewable input).
const SUPPORTED = ['stl', 'obj', 'glb'];
const RENDERABLE = ['stl', '3mf', 'obj'];
let active = null; // { renderer, controls, rafId } let active = null; // { renderer, controls, rafId }
let renderToken = 0; // bumps on every render, so a slow earlier load can't draw over a newer one
function dispose() { function dispose() {
if (!active) return; if (!active) return;
@ -26,8 +37,7 @@ function message(container, text) {
container.innerHTML = `<div class="viewer-msg">${text}</div>`; container.innerHTML = `<div class="viewer-msg">${text}</div>`;
} }
// When 3D rendering isn't possible (e.g. Bambu/MakerWorld 3MF project files that // When 3D rendering isn't possible, show the model's preview image instead of an error.
// the loader can't parse), show the model's preview image instead of an error.
function showFallback(container, image, text) { function showFallback(container, image, text) {
if (image) { if (image) {
container.innerHTML = container.innerHTML =
@ -37,33 +47,172 @@ function showFallback(container, image, text) {
} }
} }
async function loadObject(url, format) { async function fetchOk(url) {
const res = await fetch(url);
if (!res.ok) throw new Error(`HTTP ${res.status} loading ${url}`);
return res;
}
const basename = (p) => String(p).split(/[\\/]/).pop();
function defaultMaterial(color) {
return new THREE.MeshStandardMaterial({ color: color || 0xc4cad6, metalness: 0.15, roughness: 0.65 });
}
// A loading manager that points material and texture references ("wood.png",
// "../tex/wood.png") at the files shipped alongside the model. `own` is the
// URL being loaded itself, which is left alone.
function managerFor(resolve, own) {
const manager = new THREE.LoadingManager();
if (resolve) {
manager.setURLModifier((url) => {
if (url === own || url.startsWith('blob:') || url.startsWith('data:')) return url;
return resolve(decodeURIComponent(basename(url.split('?')[0]))) || url;
});
}
return manager;
}
async function loadObj(url, resolve) {
const text = await (await fetchOk(url)).text();
const loader = new OBJLoader();
const mtllib = /^mtllib\s+(.+)$/m.exec(text);
const mtlUrl = mtllib && resolve ? resolve(basename(mtllib[1].trim())) : null;
let textured = false;
if (mtlUrl) {
try {
const materials = await new MTLLoader(managerFor(resolve, mtlUrl)).loadAsync(mtlUrl);
materials.preload();
loader.setMaterials(materials);
textured = true;
} catch { /* fall back to plain grey */ }
}
const object = loader.parse(text);
if (!textured) object.traverse((c) => { if (c.isMesh) c.material = defaultMaterial(); });
return object;
}
// OpenCascade (WASM) is large, so it's fetched the first time a STEP file is opened.
let occtPromise = null;
function occt() {
if (!occtPromise) {
occtPromise = new Promise((resolve, reject) => {
const script = document.createElement('script');
script.src = '/vendor/occt/occt-import-js.js';
script.onload = () => resolve(window.occtimportjs({ locateFile: (name) => `/vendor/occt/${name}` }));
script.onerror = () => { occtPromise = null; reject(new Error('Could not load the STEP reader')); };
document.head.appendChild(script);
});
}
return occtPromise;
}
async function loadStep(url) {
const [reader, buffer] = await Promise.all([occt(), fetchOk(url).then((r) => r.arrayBuffer())]);
const result = reader.ReadStepFile(new Uint8Array(buffer), null);
if (!result.success || !result.meshes.length) throw new Error('No geometry in this STEP file');
const group = new THREE.Group();
for (const m of result.meshes) {
const geometry = new THREE.BufferGeometry();
geometry.setAttribute('position', new THREE.Float32BufferAttribute(m.attributes.position.array, 3));
if (m.attributes.normal) geometry.setAttribute('normal', new THREE.Float32BufferAttribute(m.attributes.normal.array, 3));
geometry.setIndex(Array.from(m.index.array));
if (!m.attributes.normal) geometry.computeVertexNormals();
const color = m.color ? new THREE.Color(m.color[0], m.color[1], m.color[2]) : null;
group.add(new THREE.Mesh(geometry, defaultMaterial(color)));
}
return group;
}
async function load3dm(url) {
const loader = new Rhino3dmLoader().setLibraryPath('/vendor/rhino3dm/');
try {
return await loader.loadAsync(url);
} finally {
loader.dispose();
}
}
async function loadObject(url, format, opts = {}) {
let object;
if (format === 'stl') { if (format === 'stl') {
const geometry = await new STLLoader().loadAsync(url); const geometry = await new STLLoader().loadAsync(url);
geometry.computeVertexNormals(); geometry.computeVertexNormals();
const material = new THREE.MeshStandardMaterial({ color: 0xc4cad6, metalness: 0.15, roughness: 0.65 }); object = new THREE.Mesh(geometry, defaultMaterial());
const mesh = new THREE.Mesh(geometry, material); } else if (format === 'obj') {
mesh.rotation.x = -Math.PI / 2; // STL is typically Z-up; sit it flat object = await loadObj(url, opts.resolve);
return mesh; } else if (format === 'fbx') {
object = await new FBXLoader(managerFor(opts.resolve, url)).loadAsync(url);
} else if (format === 'glb') {
object = (await new GLTFLoader().loadAsync(url)).scene;
} else if (format === '3dm') {
object = await load3dm(url);
} else if (format === 'step' || format === 'stp') {
object = await loadStep(url);
} else {
throw new Error(`Unsupported format ${format}`);
} }
if (format === 'obj') return new OBJLoader().loadAsync(url); if (Z_UP.includes(format)) object.rotation.x = -Math.PI / 2;
if (format === '3mf') return new ThreeMFLoader().loadAsync(url); return object;
if (format === 'glb') { }
const gltf = await new GLTFLoader().loadAsync(url);
gltf.scene.rotation.x = -Math.PI / 2; // 3MF/trimesh is Z-up; sit it upright async function renderDocument(container, url, format, token) {
return gltf.scene; if (IMAGE_FORMATS.includes(format)) {
const img = document.createElement('img');
img.className = 'viewer-image';
img.alt = '';
img.src = url;
container.appendChild(img);
} else if (format === 'pdf') {
const frame = document.createElement('iframe');
frame.className = 'viewer-pdf';
frame.title = 'PDF preview';
frame.src = url;
container.appendChild(frame);
} else {
let text;
try {
text = await (await fetchOk(url)).text();
} catch {
text = 'Could not load this file.';
}
if (token !== renderToken) return;
const pre = document.createElement('pre');
pre.className = 'viewer-text';
pre.textContent = text.length > 200000 ? `${text.slice(0, 200000)}\n…` : text;
container.appendChild(pre);
} }
throw new Error(`Unsupported format ${format}`);
} }
async function render(container, url, format, opts = {}) { async function render(container, url, format, opts = {}) {
dispose(); dispose();
const token = ++renderToken;
container.innerHTML = ''; container.innerHTML = '';
if (!SUPPORTED.includes(format)) { if (DOCUMENT_FORMATS.includes(format)) {
await renderDocument(container, url, format, token);
return;
}
if (!MODEL_FORMATS.includes(format)) {
showFallback(container, opts.fallbackImage, `No 3D preview for .${format} files.`); showFallback(container, opts.fallbackImage, `No 3D preview for .${format} files.`);
return; return;
} }
message(container, 'Loading preview…');
let object;
try {
object = await loadObject(url, format, opts);
} catch (err) {
console.warn('Preview failed:', err);
if (token !== renderToken) return;
const note = format === 'glb'
? 'Interactive 3D isn’t available for this 3MF.'
: 'This file couldn’t be rendered in 3D.';
showFallback(container, opts.fallbackImage, note);
return;
}
if (token !== renderToken) return; // another file was opened meanwhile
container.innerHTML = '';
const width = container.clientWidth || 600; const width = container.clientWidth || 600;
const height = container.clientHeight || 420; const height = container.clientHeight || 420;
@ -81,20 +230,6 @@ async function render(container, url, format, opts = {}) {
const controls = new OrbitControls(camera, renderer.domElement); const controls = new OrbitControls(camera, renderer.domElement);
controls.enableDamping = true; controls.enableDamping = true;
let object;
try {
object = await loadObject(url, format);
} catch (err) {
renderer.dispose();
renderer.forceContextLoss();
if (renderer.domElement.parentNode) renderer.domElement.parentNode.removeChild(renderer.domElement);
const note = format === '3mf'
? 'Interactive 3D isn’t supported for this 3MF (Bambu/MakerWorld project file).'
: 'This file couldn’t be rendered in 3D.';
showFallback(container, opts.fallbackImage, note);
return;
}
// Center the object and frame the camera around it. // Center the object and frame the camera around it.
const box = new THREE.Box3().setFromObject(object); const box = new THREE.Box3().setFromObject(object);
const size = box.getSize(new THREE.Vector3()); const size = box.getSize(new THREE.Vector3());
@ -123,8 +258,8 @@ async function render(container, url, format, opts = {}) {
// Render a model once to an offscreen canvas and return a PNG data URL. // Render a model once to an offscreen canvas and return a PNG data URL.
// Used to generate card thumbnails. Transparent background so it composites // Used to generate card thumbnails. Transparent background so it composites
// over the card surface. // over the card surface.
async function snapshot(url, format, size = 512) { async function snapshot(url, format, size = 512, opts = {}) {
if (!SUPPORTED.includes(format)) return null; if (!MODEL_FORMATS.includes(format)) return null;
const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true, preserveDrawingBuffer: true }); const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true, preserveDrawingBuffer: true });
renderer.setSize(size, size); renderer.setSize(size, size);
@ -137,7 +272,7 @@ async function snapshot(url, format, size = 512) {
const fill = new THREE.DirectionalLight(0xffffff, 0.35); fill.position.set(-1, 0.5, -1); scene.add(fill); const fill = new THREE.DirectionalLight(0xffffff, 0.35); fill.position.set(-1, 0.5, -1); scene.add(fill);
try { try {
const object = await loadObject(url, format); const object = await loadObject(url, format, opts);
const box = new THREE.Box3().setFromObject(object); const box = new THREE.Box3().setFromObject(object);
const dim = box.getSize(new THREE.Vector3()); const dim = box.getSize(new THREE.Vector3());
const center = box.getCenter(new THREE.Vector3()); const center = box.getCenter(new THREE.Vector3());
@ -159,4 +294,4 @@ async function snapshot(url, format, size = 512) {
} }
} }
window.ModelViewer = { render, dispose, snapshot, SUPPORTED, RENDERABLE }; window.ModelViewer = { render, dispose, snapshot, MODEL_FORMATS, DOCUMENT_FORMATS, RENDERABLE, PREVIEWABLE };

114
server.js
View file

@ -21,6 +21,7 @@ const auth = require('./lib/auth');
const mailer = require('./lib/mailer'); const mailer = require('./lib/mailer');
const fingerprint = require('./lib/fingerprint'); const fingerprint = require('./lib/fingerprint');
const duplicates = require('./lib/duplicates'); const duplicates = require('./lib/duplicates');
const archives = require('./lib/archives');
const app = express(); const app = express();
const PORT = process.env.PORT || 3000; const PORT = process.env.PORT || 3000;
@ -31,6 +32,9 @@ app.use(express.json({ limit: '8mb' })); // generous for base64 thumbnail PNGs
app.use(cookieParser()); app.use(cookieParser());
app.use(express.static('public')); app.use(express.static('public'));
app.use('/thumbnails', express.static('thumbnails')); app.use('/thumbnails', express.static('thumbnails'));
// In-browser readers for STEP (OpenCascade) and Rhino 3DM, served from node_modules.
app.use('/vendor/occt', express.static(path.join(__dirname, 'node_modules/occt-import-js/dist')));
app.use('/vendor/rhino3dm', express.static(path.join(__dirname, 'node_modules/rhino3dm')));
// Gate the API: everything under /api requires auth except the bootstrap config // Gate the API: everything under /api requires auth except the bootstrap config
// and the auth endpoints themselves. // and the auth endpoints themselves.
@ -57,11 +61,14 @@ function describeFile(filePath, fileName, originalName) {
return { fileName, filePath, originalName, format, size, ...meta }; return { fileName, filePath, originalName, format, size, ...meta };
} }
// Choose a thumbnail for a model from its files (embedded 3MF preview, if any). // Choose a thumbnail for a model from its files: an embedded 3MF preview, else
// a picture that came with the model.
function pickThumbnail(files) { function pickThumbnail(files) {
const threeMf = files.find((f) => f.format === '3mf'); const threeMf = files.find((f) => f.format === '3mf');
if (!threeMf) return null; const embedded = threeMf && fileProcessor.extract3mfThumbnail(threeMf.filePath, path.parse(threeMf.fileName).name);
return fileProcessor.extract3mfThumbnail(threeMf.filePath, path.parse(threeMf.fileName).name); if (embedded) return embedded;
const image = files.find((f) => ['png', 'jpg', 'jpeg', 'gif', 'webp'].includes(f.format));
return image ? fileProcessor.copyAsThumbnail(image.filePath, path.parse(image.fileName).name) : null;
} }
// Transform a stored model into the public shape (thumbnail path -> URL). // Transform a stored model into the public shape (thumbnail path -> URL).
@ -82,7 +89,7 @@ const upload = multer({
storage, storage,
fileFilter: (req, file, cb) => { fileFilter: (req, file, cb) => {
if (fileProcessor.isAllowed(file.originalname)) cb(null, true); if (fileProcessor.isAllowed(file.originalname)) cb(null, true);
else cb(new Error(`Unsupported file type. Allowed: ${fileProcessor.MODEL_FORMATS.join(', ')}`)); else cb(new Error(`Unsupported file type. Allowed: ${fileProcessor.ALLOWED_FORMATS.join(', ')}`));
}, },
}); });
@ -474,6 +481,7 @@ app.delete('/api/trash', async (req, res, next) => {
async function purgeModel(id) { async function purgeModel(id) {
const removed = await store.deleteModel(id); const removed = await store.deleteModel(id);
if (!removed) return; if (!removed) return;
removed.fileIds.forEach(archives.removeExtracted);
for (const p of [...removed.filePaths, ...removed.glbPaths, removed.thumbnailPath]) { for (const p of [...removed.filePaths, ...removed.glbPaths, removed.thumbnailPath]) {
if (!p) continue; if (!p) continue;
try { fs.unlinkSync(p); } catch (err) { try { fs.unlinkSync(p); } catch (err) {
@ -534,46 +542,95 @@ app.get('/api/models/:id/download', async (req, res, next) => {
} catch (e) { next(e); } } catch (e) { next(e); }
}); });
// Serve a raw model file (used by the in-browser 3D viewer). // Send a stored file as a download, or with ?inline=1 for display in the page
// (images, PDFs, text). SVGs are sandboxed so scripts inside them can't run.
function sendStored(req, res, absPath, name) {
res.set('X-Content-Type-Options', 'nosniff');
const ext = path.extname(name).toLowerCase();
if (ext === '.svg') res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; sandbox");
if (req.query.inline !== '1') return res.download(absPath, name);
if (['.md', '.mtl', '.txt'].includes(ext)) res.type('text/plain');
res.sendFile(absPath);
}
// Serve a stored file (downloads, and the in-browser viewers).
app.get('/api/files/:id/raw', async (req, res, next) => { app.get('/api/files/:id/raw', async (req, res, next) => {
try { try {
const { rows } = await db.query('SELECT file_path, original_name FROM files WHERE id = $1', [req.params.id]); const file = await store.getFile(req.params.id);
if (rows.length === 0) return res.status(404).json({ error: 'File not found' }); if (!file) return res.status(404).json({ error: 'File not found' });
const abs = path.resolve(rows[0].file_path); const abs = path.resolve(file.file_path);
if (!fs.existsSync(abs)) return res.status(404).json({ error: 'File missing on disk' }); if (!fs.existsSync(abs)) return res.status(404).json({ error: 'File missing on disk' });
res.download(abs, rows[0].original_name); sendStored(req, res, abs, file.original_name);
} catch (e) { next(e); }
});
// What's inside an archive file (indexed in the background).
app.get('/api/files/:id/entries', async (req, res, next) => {
try {
const file = await store.getFile(req.params.id);
if (!file) return res.status(404).json({ error: 'File not found' });
if (!fileProcessor.ARCHIVE_FORMATS.includes(file.format)) return res.status(400).json({ error: 'Not an archive' });
res.json({
status: file.archive_status || 'pending',
error: file.archive_error,
entries: await store.getArchiveEntries(file.id),
});
} catch (e) { next(e); }
});
// Serve a file from inside an archive (extracted on first use, then cached).
app.get('/api/entries/:id/raw', async (req, res, next) => {
try {
const entry = await store.getArchiveEntry(req.params.id);
if (!entry) return res.status(404).json({ error: 'Entry not found' });
const abs = await archives.extractEntry(entry.archive, entry);
sendStored(req, res, abs, path.posix.basename(entry.path));
} catch (e) { next(e); } } catch (e) { next(e); }
}); });
// Serve a 3MF as GLB, converting on first request (cached thereafter). This lets // Serve a 3MF as GLB, converting on first request (cached thereafter). This lets
// the browser render Bambu/MakerWorld 3MFs and backfills volume/bbox. // the browser render Bambu/MakerWorld 3MFs and backfills volume/bbox.
const glbConversions = new Map(); // fileId -> in-flight Promise const glbConversions = new Map(); // output path -> in-flight Promise
// Converts into a temporary file and renames it into place, so a half-written
// GLB is never served.
function convertOnce(inputPath, outPath) {
if (!glbConversions.has(outPath)) {
fs.mkdirSync(path.dirname(outPath), { recursive: true });
const tmp = `${outPath}.part`;
glbConversions.set(outPath, converter.convert3mfToGlb(inputPath, tmp)
.then((result) => { fs.renameSync(tmp, outPath); return result; })
.finally(() => { fs.rmSync(tmp, { force: true }); glbConversions.delete(outPath); }));
}
return glbConversions.get(outPath);
}
app.get('/api/files/:id/glb', async (req, res, next) => { app.get('/api/files/:id/glb', async (req, res, next) => {
try { try {
const { rows } = await db.query('SELECT id, file_path, format, glb_path FROM files WHERE id = $1', [req.params.id]); const file = await store.getFile(req.params.id);
if (rows.length === 0) return res.status(404).json({ error: 'File not found' }); if (!file) return res.status(404).json({ error: 'File not found' });
const file = rows[0];
if (file.format !== '3mf') return res.status(415).json({ error: 'GLB conversion is only for 3MF files' }); if (file.format !== '3mf') return res.status(415).json({ error: 'GLB conversion is only for 3MF files' });
let glbPath = file.glb_path; let glbPath = file.glb_path;
if (!glbPath || !fs.existsSync(glbPath)) { if (!glbPath || !fs.existsSync(glbPath)) {
if (!glbConversions.has(file.id)) { glbPath = path.join('./glb', `${file.id}.glb`);
glbConversions.set(file.id, convertFileToGlb(file).finally(() => glbConversions.delete(file.id))); const result = await convertOnce(path.resolve(file.file_path), path.resolve(glbPath));
} await store.updateFileConversion(file.id, { glb_path: glbPath, volume: result.volume_cm3, bbox: result.bbox });
glbPath = await glbConversions.get(file.id);
} }
res.sendFile(path.resolve(glbPath)); res.sendFile(path.resolve(glbPath));
} catch (e) { next(e); } } catch (e) { next(e); }
}); });
async function convertFileToGlb(file) { app.get('/api/entries/:id/glb', async (req, res, next) => {
if (!fs.existsSync('./glb')) fs.mkdirSync('./glb', { recursive: true }); try {
const out = path.join('./glb', `${file.id}.glb`); const entry = await store.getArchiveEntry(req.params.id);
const result = await converter.convert3mfToGlb(path.resolve(file.file_path), path.resolve(out)); if (!entry) return res.status(404).json({ error: 'Entry not found' });
await store.updateFileConversion(file.id, { glb_path: out, volume: result.volume_cm3, bbox: result.bbox }); if (entry.format !== '3mf') return res.status(415).json({ error: 'GLB conversion is only for 3MF files' });
return out; const glbPath = path.resolve('./glb', `entry-${entry.id}.glb`);
} if (!fs.existsSync(glbPath)) await convertOnce(await archives.extractEntry(entry.archive, entry), glbPath);
res.sendFile(glbPath);
} catch (e) { next(e); }
});
// ----------------------------------------------------------------- import // ----------------------------------------------------------------- import
// Unified import. Thingiverse downloads files automatically (needs a token); // Unified import. Thingiverse downloads files automatically (needs a token);
@ -874,7 +931,8 @@ app.get('/api/library/stats', async (req, res, next) => {
app.get('/api/duplicates', async (req, res, next) => { app.get('/api/duplicates', async (req, res, next) => {
try { try {
const [groups, pending] = await Promise.all([findDuplicateGroups(), fingerprint.pendingCount()]); const [groups, pending] = await Promise.all([findDuplicateGroups(), fingerprint.pendingCount()]);
const found = await store.getModelsByIds(groups.flatMap((g) => g.modelIds)); const ids = groups.flatMap((g) => g.modelIds);
const [found, counts] = await Promise.all([store.getModelsByIds(ids), store.matchableCounts(ids)]);
const byId = new Map(found.map((m) => [m.id, m])); const byId = new Map(found.map((m) => [m.id, m]));
const result = groups const result = groups
.map((g) => ({ ...g, models: g.modelIds.map((id) => byId.get(id)).filter(Boolean) })) .map((g) => ({ ...g, models: g.modelIds.map((id) => byId.get(id)).filter(Boolean) }))
@ -883,7 +941,11 @@ app.get('/api/duplicates', async (req, res, next) => {
signature: duplicates.signature(g.modelIds), signature: duplicates.signature(g.modelIds),
exact: g.exact, exact: g.exact,
recommendedId: duplicates.recommendKeep(g.models), recommendedId: duplicates.recommendKeep(g.models),
models: g.models.map((m) => ({ ...publicModel(m), matchedFiles: g.matchedFiles[m.id] })), models: g.models.map((m) => ({
...publicModel(m),
matchedFiles: g.matchedFiles[m.id],
matchableFiles: counts[m.id] ?? m.files.length,
})),
})); }));
res.json({ pending, groups: result }); res.json({ pending, groups: result });
} catch (e) { next(e); } } catch (e) { next(e); }

47
test/archives.test.js Normal file
View file

@ -0,0 +1,47 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const crypto = require('crypto');
const fs = require('fs');
const os = require('os');
const path = require('path');
process.env.EXTRACT_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'printhive-extract-'));
const archives = require('../lib/archives');
const fixture = (name) => path.join(__dirname, 'fixtures', name);
const sha = (p) => crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex');
const clean = (p) => p.replace(/^\.\//, '');
// Each pack holds models/cube.stl, models/inner.3mf and readme.txt, plus macOS
// junk (._ files, __MACOSX, .DS_Store); the zip also has a ../ path.
const EXPECTED = ['models/cube.stl', 'models/inner.3mf', 'readme.txt'];
test.after(() => fs.rmSync(process.env.EXTRACT_DIR, { recursive: true, force: true }));
for (const [name, format] of [['pack.zip', 'zip'], ['pack.7z', '7z'], ['pack.tar.gz', 'tgz']]) {
const file = { id: `test-${format}`, file_path: fixture(name), format };
test(`${name}: lists real files, skipping folders, junk and unsafe paths`, async () => {
const entries = await archives.listEntries(file);
assert.deepEqual(entries.map((e) => clean(e.path)).sort(), EXPECTED);
const cube = entries.find((e) => clean(e.path) === 'models/cube.stl');
assert.equal(cube.size, fs.statSync(fixture('cube.stl')).size);
});
test(`${name}: extracts one entry byte-for-byte, and caches it`, async () => {
const entries = await archives.listEntries(file);
const cube = { ...entries.find((e) => clean(e.path) === 'models/cube.stl'), id: 1 };
const abs = await archives.extractEntry(file, cube);
assert.equal(sha(abs), sha(fixture('cube.stl')));
assert.equal(await archives.extractEntry(file, cube), abs);
});
}
test('a missing entry fails clearly instead of returning a path', async () => {
const file = { id: 'test-missing', file_path: fixture('pack.zip'), format: 'zip' };
await assert.rejects(archives.extractEntry(file, { id: 2, path: 'models/nope.stl' }), /Could not extract/);
});
test('an unreadable archive rejects', async () => {
await assert.rejects(archives.listEntries({ file_path: fixture('cube.stl'), format: 'zip' }));
});

BIN
test/fixtures/pack.7z vendored Normal file

Binary file not shown.

BIN
test/fixtures/pack.tar.gz vendored Normal file

Binary file not shown.

BIN
test/fixtures/pack.zip vendored Normal file

Binary file not shown.