From d4b7e29348bb4b7bc784d17ab48b73f53c44e12a Mon Sep 17 00:00:00 2001 From: dlawler489 <104159223@student.swin.edu.au> Date: Tue, 15 Sep 2026 07:36:47 +1000 Subject: [PATCH] Browse archives, preview companion files, and view FBX/STEP/3DM/OBJ+MTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Archives - ZIP, 7Z, RAR and TAR/.tar.gz uploads are indexed in the background into archive_entries (7-Zip compiled to WASM in a worker thread; node-tar for tar). macOS/Windows clutter and paths that could escape the extraction folder are skipped. - Models inside archives are extracted once per archive to hash and geometry-fingerprint them, so duplicates are found inside packs too. Pictures, PDFs and readmes are listed but never matched. - New routes list an archive's entries and serve/convert a single entry, extracted on first use and cached under extracted/. Viewer - Model detail lists what's inside each archive with view/download links; opening an entry adds a viewer tab. - New 3D formats: FBX (with sibling textures), OBJ with its MTL and textures, STEP/STP via occt-import-js, and Rhino 3DM via rhino3dm — the WASM readers are served from node_modules, loaded on first use. - Pictures, PDFs and text files (jpg/png/gif/webp/bmp/svg, pdf, txt/md) can be uploaded and preview in the viewer; a model without an embedded 3MF preview uses its first picture as the card thumbnail. - Inline file responses send nosniff, and SVGs are sandboxed. - GLB conversions write to a temp file and rename, so a half-written preview is never served; a later tab can't be overdrawn by a slow load. - Better 3MF card image: the project thumbnail or plate 1 render, not a slicer pick map. Co-Authored-By: Claude Opus 5 --- .dockerignore | 1 + .gitignore | 1 + db/schema.sql | 21 ++++ lib/archiveWorker.js | 82 +++++++++++++++ lib/archives.js | 133 ++++++++++++++++++++++++ lib/fileProcessor.js | 52 ++++++++-- lib/fingerprint.js | 177 ++++++++++++++++++++++++------- lib/store.js | 81 +++++++++++++-- package-lock.json | 109 ++++++++++++++++++- package.json | 6 +- public/app.js | 145 ++++++++++++++++++++------ public/index.html | 6 +- public/styles.css | 14 +++ public/viewer.js | 213 +++++++++++++++++++++++++++++++------- server.js | 114 +++++++++++++++----- test/archives.test.js | 47 +++++++++ test/fixtures/pack.7z | Bin 0 -> 1608 bytes test/fixtures/pack.tar.gz | Bin 0 -> 1779 bytes test/fixtures/pack.zip | Bin 0 -> 2211 bytes 19 files changed, 1045 insertions(+), 157 deletions(-) create mode 100644 lib/archiveWorker.js create mode 100644 lib/archives.js create mode 100644 test/archives.test.js create mode 100644 test/fixtures/pack.7z create mode 100644 test/fixtures/pack.tar.gz create mode 100644 test/fixtures/pack.zip diff --git a/.dockerignore b/.dockerignore index ee63c18..51ef26e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,4 @@ npm-debug.log* Dockerfile .dockerignore docker-compose.yml +extracted diff --git a/.gitignore b/.gitignore index 3deaf18..fa332d7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ thumbnails/ glb/ tools/venv/ __pycache__/ +extracted/ diff --git a/db/schema.sql b/db/schema.sql index c14c9d8..db464c4 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -124,6 +124,27 @@ ALTER TABLE models ADD COLUMN IF NOT EXISTS notes TEXT NOT NULL DEFAULT ''; ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ; CREATE INDEX IF NOT EXISTS idx_models_deleted ON models(deleted_at); +-- Archive files (zip/7z/rar/tar) are indexed in the background: +-- null = not yet, ok | error (with archive_error). +ALTER TABLE files ADD COLUMN IF NOT EXISTS archive_status TEXT; +ALTER TABLE files ADD COLUMN IF NOT EXISTS archive_error TEXT; + +-- Files inside archives, so they can be browsed, previewed and matched as duplicates. +CREATE TABLE IF NOT EXISTS archive_entries ( + id SERIAL PRIMARY KEY, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + path TEXT NOT NULL, -- path inside the archive, as the archive stores it + format TEXT NOT NULL, + size BIGINT NOT NULL DEFAULT 0, + sha256 TEXT, + geometry_hash TEXT, + geometry_status TEXT, -- same states as files.geometry_status + fingerprinted_at TIMESTAMPTZ, + UNIQUE (file_id, path) +); +CREATE INDEX IF NOT EXISTS idx_entries_sha256 ON archive_entries(sha256); +CREATE INDEX IF NOT EXISTS idx_entries_geometry ON archive_entries(geometry_hash); + -- Duplicate groups marked "not duplicates" (signature = sorted model ids). CREATE TABLE IF NOT EXISTS duplicate_dismissals ( signature TEXT PRIMARY KEY, diff --git a/lib/archiveWorker.js b/lib/archiveWorker.js new file mode 100644 index 0000000..a8da4e5 --- /dev/null +++ b/lib/archiveWorker.js @@ -0,0 +1,82 @@ +// Worker thread running 7-Zip (compiled to WASM) for ZIP/7Z/RAR archives. +// 7-Zip calls are synchronous, so they run here to keep the server responsive. + +const { parentPort } = require('worker_threads'); +const SevenZip = require('7z-wasm'); + +let sevenZip = null; +const output = []; + +// The host filesystem is mounted at /host inside the WASM filesystem. +const host = (p) => `/host${p}`; + +async function instance() { + if (!sevenZip) { + sevenZip = await SevenZip({ print: (l) => output.push(l), printErr: (l) => output.push(l) }); + sevenZip.FS.mkdir('/host'); + sevenZip.FS.mount(sevenZip.NODEFS, { root: '/' }, '/host'); + } + return sevenZip; +} + +async function run(args) { + const sz = await instance(); + output.length = 0; + let code; + try { + code = sz.callMain(args); + } catch (err) { + code = err && typeof err.status === 'number' ? err.status : 2; + } + return { code, lines: output.slice() }; +} + +function errorText(lines) { + const line = lines.find((l) => /error|wrong password|can ?not open|unsupported|headers/i.test(l)); + return line ? line.trim() : null; +} + +// Parses `7zz l -slt` into [{ path, size }] for files (folders are dropped). +function parseListing(lines) { + const start = lines.findIndex((l) => l.startsWith('----------')); + if (start === -1) return null; + const entries = []; + let cur = {}; + const flush = () => { + const isFolder = cur.Folder === '+' || (cur.Attributes || '').startsWith('D'); + if (cur.Path !== undefined && !isFolder) entries.push({ path: cur.Path, size: Number(cur.Size) || 0 }); + cur = {}; + }; + for (const line of lines.slice(start + 1)) { + if (!line.trim()) { flush(); continue; } + const i = line.indexOf(' = '); + if (i > 0) cur[line.slice(0, i)] = line.slice(i + 3); + } + flush(); + return entries; +} + +async function list(archive) { + // A dummy password makes encrypted archives fail fast instead of prompting. + const { code, lines } = await run(['l', '-slt', '-pprinthive', host(archive)]); + const entries = parseListing(lines); + if (code !== 0 || !entries) throw new Error(errorText(lines) || `7-Zip could not read the archive (exit ${code})`); + return entries; +} + +// Extracts the named entries (keeping their folders) under destDir. Callers +// check which files actually arrived. +async function extract(archive, entries, destDir) { + const { code, lines } = await run( + ['x', host(archive), `-o${host(destDir)}`, '-y', '-spd', '-bsp0', '-pprinthive', '--', ...entries]); + return { code, error: code === 0 ? null : errorText(lines) }; +} + +parentPort.on('message', async ({ id, op, archive, entries, destDir }) => { + try { + const result = op === 'list' ? await list(archive) : await extract(archive, entries, destDir); + parentPort.postMessage({ id, result }); + } catch (err) { + parentPort.postMessage({ id, error: err.message }); + } +}); diff --git a/lib/archives.js b/lib/archives.js new file mode 100644 index 0000000..0ec1fc4 --- /dev/null +++ b/lib/archives.js @@ -0,0 +1,133 @@ +// Lists and extracts files inside archives. ZIP/7Z/RAR go through 7-Zip (WASM, +// in a worker thread); TAR and .tar.gz go through node-tar. Extracted entries +// are cached under EXTRACT_DIR// and can always be re-extracted. + +const fs = require('fs'); +const path = require('path'); +const { Worker } = require('worker_threads'); +const tar = require('tar'); + +const EXTRACT_DIR = path.resolve(process.env.EXTRACT_DIR || './extracted'); +const TAR_FILE_TYPES = ['File', 'OldFile', 'ContiguousFile']; + +let worker = null; +let nextId = 1; +const pending = new Map(); + +function failPending(err) { + for (const p of pending.values()) p.reject(err); + pending.clear(); + worker = null; +} + +// The worker keeps the process alive only while a request is outstanding, so +// an idle worker never stops Node from exiting. +function sevenZip(message) { + if (!worker) { + worker = new Worker(path.join(__dirname, 'archiveWorker.js')); + worker.on('message', ({ id, result, error }) => { + const p = pending.get(id); + pending.delete(id); + if (pending.size === 0 && worker) worker.unref(); + if (p) error ? p.reject(new Error(error)) : p.resolve(result); + }); + worker.on('error', failPending); + worker.on('exit', () => failPending(new Error('Archive worker stopped'))); + } + return new Promise((resolve, reject) => { + const id = nextId++; + pending.set(id, { resolve, reject }); + worker.ref(); + worker.postMessage({ id, ...message }); + }); +} + +const isTar = (format) => format === 'tar' || format === 'tgz'; +const clean = (p) => p.replace(/^\.\//, ''); + +// Clutter archivers add (macOS resource forks, Finder/Explorer files). +function isJunk(entryPath) { + const parts = clean(entryPath).split('/'); + const name = parts[parts.length - 1]; + return parts.includes('__MACOSX') || name.startsWith('._') || name === '.DS_Store' || name === 'Thumbs.db'; +} + +// Paths that could land outside the extraction folder are never used. +function isSafe(entryPath) { + return Boolean(entryPath) && !path.isAbsolute(entryPath) && !entryPath.split(/[\\/]/).includes('..'); +} + +// file: { file_path, format }. Returns [{ path, size }] of real files. +async function listEntries(file) { + const abs = path.resolve(file.file_path); + let raw; + if (isTar(file.format)) { + raw = []; + await tar.t({ + file: abs, + onReadEntry: (e) => { if (TAR_FILE_TYPES.includes(e.type)) raw.push({ path: e.path, size: e.size }); }, + }); + } else { + raw = await sevenZip({ op: 'list', archive: abs }); + } + return raw.filter((e) => isSafe(e.path) && !isJunk(e.path)); +} + +function tempDir() { + fs.mkdirSync(EXTRACT_DIR, { recursive: true }); + return fs.mkdtempSync(path.join(EXTRACT_DIR, '.tmp-')); +} + +// Extracts entries into destDir. Returns Map(entry path -> absolute path) for +// the ones that came out; archive-level failures throw. +async function extractMany(file, entryPaths, destDir) { + const abs = path.resolve(file.file_path); + const wanted = new Set(entryPaths.filter(isSafe)); + if (isTar(file.format)) { + await tar.x({ file: abs, cwd: destDir, filter: (p) => wanted.has(p) }); + } else { + const { code, error } = await sevenZip({ op: 'extract', archive: abs, entries: [...wanted], destDir }); + if (code !== 0 && error) console.warn(`7-Zip reported a problem extracting from ${abs}: ${error}`); + } + const out = new Map(); + for (const p of wanted) { + const target = path.resolve(destDir, p); + if (target.startsWith(destDir + path.sep) && fs.existsSync(target)) out.set(p, target); + } + return out; +} + +function entryCachePath(fileId, entry) { + return path.join(EXTRACT_DIR, String(fileId), `${entry.id}${path.extname(entry.path).toLowerCase()}`); +} + +const inflight = new Map(); + +// file: { id, file_path, format }; entry: { id, path }. Returns the absolute +// path of a cached copy, extracting it on first use. +async function extractEntry(file, entry) { + const dest = entryCachePath(file.id, entry); + if (fs.existsSync(dest)) return dest; + if (!inflight.has(dest)) { + const job = (async () => { + fs.mkdirSync(path.dirname(dest), { recursive: true }); + const tmp = tempDir(); + try { + const got = (await extractMany(file, [entry.path], tmp)).get(entry.path); + if (!got) throw new Error(`Could not extract ${clean(entry.path)} from the archive`); + fs.renameSync(got, dest); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + })().finally(() => inflight.delete(dest)); + inflight.set(dest, job); + } + await inflight.get(dest); + return dest; +} + +function removeExtracted(fileId) { + fs.rmSync(path.join(EXTRACT_DIR, String(fileId)), { recursive: true, force: true }); +} + +module.exports = { listEntries, extractMany, extractEntry, removeExtracted, tempDir, isJunk }; diff --git a/lib/fileProcessor.js b/lib/fileProcessor.js index 2e97843..2da16f9 100644 --- a/lib/fileProcessor.js +++ b/lib/fileProcessor.js @@ -5,18 +5,25 @@ const AdmZip = require('adm-zip'); const THUMBNAIL_DIR = './thumbnails'; -const MODEL_FORMATS = ['stl', '3mf', 'obj', 'step', 'stp', 'gcode', 'gco', 'zip']; +const MODEL_FORMATS = ['stl', '3mf', 'obj', 'fbx', 'step', 'stp', '3dm', 'gcode', 'gco']; +const ARCHIVE_FORMATS = ['zip', '7z', 'rar', 'tar', 'tgz']; +const IMAGE_FORMATS = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'svg']; +// Kept with a model and viewable, but never matched as duplicates. +const COMPANION_FORMATS = [...IMAGE_FORMATS, 'pdf', 'txt', 'md', 'mtl']; +const ALLOWED_FORMATS = [...MODEL_FORMATS, ...ARCHIVE_FORMATS, ...COMPANION_FORMATS]; function ensureThumbnailDir() { if (!fs.existsSync(THUMBNAIL_DIR)) fs.mkdirSync(THUMBNAIL_DIR, { recursive: true }); } function formatOf(filename) { - return path.extname(filename).slice(1).toLowerCase(); + const lower = filename.toLowerCase(); + if (lower.endsWith('.tar.gz')) return 'tgz'; + return path.extname(lower).slice(1); } function isAllowed(filename) { - return MODEL_FORMATS.includes(formatOf(filename)); + return ALLOWED_FORMATS.includes(formatOf(filename)); } // Geometry metadata. Only STL is computed natively today; others return nulls @@ -37,16 +44,28 @@ function extractMetadata(filePath, format) { } } +// Which embedded PNG makes the best card image: the project thumbnail, then the +// first plate render. Slicer pick maps, top views and small/unlit variants last. +function thumbnailRank(name) { + if (name === 'metadata/thumbnail.png') return 0; + if (/metadata\/plate_1\.png$/.test(name)) return 1; + if (/metadata\/plate_\d+\.png$/.test(name)) return 2; + if (/(pick_|top_|no_light|_small)/.test(name)) return 9; + return 5; +} + // A .3mf is a zip; slicers embed a PNG preview under /Metadata/. If present, // pull it out as the model thumbnail. Returns a path under thumbnails/ or null. function extract3mfThumbnail(filePath, baseName) { try { ensureThumbnailDir(); const zip = new AdmZip(filePath); - const entry = zip.getEntries().find((e) => { - const n = e.entryName.toLowerCase(); - return n.endsWith('.png') && (n.includes('thumbnail') || n.includes('/metadata/')); - }); + const entry = zip.getEntries() + .filter((e) => { + const n = e.entryName.toLowerCase(); + return n.endsWith('.png') && (n.includes('thumbnail') || n.includes('metadata/')); + }) + .sort((a, b) => thumbnailRank(a.entryName.toLowerCase()) - thumbnailRank(b.entryName.toLowerCase()))[0]; if (!entry) return null; const out = path.join(THUMBNAIL_DIR, `${baseName}.png`); fs.writeFileSync(out, entry.getData()); @@ -57,4 +76,21 @@ function extract3mfThumbnail(filePath, baseName) { } } -module.exports = { MODEL_FORMATS, formatOf, isAllowed, extractMetadata, extract3mfThumbnail }; +// Copy a picture that came with a model into thumbnails/ (a copy, so replacing +// the thumbnail later never deletes the model's own file). +function copyAsThumbnail(filePath, baseName) { + try { + ensureThumbnailDir(); + const out = path.join(THUMBNAIL_DIR, `${baseName}-image${path.extname(filePath).toLowerCase()}`); + fs.copyFileSync(filePath, out); + return out; + } catch (err) { + console.error('Image thumbnail copy failed:', err.message); + return null; + } +} + +module.exports = { + MODEL_FORMATS, ARCHIVE_FORMATS, IMAGE_FORMATS, COMPANION_FORMATS, ALLOWED_FORMATS, + formatOf, isAllowed, extractMetadata, extract3mfThumbnail, copyAsThumbnail, +}; diff --git a/lib/fingerprint.js b/lib/fingerprint.js index d4031e4..3f936db 100644 --- a/lib/fingerprint.js +++ b/lib/fingerprint.js @@ -1,16 +1,21 @@ -// Background worker that fingerprints stored files for the duplicate finder: a -// SHA-256 of the bytes for every file, plus a geometry fingerprint for mesh +// Background worker that indexes and fingerprints stored files for the +// duplicate finder: a SHA-256 of the bytes, plus a geometry fingerprint for mesh // formats so the same model still matches after being renamed, re-exported -// (ASCII/binary STL) or converted (STL -> 3MF). +// (ASCII/binary STL) or converted (STL -> 3MF). Archives are listed, and the +// models inside them are fingerprinted too. const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const db = require('./db'); +const store = require('./store'); +const archives = require('./archives'); +const { MODEL_FORMATS, ARCHIVE_FORMATS, formatOf } = require('./fileProcessor'); const { PYTHON, runScript } = require('./python'); const GEOMETRY_FORMATS = ['stl', 'obj', '3mf']; const BATCH = 10; +const TABLES = ['files', 'archive_entries']; let pythonAvailable = true; let running = false; @@ -58,64 +63,153 @@ async function measure(paths) { return results; } -async function fingerprintGeometry(rows) { - const abs = (row) => path.resolve(row.file_path); +// rows: [{ id, format, abs }] from `table` ('files' or 'archive_entries'). +async function fingerprintGeometry(rows, table) { + if (!TABLES.includes(table)) throw new Error(`Unknown table ${table}`); // 3MF projects can be big, so each gets its own run; that keeps a queued // preview from waiting behind a whole batch of them. const chunks = [rows.filter((r) => r.format !== '3mf'), ...rows.filter((r) => r.format === '3mf').map((r) => [r])]; for (const chunk of chunks.filter((c) => c.length)) { - const results = await measure(chunk.map(abs)); + const results = await measure(chunk.map((r) => r.abs)); for (const row of chunk) { - let r = results.get(abs(row)); + let r = results.get(row.abs); // A batch that died part-way: retry the missing file on its own. - if (!r && chunk.length > 1) r = (await measure([abs(row)])).get(abs(row)); + if (!r && chunk.length > 1) r = (await measure([row.abs])).get(row.abs); const key = r && r.fp ? geometryKey(r.fp) : null; - await db.query('UPDATE files SET geometry_hash = $2, geometry_status = $3 WHERE id = $1', + await db.query(`UPDATE ${table} SET geometry_hash = $2, geometry_status = $3 WHERE id = $1`, [row.id, key, key ? 'ok' : 'error']); } } } -async function run() { - for (;;) { - const { rows } = await db.query( - `SELECT id, file_path, format, fingerprinted_at FROM files - WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean) - ORDER BY id LIMIT $2`, [pythonAvailable, BATCH]); - if (rows.length === 0) return; +// Runs fn; if Python turns out to be missing, geometry fingerprints are switched +// off for this process (content hashes still run). +async function withPython(fn) { + try { + await fn(); + } catch (err) { + if (err.code !== 'ENOENT') throw err; + pythonAvailable = false; + console.warn(`Geometry fingerprints disabled: no Python at ${PYTHON} (content hashes still run).`); + } +} +async function fingerprintFiles() { + const { rows } = await db.query( + `SELECT id, file_path, format, fingerprinted_at FROM files + WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean) + ORDER BY id LIMIT $2`, [pythonAvailable, BATCH]); + if (rows.length === 0) return false; + + const geometry = []; + for (const row of rows) { + const abs = path.resolve(row.file_path); + try { + if (!fs.existsSync(abs)) { + await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'missing' WHERE id = $1`, [row.id]); + continue; + } + if (!row.fingerprinted_at) { + await db.query('UPDATE files SET sha256 = $2, fingerprinted_at = now() WHERE id = $1', + [row.id, await sha256File(abs)]); + } + if (!GEOMETRY_FORMATS.includes(row.format)) { + await db.query(`UPDATE files SET geometry_status = 'na' WHERE id = $1`, [row.id]); + } else if (pythonAvailable) { + geometry.push({ id: row.id, format: row.format, abs }); + } + } catch (err) { + console.warn(`Fingerprint failed for file ${row.id}:`, err.message); + await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [row.id]); + } + } + if (geometry.length) await withPython(() => fingerprintGeometry(geometry, 'files')); + return true; +} + +async function indexArchives() { + const { rows } = await db.query( + `SELECT id, file_path, format FROM files + WHERE format = ANY($1) AND archive_status IS NULL ORDER BY id LIMIT 5`, [ARCHIVE_FORMATS]); + for (const file of rows) { + try { + const entries = await archives.listEntries(file); + await store.replaceArchiveEntries(file.id, + entries.map((e) => ({ path: e.path, size: e.size, format: formatOf(e.path) }))); + await db.query(`UPDATE files SET archive_status = 'ok', archive_error = NULL WHERE id = $1`, [file.id]); + } catch (err) { + console.warn(`Could not read archive (file ${file.id}):`, err.message); + await db.query(`UPDATE files SET archive_status = 'error', archive_error = $2 WHERE id = $1`, + [file.id, err.message.slice(0, 300)]); + } + } + return rows.length > 0; +} + +const ENTRY_PENDING = 'e.fingerprinted_at IS NULL OR (e.geometry_status IS NULL AND $1::boolean)'; + +// Fingerprints the pending entries of one archive, extracting its models in a +// single pass (cheaper than one extraction per entry, especially for solid 7z). +async function fingerprintEntries() { + const { rows: [file] } = await db.query( + `SELECT f.id, f.file_path, f.format FROM files f + WHERE EXISTS (SELECT 1 FROM archive_entries e WHERE e.file_id = f.id AND (${ENTRY_PENDING})) + ORDER BY f.id LIMIT 1`, [pythonAvailable]); + if (!file) return false; + const { rows: entries } = await db.query( + `SELECT e.id, e.path, e.format, e.fingerprinted_at FROM archive_entries e + WHERE e.file_id = $2 AND (${ENTRY_PENDING}) ORDER BY e.id LIMIT 500`, [pythonAvailable, file.id]); + + // Pictures, documents and nested archives are listed but not matched as duplicates. + const others = entries.filter((e) => !MODEL_FORMATS.includes(e.format)); + if (others.length) { + await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'na' WHERE id = ANY($1::int[])`, + [others.map((e) => e.id)]); + } + const models = entries.filter((e) => MODEL_FORMATS.includes(e.format)); + if (!models.length) return true; + + const tmp = archives.tempDir(); + try { + let extracted = new Map(); + try { + extracted = await archives.extractMany(file, models.map((e) => e.path), tmp); + } catch (err) { + console.warn(`Could not extract from archive (file ${file.id}):`, err.message); + } const geometry = []; - for (const row of rows) { - const abs = path.resolve(row.file_path); + for (const entry of models) { + const abs = extracted.get(entry.path); try { - if (!fs.existsSync(abs)) { - await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'missing' WHERE id = $1`, [row.id]); + if (!abs) { + await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [entry.id]); continue; } - if (!row.fingerprinted_at) { - await db.query('UPDATE files SET sha256 = $2, fingerprinted_at = now() WHERE id = $1', - [row.id, await sha256File(abs)]); + if (!entry.fingerprinted_at) { + await db.query('UPDATE archive_entries SET sha256 = $2, fingerprinted_at = now() WHERE id = $1', + [entry.id, await sha256File(abs)]); } - if (!GEOMETRY_FORMATS.includes(row.format)) { - await db.query(`UPDATE files SET geometry_status = 'na' WHERE id = $1`, [row.id]); + if (!GEOMETRY_FORMATS.includes(entry.format)) { + await db.query(`UPDATE archive_entries SET geometry_status = 'na' WHERE id = $1`, [entry.id]); } else if (pythonAvailable) { - geometry.push(row); + geometry.push({ id: entry.id, format: entry.format, abs }); } } catch (err) { - console.warn(`Fingerprint failed for file ${row.id}:`, err.message); - await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [row.id]); + console.warn(`Fingerprint failed for archive entry ${entry.id}:`, err.message); + await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [entry.id]); } } + if (geometry.length) await withPython(() => fingerprintGeometry(geometry, 'archive_entries')); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + return true; +} - if (geometry.length) { - try { - await fingerprintGeometry(geometry); - } catch (err) { - if (err.code !== 'ENOENT') throw err; - pythonAvailable = false; - console.warn(`Geometry fingerprints disabled: no Python at ${PYTHON} (content hashes still run).`); - } - } +async function run() { + for (;;) { + const worked = [await fingerprintFiles(), await indexArchives(), await fingerprintEntries()]; + if (!worked.some(Boolean)) return; } } @@ -133,9 +227,12 @@ function kick() { async function pendingCount() { const { rows } = await db.query( - `SELECT COUNT(*)::int AS n FROM files - WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean)`, [pythonAvailable]); - return rows[0].n; + `SELECT (SELECT COUNT(*) FROM files + WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean)) + + (SELECT COUNT(*) FROM files WHERE format = ANY($2) AND archive_status IS NULL) + + (SELECT COUNT(*) FROM archive_entries e WHERE ${ENTRY_PENDING}) AS n`, + [pythonAvailable, ARCHIVE_FORMATS]); + return Number(rows[0].n); } module.exports = { kick, pendingCount, geometryKey, sha256File }; diff --git a/lib/store.js b/lib/store.js index 4144118..a18aa21 100644 --- a/lib/store.js +++ b/lib/store.js @@ -1,4 +1,5 @@ const db = require('./db'); +const { MODEL_FORMATS, ARCHIVE_FORMATS } = require('./fileProcessor'); // ---- shared SELECT that hydrates a model with tags, projects, files, designer ---- const MODEL_SELECT = ` @@ -25,7 +26,8 @@ const MODEL_SELECT = ` SELECT json_agg(json_build_object( 'id', fl.id, 'originalName', fl.original_name, 'fileName', fl.file_name, 'format', fl.format, 'size', fl.size, 'volume', fl.volume, - 'bbox', fl.bbox, 'facets', fl.facets) ORDER BY fl.id) AS files + 'bbox', fl.bbox, 'facets', fl.facets, 'archiveStatus', fl.archive_status, + 'entryCount', (SELECT COUNT(*)::int FROM archive_entries ae WHERE ae.file_id = fl.id)) ORDER BY fl.id) AS files FROM files fl WHERE fl.model_id = m.id ) f ON true `; @@ -116,17 +118,60 @@ async function updateModel(id, updates) { // Returns the on-disk file rows so the caller can unlink them. async function deleteModel(id) { - const { rows } = await db.query('SELECT file_path, glb_path FROM files WHERE model_id = $1', [id]); + const { rows } = await db.query('SELECT id, file_path, glb_path FROM files WHERE model_id = $1', [id]); + const entries = await db.query( + 'SELECT e.id FROM archive_entries e JOIN files f ON f.id = e.file_id WHERE f.model_id = $1', [id]); const model = await db.query('SELECT thumbnail_path FROM models WHERE id = $1', [id]); const result = await db.query('DELETE FROM models WHERE id = $1 RETURNING id', [id]); if (result.rowCount === 0) return null; return { + fileIds: rows.map((r) => r.id), filePaths: rows.map((r) => r.file_path), - glbPaths: rows.map((r) => r.glb_path).filter(Boolean), + glbPaths: [ + ...rows.map((r) => r.glb_path).filter(Boolean), + ...entries.rows.map((e) => `glb/entry-${e.id}.glb`), + ], thumbnailPath: model.rows[0] && model.rows[0].thumbnail_path, }; } +// ---------------------------------------------------------------- files / archive entries +async function getFile(id) { + const { rows } = await db.query('SELECT * FROM files WHERE id = $1', [id]); + return rows[0] || null; +} + +async function getArchiveEntries(fileId) { + const { rows } = await db.query( + 'SELECT id, path, format, size FROM archive_entries WHERE file_id = $1 ORDER BY path', [fileId]); + return rows.map((r) => ({ ...r, size: Number(r.size) })); +} + +// An entry plus the archive it lives in ({ id, file_path, format }). +async function getArchiveEntry(id) { + const { rows } = await db.query( + `SELECT e.id, e.file_id, e.path, e.format, e.size, f.file_path, f.format AS archive_format + FROM archive_entries e JOIN files f ON f.id = e.file_id WHERE e.id = $1`, [id]); + const r = rows[0]; + if (!r) return null; + return { + id: r.id, file_id: r.file_id, path: r.path, format: r.format, size: Number(r.size), + archive: { id: r.file_id, file_path: r.file_path, format: r.archive_format }, + }; +} + +async function replaceArchiveEntries(fileId, entries) { + await db.transaction(async (client) => { + await client.query('DELETE FROM archive_entries WHERE file_id = $1', [fileId]); + if (!entries.length) return; + await client.query( + `INSERT INTO archive_entries (file_id, path, format, size) + SELECT $1::int, * FROM unnest($2::text[], $3::text[], $4::bigint[]) + ON CONFLICT (file_id, path) DO NOTHING`, + [fileId, entries.map((e) => e.path), entries.map((e) => e.format), entries.map((e) => e.size)]); + }); +} + // ---------------------------------------------------------------- trash async function trashModels(ids) { const { rowCount } = await db.query( @@ -153,21 +198,42 @@ async function countTrashed() { // ---------------------------------------------------------------- duplicates // Files of non-trashed models whose content hash or geometry fingerprint also // appears in another non-trashed model — the input to the duplicate finder. +// Ids are prefixed ('f12' for a file, 'e7' for an archive entry) so both kinds +// can be matched together. Pictures and documents never count. async function duplicateCandidateFiles() { const { rows } = await db.query(` WITH live AS ( - SELECT f.id, f.model_id, f.sha256, f.geometry_hash + SELECT 'f' || f.id AS id, f.model_id, f.sha256, f.geometry_hash FROM files f JOIN models m ON m.id = f.model_id - WHERE m.deleted_at IS NULL AND f.size > 0 + WHERE m.deleted_at IS NULL AND f.size > 0 AND f.format = ANY($1) + UNION ALL + SELECT 'e' || e.id, f.model_id, e.sha256, e.geometry_hash + FROM archive_entries e JOIN files f ON f.id = e.file_id JOIN models m ON m.id = f.model_id + WHERE m.deleted_at IS NULL AND e.size > 0 AND e.format = ANY($2) ) SELECT * FROM live WHERE sha256 IN (SELECT sha256 FROM live WHERE sha256 IS NOT NULL GROUP BY sha256 HAVING COUNT(DISTINCT model_id) > 1) OR geometry_hash IN (SELECT geometry_hash FROM live WHERE geometry_hash IS NOT NULL - GROUP BY geometry_hash HAVING COUNT(DISTINCT model_id) > 1)`); + GROUP BY geometry_hash HAVING COUNT(DISTINCT model_id) > 1)`, + [[...MODEL_FORMATS, ...ARCHIVE_FORMATS], MODEL_FORMATS]); return rows; } +// How many duplicate-matchable items (model files, archives, models inside +// archives) each model holds: { modelId: n }. +async function matchableCounts(modelIds) { + if (!modelIds.length) return {}; + const { rows } = await db.query(` + SELECT m.id, + (SELECT COUNT(*) FROM files f WHERE f.model_id = m.id AND f.format = ANY($2)) + + (SELECT COUNT(*) FROM archive_entries e JOIN files f ON f.id = e.file_id + WHERE f.model_id = m.id AND e.format = ANY($3)) AS n + FROM models m WHERE m.id = ANY($1::int[])`, + [modelIds, [...MODEL_FORMATS, ...ARCHIVE_FORMATS], MODEL_FORMATS]); + return Object.fromEntries(rows.map((r) => [r.id, Number(r.n)])); +} + // Keep one model of a duplicate group: fold the others' tags, collections and // notes into it, then move the others to the trash. async function mergeAndTrash(keepId, otherIds) { @@ -407,7 +473,8 @@ module.exports = { getAllModels, getModel, getModelsByIds, findModelBySource, createModel, updateModel, deleteModel, addFiles, updateFileConversion, trashModels, restoreModel, trashedModelIds, countTrashed, - duplicateCandidateFiles, mergeAndTrash, dismissDuplicateGroup, dismissedDuplicateSignatures, + duplicateCandidateFiles, matchableCounts, mergeAndTrash, dismissDuplicateGroup, dismissedDuplicateSignatures, + getFile, getArchiveEntries, getArchiveEntry, replaceArchiveEntries, getAllTags, getAllProjects, getProject, createProject, updateProject, deleteProject, getSetting, setSetting, diff --git a/package-lock.json b/package-lock.json index f1e5660..c829c8d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "7z-wasm": "^1.2.0", "adm-zip": "^0.5.17", "bcryptjs": "^3.0.3", "cookie-parser": "^1.4.7", @@ -18,12 +19,42 @@ "multer": "^1.4.5-lts.1", "node-stl": "^0.7.0", "nodemailer": "^9.0.1", - "pg": "^8.22.0" + "occt-import-js": "^0.0.23", + "pg": "^8.22.0", + "rhino3dm": "^8.32.2", + "tar": "^7.5.22" }, "devDependencies": { "nodemon": "^3.0.2" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/7z-wasm": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/7z-wasm/-/7z-wasm-1.2.0.tgz", + "integrity": "sha512-AZjtWXFleAe5nQ27JYISs70cICjO0zZr4LU45I9pXKcrhG0qiEWgq3Vjwr9JHHihFLuqgc7uSHYVn/z3vXCjdw==", + "license": "SEE LICENSE IN License.txt", + "dependencies": { + "readline-sync": "^1.4.10" + }, + "bin": { + "7z-wasm": "bin/cli" + }, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -240,6 +271,15 @@ "fsevents": "~2.3.2" } }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/concat-stream": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", @@ -945,6 +985,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/mkdirp": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", @@ -1091,6 +1152,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/occt-import-js": { + "version": "0.0.23", + "resolved": "https://registry.npmjs.org/occt-import-js/-/occt-import-js-0.0.23.tgz", + "integrity": "sha512-RFfYQXYFX5C1mB1Aywm0ShcUKzXOr/VzTnlzhBSDJOR6YCAPt1HYCzeXWg1vwwjn/cUxwqRNhhtf1dlewoZYCQ==", + "license": "LGPL-2.1" + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -1358,6 +1425,21 @@ "node": ">=8.10.0" } }, + "node_modules/readline-sync": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz", + "integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==", + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/rhino3dm": { + "version": "8.32.2", + "resolved": "https://registry.npmjs.org/rhino3dm/-/rhino3dm-8.32.2.tgz", + "integrity": "sha512-Q3/5iHIKJeoE5doXrRy7C1vJer4F8uMYe8K6B8VQHqhGj5BCS1L3EkqNpKyUTERoHyKh7rMmiiZpw9iEUkCnPQ==", + "license": "MIT" + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1586,6 +1668,22 @@ "node": ">=4" } }, + "node_modules/tar": { + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -1685,6 +1783,15 @@ "engines": { "node": ">=0.4" } + }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } } } } diff --git a/package.json b/package.json index 91e1354..e348bd7 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "author": "", "license": "MIT", "dependencies": { + "7z-wasm": "^1.2.0", "adm-zip": "^0.5.17", "bcryptjs": "^3.0.3", "cookie-parser": "^1.4.7", @@ -26,7 +27,10 @@ "multer": "^1.4.5-lts.1", "node-stl": "^0.7.0", "nodemailer": "^9.0.1", - "pg": "^8.22.0" + "occt-import-js": "^0.0.23", + "pg": "^8.22.0", + "rhino3dm": "^8.32.2", + "tar": "^7.5.22" }, "devDependencies": { "nodemon": "^3.0.2" diff --git a/public/app.js b/public/app.js index ef0f2a5..9fb77fa 100644 --- a/public/app.js +++ b/public/app.js @@ -189,8 +189,8 @@ async function generateMissingThumbnails() { if (!file) continue; thumbAttempted.add(m.id); try { - const src = viewerSourceFor(file.id, file.format); - const image = await window.ModelViewer.snapshot(src.url, src.format, 512); + const src = viewerSourceFor({ kind: 'file', id: file.id, format: file.format }); + const image = await window.ModelViewer.snapshot(src.url, src.format, 512, { resolve: resolverFor(m) }); if (!image) continue; const res = await fetch(`/api/models/${m.id}/thumbnail`, { method: 'POST', @@ -484,7 +484,7 @@ function duplicateGroupHtml(g) { function duplicateCardHtml(g, m) { const recommended = m.id === g.recommendedId; - const unique = m.files.length - m.matchedFiles; + const unique = m.matchableFiles - m.matchedFiles; return `
${thumbHtml(m)}
@@ -492,7 +492,7 @@ function duplicateCardHtml(g, m) { ${recommended ? 'Recommended keep' : ''}
${escapeHtml(m.name)}
${escapeHtml(sourceLabel(sourceKey(m)))} • ${formatDate(m.date_added)}
-
${m.matchedFiles} of ${m.files.length} files match${unique ? ` • ${unique} unique` : ''}
+
${m.matchedFiles} of ${m.matchableFiles} files match${unique ? ` • ${unique} unique` : ''}
`; @@ -505,7 +505,7 @@ async function handleKeepDuplicate(signature, keepId) { const others = group.models.filter((m) => m.id !== keepId); let message = `Keep "${keep.name}" and move ${others.length} other model${others.length === 1 ? '' : 's'} to the trash?\n\n` + 'Their tags, collections and notes are merged into the one you keep.'; - const withUnique = others.filter((m) => m.files.length > m.matchedFiles); + const withUnique = others.filter((m) => m.matchableFiles > m.matchedFiles); if (withUnique.length) { message += `\n\nHeads up: ${withUnique.map((m) => `"${m.name}"`).join(', ')} also contain files that aren't duplicated — they go to the trash too.`; } @@ -700,6 +700,22 @@ function setupEventListeners() { }); document.getElementById('detailNotes').addEventListener('input', queueNotesSave); document.getElementById('detailNotes').addEventListener('blur', flushNotes); + document.getElementById('detailFileTabs').addEventListener('click', (e) => { + const tab = e.target.closest('.file-tab'); + const item = tab && detailItems.find((i) => itemKey(i) === tab.dataset.key); + if (!item) return; + renderViewerTabs(itemKey(item)); + loadViewer(item); + }); + document.getElementById('detailFileInfo').addEventListener('click', (e) => { + const btn = e.target.closest('[data-view-entry]'); + if (!btn) return; + const archiveId = Number(btn.dataset.archive); + const entry = (archiveEntryCache.get(archiveId) || []).find((x) => String(x.id) === btn.dataset.viewEntry); + if (!entry) return; + openViewerItem({ kind: 'entry', id: entry.id, format: entry.format, name: entry.path.split('/').pop(), archiveId }); + document.getElementById('detailViewer').scrollIntoView({ block: 'nearest', behavior: 'smooth' }); + }); setupTagAutocomplete(document.getElementById('fileTags')); setupTagAutocomplete(document.getElementById('editTags')); @@ -924,38 +940,98 @@ function openModelDetail(modelId) {
${escapeHtml(f.originalName)} (${formatFileSize(f.size)}${f.volume ? `, ${Number(f.volume).toFixed(2)} cm³` : ''}) download -
`).join(''); + + ${ARCHIVE_FORMATS.includes(f.format) ? `
Reading archive…
` : ''}`).join(''); + model.files.filter((f) => ARCHIVE_FORMATS.includes(f.format)).forEach((f) => loadArchiveEntries(model.id, f.id)); } - // Viewer tabs - const renderable = model.files.filter((f) => window.ModelViewer.RENDERABLE.includes(f.format)); - const tabs = document.getElementById('detailFileTabs'); - tabs.innerHTML = renderable.map((f, i) => ` - `).join(''); - tabs.querySelectorAll('.file-tab').forEach((tab) => { - tab.addEventListener('click', () => { - tabs.querySelectorAll('.file-tab').forEach((t) => t.classList.remove('active')); - tab.classList.add('active'); - loadViewer(tab.dataset.fileId, tab.dataset.format); - }); - }); + // Viewer tabs: 3D files first, then pictures/PDFs/text (entries inside + // archives get a tab when opened from the file list). + const { RENDERABLE, DOCUMENT_FORMATS } = window.ModelViewer; + detailItems = [ + ...model.files.filter((f) => RENDERABLE.includes(f.format)), + ...model.files.filter((f) => DOCUMENT_FORMATS.includes(f.format) && f.format !== 'mtl'), + ].map((f) => ({ kind: 'file', id: f.id, format: f.format, name: f.originalName })); + renderViewerTabs(detailItems.length ? itemKey(detailItems[0]) : null); fileDetailModal.classList.add('active'); currentDetailFile = null; - if (renderable.length) { - loadViewer(renderable[0].id, renderable[0].format); + if (detailItems.length) { + loadViewer(detailItems[0]); } else { document.getElementById('regenThumbBtn').style.display = 'none'; window.ModelViewer.render(document.getElementById('detailViewer'), '', 'none', { fallbackImage: currentDetailThumb }); } } -// Maps a file to what the viewer should load: 3MF goes through the server's -// GLB conversion; everything else is served raw. -function viewerSourceFor(fileId, format) { - return format === '3mf' - ? { url: `/api/files/${fileId}/glb`, format: 'glb' } - : { url: `/api/files/${fileId}/raw`, format }; +// ---------------------------------------------------------------- viewer items +// An item is something the viewer can show: { kind: 'file' | 'entry', id, format, name, archiveId? }. +const ARCHIVE_FORMATS = ['zip', '7z', 'rar', 'tar', 'tgz']; +let detailItems = []; +const archiveEntryCache = new Map(); // archive file id -> entries +const itemKey = (item) => `${item.kind}:${item.id}`; + +function renderViewerTabs(activeKey) { + document.getElementById('detailFileTabs').innerHTML = detailItems.map((item) => ` + `).join(''); +} + +function openViewerItem(item) { + if (!detailItems.some((i) => itemKey(i) === itemKey(item))) detailItems.push(item); + renderViewerTabs(itemKey(item)); + loadViewer(item); +} + +// What the viewer should load: 3MF goes through the server's GLB conversion, +// documents are served inline, everything else raw. +function viewerSourceFor(item) { + const base = `/api/${item.kind === 'entry' ? 'entries' : 'files'}/${item.id}`; + if (item.format === '3mf') return { url: `${base}/glb`, format: 'glb' }; + if (window.ModelViewer.DOCUMENT_FORMATS.includes(item.format)) return { url: `${base}/raw?inline=1`, format: item.format }; + return { url: `${base}/raw`, format: item.format }; +} + +// Finds files shipped alongside a model by name — OBJ materials, FBX/MTL +// textures. Entries of the same archive win, then the model's own files. +function resolverFor(model, archiveEntries = null) { + const byName = new Map(); + const key = (name) => String(name).split(/[\\/]/).pop().toLowerCase(); + const add = (name, url) => { if (!byName.has(key(name))) byName.set(key(name), url); }; + (archiveEntries || []).forEach((e) => add(e.path, `/api/entries/${e.id}/raw`)); + ((model && model.files) || []).forEach((f) => add(f.originalName, `/api/files/${f.id}/raw`)); + return (name) => byName.get(key(name)) || null; +} + +async function loadArchiveEntries(modelId, fileId) { + const container = () => (fileDetailModal.dataset.currentModelId === String(modelId) + ? document.querySelector(`[data-entries-for="${fileId}"]`) : null); + try { + const res = await fetch(`/api/files/${fileId}/entries`); + const data = await res.json(); + if (!res.ok) throw new Error(data.error); + if (!container()) return; + if (data.status === 'pending') { + setTimeout(() => { if (container()) loadArchiveEntries(modelId, fileId); }, 2000); + return; + } + if (data.status === 'error') { + container().innerHTML = `
Couldn't read this archive: ${escapeHtml(data.error || 'unknown error')}
`; + return; + } + archiveEntryCache.set(fileId, data.entries); + const { PREVIEWABLE } = window.ModelViewer; + container().innerHTML = `
${data.entries.length} file${data.entries.length === 1 ? '' : 's'} inside
` + + data.entries.map((e) => ` +
+ ${escapeHtml(e.path.replace(/^\.\//, ''))} ${formatFileSize(e.size)} + + ${PREVIEWABLE.includes(e.format) && e.format !== 'mtl' ? `` : ''} + download + +
`).join(''); + } catch (err) { + if (container()) container().innerHTML = `
Couldn't list this archive: ${escapeHtml(err.message)}
`; + } } function renderDetailLicense(model) { @@ -1068,11 +1144,15 @@ async function handleRefreshMetadata() { } } -function loadViewer(fileId, format) { - const src = viewerSourceFor(fileId, format); - currentDetailFile = { id: fileId, viewerUrl: src.url, viewerFormat: src.format }; - document.getElementById('regenThumbBtn').style.display = ''; - window.ModelViewer.render(document.getElementById('detailViewer'), src.url, src.format, { fallbackImage: currentDetailThumb }); +function loadViewer(item) { + const src = viewerSourceFor(item); + const model = findModel(fileDetailModal.dataset.currentModelId); + const resolve = resolverFor(model, item.kind === 'entry' ? archiveEntryCache.get(item.archiveId) : null); + const is3d = window.ModelViewer.MODEL_FORMATS.includes(src.format); + currentDetailFile = is3d ? { viewerUrl: src.url, viewerFormat: src.format, resolve } : null; + document.getElementById('regenThumbBtn').style.display = is3d ? '' : 'none'; + window.ModelViewer.render(document.getElementById('detailViewer'), src.url, src.format, + { fallbackImage: currentDetailThumb, resolve }); } function updateCardThumbnail(modelId, thumbnailUrl) { @@ -1089,7 +1169,8 @@ async function handleRegenThumbnail() { btn.disabled = true; btn.textContent = 'Rendering…'; try { - const image = await window.ModelViewer.snapshot(currentDetailFile.viewerUrl, currentDetailFile.viewerFormat, 512); + const image = await window.ModelViewer.snapshot(currentDetailFile.viewerUrl, currentDetailFile.viewerFormat, 512, + { resolve: currentDetailFile.resolve }); const res = await fetch(`/api/models/${modelId}/thumbnail`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, diff --git a/public/index.html b/public/index.html index 92fd9c0..e38ceb1 100644 --- a/public/index.html +++ b/public/index.html @@ -329,8 +329,8 @@