diff --git a/.dockerignore b/.dockerignore index ee63c18..51ef26e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,4 @@ npm-debug.log* Dockerfile .dockerignore docker-compose.yml +extracted diff --git a/.gitignore b/.gitignore index 3deaf18..fa332d7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ thumbnails/ glb/ tools/venv/ __pycache__/ +extracted/ diff --git a/README.md b/README.md index 2d87e40..7f7c104 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Printhive -A self-hosted web app for organizing your 3D-printing files (STL, 3MF, OBJ, STEP, GCODE, ZIP) into a searchable library — with in-browser 3D previews, collections, license/commercial-use tracking, and one-paste **import from popular model sites**. +A self-hosted web app for organizing your 3D-printing files (STL, 3MF, OBJ, FBX, STEP, 3DM, GCODE — loose or in ZIP/7Z/RAR/TAR archives) into a searchable library — with in-browser 3D previews, collections, license/commercial-use tracking, and one-paste **import from popular model sites**. ## Features @@ -10,8 +10,11 @@ A self-hosted web app for organizing your 3D-printing files (STL, 3MF, OBJ, STEP - **Printables** — downloads files automatically via its public GraphQL API (no auth needed for free models). - **MakerWorld** — downloads files via FlareSolverr (solves Cloudflare) + your account token. - **cubee3d (Hive)** and any other URL — imports title/description/preview image; you add the files manually. -- **In-browser 3D viewer** (Three.js) for STL/OBJ, and for 3MF via a server-side **trimesh GLB conversion** (handles Bambu/MakerWorld production-extension files the JS loader can't). -- **Auto thumbnails** rendered from the model; embedded 3MF thumbnails used when present. +- **In-browser 3D viewer** (Three.js) for STL, OBJ (with its MTL colours/textures), FBX (with textures), STEP/STP (OpenCascade WASM) and Rhino 3DM, and for 3MF via a server-side **trimesh GLB conversion** that streams even huge Bambu/MakerWorld projects. +- **Slicer projects plate by plate** — Bambu Studio / OrcaSlicer 3MFs open on plate 1 with the slicer's plate images as a switcher, parts shown in their filament colours, and a Filaments list; negative/modifier parts are hidden. +- **Archives as folders** — ZIP, 7Z, RAR and TAR/.tar.gz uploads are indexed in the background; browse what's inside and preview or download any file without unpacking. Models inside archives count for duplicate detection. +- **Pictures, PDFs & readmes** — keep them with a model and view them in the app. +- **Auto thumbnails** made in the background: an embedded 3MF/plate preview, else a picture that came with the model (or one inside its archive), else a server-side render of the STL/OBJ/3MF; the browser renders FBX/STEP/3DM ones. - **License & commercial-use tracking** — captured per source, with a "sellable / non-commercial" indicator (a guide; verify before selling). - **Collections** — group models (a model can be in many); rename/delete without losing models. - **Editing** — rename/retag, edit print settings, override commercial-use, refresh metadata from source, re-download files, download all files as a zip. @@ -31,10 +34,10 @@ A self-hosted web app for organizing your 3D-printing files (STL, 3MF, OBJ, STEP ## Setup 1. **Postgres** — create a database and point `DATABASE_URL` at it. The schema is applied automatically on startup. -2. **Python sidecar** (for 3MF previews/volume and shape-based duplicate detection — without it, duplicates are still found by content hash): +2. **Python sidecar** (for 3MF previews, plates, server-side thumbnails and shape-based duplicate detection — without it, duplicates are still found by content hash): ```bash python3 -m venv tools/venv - tools/venv/bin/pip install trimesh numpy networkx lxml + tools/venv/bin/pip install -r tools/requirements.txt ``` 3. **Install deps and run**: ```bash diff --git a/db/schema.sql b/db/schema.sql index c14c9d8..62451b5 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -124,6 +124,31 @@ ALTER TABLE models ADD COLUMN IF NOT EXISTS notes TEXT NOT NULL DEFAULT ''; ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ; CREATE INDEX IF NOT EXISTS idx_models_deleted ON models(deleted_at); +-- Archive files (zip/7z/rar/tar) are indexed in the background: +-- null = not yet, ok | error (with archive_error). +ALTER TABLE files ADD COLUMN IF NOT EXISTS archive_status TEXT; +ALTER TABLE files ADD COLUMN IF NOT EXISTS archive_error TEXT; + +-- Files inside archives, so they can be browsed, previewed and matched as duplicates. +CREATE TABLE IF NOT EXISTS archive_entries ( + id SERIAL PRIMARY KEY, + file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, + path TEXT NOT NULL, -- path inside the archive, as the archive stores it + format TEXT NOT NULL, + size BIGINT NOT NULL DEFAULT 0, + sha256 TEXT, + geometry_hash TEXT, + geometry_status TEXT, -- same states as files.geometry_status + fingerprinted_at TIMESTAMPTZ, + UNIQUE (file_id, path) +); +CREATE INDEX IF NOT EXISTS idx_entries_sha256 ON archive_entries(sha256); +CREATE INDEX IF NOT EXISTS idx_entries_geometry ON archive_entries(geometry_hash); + +-- Background thumbnails: null = not tried yet, none = nothing to make one from +-- (left to the browser), error. Cleared when files are added. +ALTER TABLE models ADD COLUMN IF NOT EXISTS thumbnail_status TEXT; + -- Duplicate groups marked "not duplicates" (signature = sorted model ids). CREATE TABLE IF NOT EXISTS duplicate_dismissals ( signature TEXT PRIMARY KEY, diff --git a/docker-compose.yml b/docker-compose.yml index 1d27f68..1ae9204 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,7 @@ services: - uploads:/app/uploads - thumbnails:/app/thumbnails - glb:/app/glb + - extracted:/app/extracted # cache of files unpacked from archives # No host port published — Traefik auto-discovers the app on the shared network. networks: - default # talk to the db @@ -56,6 +57,7 @@ volumes: uploads: thumbnails: glb: + extracted: networks: default: diff --git a/lib/archiveWorker.js b/lib/archiveWorker.js new file mode 100644 index 0000000..a8da4e5 --- /dev/null +++ b/lib/archiveWorker.js @@ -0,0 +1,82 @@ +// Worker thread running 7-Zip (compiled to WASM) for ZIP/7Z/RAR archives. +// 7-Zip calls are synchronous, so they run here to keep the server responsive. + +const { parentPort } = require('worker_threads'); +const SevenZip = require('7z-wasm'); + +let sevenZip = null; +const output = []; + +// The host filesystem is mounted at /host inside the WASM filesystem. +const host = (p) => `/host${p}`; + +async function instance() { + if (!sevenZip) { + sevenZip = await SevenZip({ print: (l) => output.push(l), printErr: (l) => output.push(l) }); + sevenZip.FS.mkdir('/host'); + sevenZip.FS.mount(sevenZip.NODEFS, { root: '/' }, '/host'); + } + return sevenZip; +} + +async function run(args) { + const sz = await instance(); + output.length = 0; + let code; + try { + code = sz.callMain(args); + } catch (err) { + code = err && typeof err.status === 'number' ? err.status : 2; + } + return { code, lines: output.slice() }; +} + +function errorText(lines) { + const line = lines.find((l) => /error|wrong password|can ?not open|unsupported|headers/i.test(l)); + return line ? line.trim() : null; +} + +// Parses `7zz l -slt` into [{ path, size }] for files (folders are dropped). +function parseListing(lines) { + const start = lines.findIndex((l) => l.startsWith('----------')); + if (start === -1) return null; + const entries = []; + let cur = {}; + const flush = () => { + const isFolder = cur.Folder === '+' || (cur.Attributes || '').startsWith('D'); + if (cur.Path !== undefined && !isFolder) entries.push({ path: cur.Path, size: Number(cur.Size) || 0 }); + cur = {}; + }; + for (const line of lines.slice(start + 1)) { + if (!line.trim()) { flush(); continue; } + const i = line.indexOf(' = '); + if (i > 0) cur[line.slice(0, i)] = line.slice(i + 3); + } + flush(); + return entries; +} + +async function list(archive) { + // A dummy password makes encrypted archives fail fast instead of prompting. + const { code, lines } = await run(['l', '-slt', '-pprinthive', host(archive)]); + const entries = parseListing(lines); + if (code !== 0 || !entries) throw new Error(errorText(lines) || `7-Zip could not read the archive (exit ${code})`); + return entries; +} + +// Extracts the named entries (keeping their folders) under destDir. Callers +// check which files actually arrived. +async function extract(archive, entries, destDir) { + const { code, lines } = await run( + ['x', host(archive), `-o${host(destDir)}`, '-y', '-spd', '-bsp0', '-pprinthive', '--', ...entries]); + return { code, error: code === 0 ? null : errorText(lines) }; +} + +parentPort.on('message', async ({ id, op, archive, entries, destDir }) => { + try { + const result = op === 'list' ? await list(archive) : await extract(archive, entries, destDir); + parentPort.postMessage({ id, result }); + } catch (err) { + parentPort.postMessage({ id, error: err.message }); + } +}); diff --git a/lib/archives.js b/lib/archives.js new file mode 100644 index 0000000..0ec1fc4 --- /dev/null +++ b/lib/archives.js @@ -0,0 +1,133 @@ +// Lists and extracts files inside archives. ZIP/7Z/RAR go through 7-Zip (WASM, +// in a worker thread); TAR and .tar.gz go through node-tar. Extracted entries +// are cached under EXTRACT_DIR// and can always be re-extracted. + +const fs = require('fs'); +const path = require('path'); +const { Worker } = require('worker_threads'); +const tar = require('tar'); + +const EXTRACT_DIR = path.resolve(process.env.EXTRACT_DIR || './extracted'); +const TAR_FILE_TYPES = ['File', 'OldFile', 'ContiguousFile']; + +let worker = null; +let nextId = 1; +const pending = new Map(); + +function failPending(err) { + for (const p of pending.values()) p.reject(err); + pending.clear(); + worker = null; +} + +// The worker keeps the process alive only while a request is outstanding, so +// an idle worker never stops Node from exiting. +function sevenZip(message) { + if (!worker) { + worker = new Worker(path.join(__dirname, 'archiveWorker.js')); + worker.on('message', ({ id, result, error }) => { + const p = pending.get(id); + pending.delete(id); + if (pending.size === 0 && worker) worker.unref(); + if (p) error ? p.reject(new Error(error)) : p.resolve(result); + }); + worker.on('error', failPending); + worker.on('exit', () => failPending(new Error('Archive worker stopped'))); + } + return new Promise((resolve, reject) => { + const id = nextId++; + pending.set(id, { resolve, reject }); + worker.ref(); + worker.postMessage({ id, ...message }); + }); +} + +const isTar = (format) => format === 'tar' || format === 'tgz'; +const clean = (p) => p.replace(/^\.\//, ''); + +// Clutter archivers add (macOS resource forks, Finder/Explorer files). +function isJunk(entryPath) { + const parts = clean(entryPath).split('/'); + const name = parts[parts.length - 1]; + return parts.includes('__MACOSX') || name.startsWith('._') || name === '.DS_Store' || name === 'Thumbs.db'; +} + +// Paths that could land outside the extraction folder are never used. +function isSafe(entryPath) { + return Boolean(entryPath) && !path.isAbsolute(entryPath) && !entryPath.split(/[\\/]/).includes('..'); +} + +// file: { file_path, format }. Returns [{ path, size }] of real files. +async function listEntries(file) { + const abs = path.resolve(file.file_path); + let raw; + if (isTar(file.format)) { + raw = []; + await tar.t({ + file: abs, + onReadEntry: (e) => { if (TAR_FILE_TYPES.includes(e.type)) raw.push({ path: e.path, size: e.size }); }, + }); + } else { + raw = await sevenZip({ op: 'list', archive: abs }); + } + return raw.filter((e) => isSafe(e.path) && !isJunk(e.path)); +} + +function tempDir() { + fs.mkdirSync(EXTRACT_DIR, { recursive: true }); + return fs.mkdtempSync(path.join(EXTRACT_DIR, '.tmp-')); +} + +// Extracts entries into destDir. Returns Map(entry path -> absolute path) for +// the ones that came out; archive-level failures throw. +async function extractMany(file, entryPaths, destDir) { + const abs = path.resolve(file.file_path); + const wanted = new Set(entryPaths.filter(isSafe)); + if (isTar(file.format)) { + await tar.x({ file: abs, cwd: destDir, filter: (p) => wanted.has(p) }); + } else { + const { code, error } = await sevenZip({ op: 'extract', archive: abs, entries: [...wanted], destDir }); + if (code !== 0 && error) console.warn(`7-Zip reported a problem extracting from ${abs}: ${error}`); + } + const out = new Map(); + for (const p of wanted) { + const target = path.resolve(destDir, p); + if (target.startsWith(destDir + path.sep) && fs.existsSync(target)) out.set(p, target); + } + return out; +} + +function entryCachePath(fileId, entry) { + return path.join(EXTRACT_DIR, String(fileId), `${entry.id}${path.extname(entry.path).toLowerCase()}`); +} + +const inflight = new Map(); + +// file: { id, file_path, format }; entry: { id, path }. Returns the absolute +// path of a cached copy, extracting it on first use. +async function extractEntry(file, entry) { + const dest = entryCachePath(file.id, entry); + if (fs.existsSync(dest)) return dest; + if (!inflight.has(dest)) { + const job = (async () => { + fs.mkdirSync(path.dirname(dest), { recursive: true }); + const tmp = tempDir(); + try { + const got = (await extractMany(file, [entry.path], tmp)).get(entry.path); + if (!got) throw new Error(`Could not extract ${clean(entry.path)} from the archive`); + fs.renameSync(got, dest); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + })().finally(() => inflight.delete(dest)); + inflight.set(dest, job); + } + await inflight.get(dest); + return dest; +} + +function removeExtracted(fileId) { + fs.rmSync(path.join(EXTRACT_DIR, String(fileId)), { recursive: true, force: true }); +} + +module.exports = { listEntries, extractMany, extractEntry, removeExtracted, tempDir, isJunk }; diff --git a/lib/converter.js b/lib/converter.js index 827db60..68f199e 100644 --- a/lib/converter.js +++ b/lib/converter.js @@ -1,22 +1,37 @@ // Converts 3MF files to GLB via the Python/trimesh sidecar, so the browser can -// render Bambu/MakerWorld production-extension 3MFs (which the JS 3MFLoader can't). +// render Bambu/MakerWorld production-extension 3MFs (which the JS 3MFLoader can't), +// and reads their slicer project details (plates, filaments). const { runScript } = require('./python'); -// Returns { glb, volume_cm3, bbox, faces } or throws. -async function convert3mfToGlb(inputPath, outputPath) { - const { err, stdout, stderr } = await runScript('convert_3mf.py', [inputPath, outputPath], { timeout: 120000 }); +// Parses a sidecar script's last JSON line, turning its error / failure into a throw. +function parseResult({ err, stdout, stderr }, what) { let result = null; try { result = JSON.parse(stdout.trim().split('\n').pop()); } catch { /* handled below */ } if (result && result.error) throw new Error(result.error); if (err) { - if (err.killed) throw new Error('Timed out converting this 3MF for preview.'); + if (err.killed) throw new Error(`Timed out ${what}.`); throw new Error(stderr ? stderr.trim().split('\n').slice(-3).join(' ') : err.message); } - if (!result) throw new Error('Converter returned unparseable output.'); + if (!result) throw new Error(`Unparseable output ${what}.`); return result; } -module.exports = { convert3mfToGlb }; +// Returns { glb, plate, volume_cm3, bbox, faces } or throws. `plate` limits the +// output to one plate of a slicer project. +async function convert3mfToGlb(inputPath, outputPath, { plate = null } = {}) { + const args = [inputPath, outputPath]; + if (plate) args.push('--plate', String(plate)); + return parseResult(await runScript('convert_3mf.py', args, { timeout: 120000 }), 'converting this 3MF for preview'); +} + +// Returns { plates: [{ index, name, objectCount, thumbnail }], filaments: [{ colour, type }] }. +// Plate previews are written into imagesDir as .png / -small.png. +async function readProjectInfo(inputPath, imagesDir) { + return parseResult(await runScript('threemf_info.py', [inputPath, '--images', imagesDir], { timeout: 60000 }), + 'reading this 3MF project'); +} + +module.exports = { convert3mfToGlb, readProjectInfo }; diff --git a/lib/fileProcessor.js b/lib/fileProcessor.js index 2e97843..2da16f9 100644 --- a/lib/fileProcessor.js +++ b/lib/fileProcessor.js @@ -5,18 +5,25 @@ const AdmZip = require('adm-zip'); const THUMBNAIL_DIR = './thumbnails'; -const MODEL_FORMATS = ['stl', '3mf', 'obj', 'step', 'stp', 'gcode', 'gco', 'zip']; +const MODEL_FORMATS = ['stl', '3mf', 'obj', 'fbx', 'step', 'stp', '3dm', 'gcode', 'gco']; +const ARCHIVE_FORMATS = ['zip', '7z', 'rar', 'tar', 'tgz']; +const IMAGE_FORMATS = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'svg']; +// Kept with a model and viewable, but never matched as duplicates. +const COMPANION_FORMATS = [...IMAGE_FORMATS, 'pdf', 'txt', 'md', 'mtl']; +const ALLOWED_FORMATS = [...MODEL_FORMATS, ...ARCHIVE_FORMATS, ...COMPANION_FORMATS]; function ensureThumbnailDir() { if (!fs.existsSync(THUMBNAIL_DIR)) fs.mkdirSync(THUMBNAIL_DIR, { recursive: true }); } function formatOf(filename) { - return path.extname(filename).slice(1).toLowerCase(); + const lower = filename.toLowerCase(); + if (lower.endsWith('.tar.gz')) return 'tgz'; + return path.extname(lower).slice(1); } function isAllowed(filename) { - return MODEL_FORMATS.includes(formatOf(filename)); + return ALLOWED_FORMATS.includes(formatOf(filename)); } // Geometry metadata. Only STL is computed natively today; others return nulls @@ -37,16 +44,28 @@ function extractMetadata(filePath, format) { } } +// Which embedded PNG makes the best card image: the project thumbnail, then the +// first plate render. Slicer pick maps, top views and small/unlit variants last. +function thumbnailRank(name) { + if (name === 'metadata/thumbnail.png') return 0; + if (/metadata\/plate_1\.png$/.test(name)) return 1; + if (/metadata\/plate_\d+\.png$/.test(name)) return 2; + if (/(pick_|top_|no_light|_small)/.test(name)) return 9; + return 5; +} + // A .3mf is a zip; slicers embed a PNG preview under /Metadata/. If present, // pull it out as the model thumbnail. Returns a path under thumbnails/ or null. function extract3mfThumbnail(filePath, baseName) { try { ensureThumbnailDir(); const zip = new AdmZip(filePath); - const entry = zip.getEntries().find((e) => { - const n = e.entryName.toLowerCase(); - return n.endsWith('.png') && (n.includes('thumbnail') || n.includes('/metadata/')); - }); + const entry = zip.getEntries() + .filter((e) => { + const n = e.entryName.toLowerCase(); + return n.endsWith('.png') && (n.includes('thumbnail') || n.includes('metadata/')); + }) + .sort((a, b) => thumbnailRank(a.entryName.toLowerCase()) - thumbnailRank(b.entryName.toLowerCase()))[0]; if (!entry) return null; const out = path.join(THUMBNAIL_DIR, `${baseName}.png`); fs.writeFileSync(out, entry.getData()); @@ -57,4 +76,21 @@ function extract3mfThumbnail(filePath, baseName) { } } -module.exports = { MODEL_FORMATS, formatOf, isAllowed, extractMetadata, extract3mfThumbnail }; +// Copy a picture that came with a model into thumbnails/ (a copy, so replacing +// the thumbnail later never deletes the model's own file). +function copyAsThumbnail(filePath, baseName) { + try { + ensureThumbnailDir(); + const out = path.join(THUMBNAIL_DIR, `${baseName}-image${path.extname(filePath).toLowerCase()}`); + fs.copyFileSync(filePath, out); + return out; + } catch (err) { + console.error('Image thumbnail copy failed:', err.message); + return null; + } +} + +module.exports = { + MODEL_FORMATS, ARCHIVE_FORMATS, IMAGE_FORMATS, COMPANION_FORMATS, ALLOWED_FORMATS, + formatOf, isAllowed, extractMetadata, extract3mfThumbnail, copyAsThumbnail, +}; diff --git a/lib/fingerprint.js b/lib/fingerprint.js index d4031e4..949a8d0 100644 --- a/lib/fingerprint.js +++ b/lib/fingerprint.js @@ -1,16 +1,22 @@ -// Background worker that fingerprints stored files for the duplicate finder: a -// SHA-256 of the bytes for every file, plus a geometry fingerprint for mesh +// Background worker that indexes and fingerprints stored files for the +// duplicate finder: a SHA-256 of the bytes, plus a geometry fingerprint for mesh // formats so the same model still matches after being renamed, re-exported -// (ASCII/binary STL) or converted (STL -> 3MF). +// (ASCII/binary STL) or converted (STL -> 3MF). Archives are listed, and the +// models inside them are fingerprinted too. const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const db = require('./db'); +const store = require('./store'); +const archives = require('./archives'); +const thumbnails = require('./thumbnails'); +const { MODEL_FORMATS, ARCHIVE_FORMATS, formatOf } = require('./fileProcessor'); const { PYTHON, runScript } = require('./python'); const GEOMETRY_FORMATS = ['stl', 'obj', '3mf']; const BATCH = 10; +const TABLES = ['files', 'archive_entries']; let pythonAvailable = true; let running = false; @@ -58,64 +64,154 @@ async function measure(paths) { return results; } -async function fingerprintGeometry(rows) { - const abs = (row) => path.resolve(row.file_path); +// rows: [{ id, format, abs }] from `table` ('files' or 'archive_entries'). +async function fingerprintGeometry(rows, table) { + if (!TABLES.includes(table)) throw new Error(`Unknown table ${table}`); // 3MF projects can be big, so each gets its own run; that keeps a queued // preview from waiting behind a whole batch of them. const chunks = [rows.filter((r) => r.format !== '3mf'), ...rows.filter((r) => r.format === '3mf').map((r) => [r])]; for (const chunk of chunks.filter((c) => c.length)) { - const results = await measure(chunk.map(abs)); + const results = await measure(chunk.map((r) => r.abs)); for (const row of chunk) { - let r = results.get(abs(row)); + let r = results.get(row.abs); // A batch that died part-way: retry the missing file on its own. - if (!r && chunk.length > 1) r = (await measure([abs(row)])).get(abs(row)); + if (!r && chunk.length > 1) r = (await measure([row.abs])).get(row.abs); const key = r && r.fp ? geometryKey(r.fp) : null; - await db.query('UPDATE files SET geometry_hash = $2, geometry_status = $3 WHERE id = $1', + await db.query(`UPDATE ${table} SET geometry_hash = $2, geometry_status = $3 WHERE id = $1`, [row.id, key, key ? 'ok' : 'error']); } } } -async function run() { - for (;;) { - const { rows } = await db.query( - `SELECT id, file_path, format, fingerprinted_at FROM files - WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean) - ORDER BY id LIMIT $2`, [pythonAvailable, BATCH]); - if (rows.length === 0) return; +// Runs fn; if Python turns out to be missing, geometry fingerprints are switched +// off for this process (content hashes still run). +async function withPython(fn) { + try { + await fn(); + } catch (err) { + if (err.code !== 'ENOENT') throw err; + pythonAvailable = false; + console.warn(`Geometry fingerprints disabled: no Python at ${PYTHON} (content hashes still run).`); + } +} +async function fingerprintFiles() { + const { rows } = await db.query( + `SELECT id, file_path, format, fingerprinted_at FROM files + WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean) + ORDER BY id LIMIT $2`, [pythonAvailable, BATCH]); + if (rows.length === 0) return false; + + const geometry = []; + for (const row of rows) { + const abs = path.resolve(row.file_path); + try { + if (!fs.existsSync(abs)) { + await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'missing' WHERE id = $1`, [row.id]); + continue; + } + if (!row.fingerprinted_at) { + await db.query('UPDATE files SET sha256 = $2, fingerprinted_at = now() WHERE id = $1', + [row.id, await sha256File(abs)]); + } + if (!GEOMETRY_FORMATS.includes(row.format)) { + await db.query(`UPDATE files SET geometry_status = 'na' WHERE id = $1`, [row.id]); + } else if (pythonAvailable) { + geometry.push({ id: row.id, format: row.format, abs }); + } + } catch (err) { + console.warn(`Fingerprint failed for file ${row.id}:`, err.message); + await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [row.id]); + } + } + if (geometry.length) await withPython(() => fingerprintGeometry(geometry, 'files')); + return true; +} + +async function indexArchives() { + const { rows } = await db.query( + `SELECT id, file_path, format FROM files + WHERE format = ANY($1) AND archive_status IS NULL ORDER BY id LIMIT 5`, [ARCHIVE_FORMATS]); + for (const file of rows) { + try { + const entries = await archives.listEntries(file); + await store.replaceArchiveEntries(file.id, + entries.map((e) => ({ path: e.path, size: e.size, format: formatOf(e.path) }))); + await db.query(`UPDATE files SET archive_status = 'ok', archive_error = NULL WHERE id = $1`, [file.id]); + } catch (err) { + console.warn(`Could not read archive (file ${file.id}):`, err.message); + await db.query(`UPDATE files SET archive_status = 'error', archive_error = $2 WHERE id = $1`, + [file.id, err.message.slice(0, 300)]); + } + } + return rows.length > 0; +} + +const ENTRY_PENDING = 'e.fingerprinted_at IS NULL OR (e.geometry_status IS NULL AND $1::boolean)'; + +// Fingerprints the pending entries of one archive, extracting its models in a +// single pass (cheaper than one extraction per entry, especially for solid 7z). +async function fingerprintEntries() { + const { rows: [file] } = await db.query( + `SELECT f.id, f.file_path, f.format FROM files f + WHERE EXISTS (SELECT 1 FROM archive_entries e WHERE e.file_id = f.id AND (${ENTRY_PENDING})) + ORDER BY f.id LIMIT 1`, [pythonAvailable]); + if (!file) return false; + const { rows: entries } = await db.query( + `SELECT e.id, e.path, e.format, e.fingerprinted_at FROM archive_entries e + WHERE e.file_id = $2 AND (${ENTRY_PENDING}) ORDER BY e.id LIMIT 500`, [pythonAvailable, file.id]); + + // Pictures, documents and nested archives are listed but not matched as duplicates. + const others = entries.filter((e) => !MODEL_FORMATS.includes(e.format)); + if (others.length) { + await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'na' WHERE id = ANY($1::int[])`, + [others.map((e) => e.id)]); + } + const models = entries.filter((e) => MODEL_FORMATS.includes(e.format)); + if (!models.length) return true; + + const tmp = archives.tempDir(); + try { + let extracted = new Map(); + try { + extracted = await archives.extractMany(file, models.map((e) => e.path), tmp); + } catch (err) { + console.warn(`Could not extract from archive (file ${file.id}):`, err.message); + } const geometry = []; - for (const row of rows) { - const abs = path.resolve(row.file_path); + for (const entry of models) { + const abs = extracted.get(entry.path); try { - if (!fs.existsSync(abs)) { - await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'missing' WHERE id = $1`, [row.id]); + if (!abs) { + await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [entry.id]); continue; } - if (!row.fingerprinted_at) { - await db.query('UPDATE files SET sha256 = $2, fingerprinted_at = now() WHERE id = $1', - [row.id, await sha256File(abs)]); + if (!entry.fingerprinted_at) { + await db.query('UPDATE archive_entries SET sha256 = $2, fingerprinted_at = now() WHERE id = $1', + [entry.id, await sha256File(abs)]); } - if (!GEOMETRY_FORMATS.includes(row.format)) { - await db.query(`UPDATE files SET geometry_status = 'na' WHERE id = $1`, [row.id]); + if (!GEOMETRY_FORMATS.includes(entry.format)) { + await db.query(`UPDATE archive_entries SET geometry_status = 'na' WHERE id = $1`, [entry.id]); } else if (pythonAvailable) { - geometry.push(row); + geometry.push({ id: entry.id, format: entry.format, abs }); } } catch (err) { - console.warn(`Fingerprint failed for file ${row.id}:`, err.message); - await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [row.id]); + console.warn(`Fingerprint failed for archive entry ${entry.id}:`, err.message); + await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [entry.id]); } } + if (geometry.length) await withPython(() => fingerprintGeometry(geometry, 'archive_entries')); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + return true; +} - if (geometry.length) { - try { - await fingerprintGeometry(geometry); - } catch (err) { - if (err.code !== 'ENOENT') throw err; - pythonAvailable = false; - console.warn(`Geometry fingerprints disabled: no Python at ${PYTHON} (content hashes still run).`); - } - } +async function run() { + for (;;) { + // Thumbnails go last so they can use what's inside newly indexed archives. + const worked = [await fingerprintFiles(), await indexArchives(), await fingerprintEntries(), await thumbnails.generate()]; + if (!worked.some(Boolean)) return; } } @@ -133,9 +229,12 @@ function kick() { async function pendingCount() { const { rows } = await db.query( - `SELECT COUNT(*)::int AS n FROM files - WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean)`, [pythonAvailable]); - return rows[0].n; + `SELECT (SELECT COUNT(*) FROM files + WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean)) + + (SELECT COUNT(*) FROM files WHERE format = ANY($2) AND archive_status IS NULL) + + (SELECT COUNT(*) FROM archive_entries e WHERE ${ENTRY_PENDING}) AS n`, + [pythonAvailable, ARCHIVE_FORMATS]); + return Number(rows[0].n); } module.exports = { kick, pendingCount, geometryKey, sha256File }; diff --git a/lib/store.js b/lib/store.js index 4144118..9fab0e4 100644 --- a/lib/store.js +++ b/lib/store.js @@ -1,4 +1,5 @@ const db = require('./db'); +const { MODEL_FORMATS, ARCHIVE_FORMATS } = require('./fileProcessor'); // ---- shared SELECT that hydrates a model with tags, projects, files, designer ---- const MODEL_SELECT = ` @@ -25,7 +26,8 @@ const MODEL_SELECT = ` SELECT json_agg(json_build_object( 'id', fl.id, 'originalName', fl.original_name, 'fileName', fl.file_name, 'format', fl.format, 'size', fl.size, 'volume', fl.volume, - 'bbox', fl.bbox, 'facets', fl.facets) ORDER BY fl.id) AS files + 'bbox', fl.bbox, 'facets', fl.facets, 'archiveStatus', fl.archive_status, + 'entryCount', (SELECT COUNT(*)::int FROM archive_entries ae WHERE ae.file_id = fl.id)) ORDER BY fl.id) AS files FROM files fl WHERE fl.model_id = m.id ) f ON true `; @@ -116,17 +118,61 @@ async function updateModel(id, updates) { // Returns the on-disk file rows so the caller can unlink them. async function deleteModel(id) { - const { rows } = await db.query('SELECT file_path, glb_path FROM files WHERE model_id = $1', [id]); + const { rows } = await db.query('SELECT id, file_path, glb_path FROM files WHERE model_id = $1', [id]); + const entries = await db.query( + 'SELECT e.id FROM archive_entries e JOIN files f ON f.id = e.file_id WHERE f.model_id = $1', [id]); const model = await db.query('SELECT thumbnail_path FROM models WHERE id = $1', [id]); const result = await db.query('DELETE FROM models WHERE id = $1 RETURNING id', [id]); if (result.rowCount === 0) return null; return { + fileIds: rows.map((r) => r.id), + entryIds: entries.rows.map((e) => e.id), filePaths: rows.map((r) => r.file_path), - glbPaths: rows.map((r) => r.glb_path).filter(Boolean), + glbPaths: [ + ...rows.map((r) => r.glb_path).filter(Boolean), + ...entries.rows.map((e) => `glb/entry-${e.id}.glb`), + ], thumbnailPath: model.rows[0] && model.rows[0].thumbnail_path, }; } +// ---------------------------------------------------------------- files / archive entries +async function getFile(id) { + const { rows } = await db.query('SELECT * FROM files WHERE id = $1', [id]); + return rows[0] || null; +} + +async function getArchiveEntries(fileId) { + const { rows } = await db.query( + 'SELECT id, path, format, size FROM archive_entries WHERE file_id = $1 ORDER BY path', [fileId]); + return rows.map((r) => ({ ...r, size: Number(r.size) })); +} + +// An entry plus the archive it lives in ({ id, file_path, format }). +async function getArchiveEntry(id) { + const { rows } = await db.query( + `SELECT e.id, e.file_id, e.path, e.format, e.size, f.file_path, f.format AS archive_format + FROM archive_entries e JOIN files f ON f.id = e.file_id WHERE e.id = $1`, [id]); + const r = rows[0]; + if (!r) return null; + return { + id: r.id, file_id: r.file_id, path: r.path, format: r.format, size: Number(r.size), + archive: { id: r.file_id, file_path: r.file_path, format: r.archive_format }, + }; +} + +async function replaceArchiveEntries(fileId, entries) { + await db.transaction(async (client) => { + await client.query('DELETE FROM archive_entries WHERE file_id = $1', [fileId]); + if (!entries.length) return; + await client.query( + `INSERT INTO archive_entries (file_id, path, format, size) + SELECT $1::int, * FROM unnest($2::text[], $3::text[], $4::bigint[]) + ON CONFLICT (file_id, path) DO NOTHING`, + [fileId, entries.map((e) => e.path), entries.map((e) => e.format), entries.map((e) => e.size)]); + }); +} + // ---------------------------------------------------------------- trash async function trashModels(ids) { const { rowCount } = await db.query( @@ -153,21 +199,42 @@ async function countTrashed() { // ---------------------------------------------------------------- duplicates // Files of non-trashed models whose content hash or geometry fingerprint also // appears in another non-trashed model — the input to the duplicate finder. +// Ids are prefixed ('f12' for a file, 'e7' for an archive entry) so both kinds +// can be matched together. Pictures and documents never count. async function duplicateCandidateFiles() { const { rows } = await db.query(` WITH live AS ( - SELECT f.id, f.model_id, f.sha256, f.geometry_hash + SELECT 'f' || f.id AS id, f.model_id, f.sha256, f.geometry_hash FROM files f JOIN models m ON m.id = f.model_id - WHERE m.deleted_at IS NULL AND f.size > 0 + WHERE m.deleted_at IS NULL AND f.size > 0 AND f.format = ANY($1) + UNION ALL + SELECT 'e' || e.id, f.model_id, e.sha256, e.geometry_hash + FROM archive_entries e JOIN files f ON f.id = e.file_id JOIN models m ON m.id = f.model_id + WHERE m.deleted_at IS NULL AND e.size > 0 AND e.format = ANY($2) ) SELECT * FROM live WHERE sha256 IN (SELECT sha256 FROM live WHERE sha256 IS NOT NULL GROUP BY sha256 HAVING COUNT(DISTINCT model_id) > 1) OR geometry_hash IN (SELECT geometry_hash FROM live WHERE geometry_hash IS NOT NULL - GROUP BY geometry_hash HAVING COUNT(DISTINCT model_id) > 1)`); + GROUP BY geometry_hash HAVING COUNT(DISTINCT model_id) > 1)`, + [[...MODEL_FORMATS, ...ARCHIVE_FORMATS], MODEL_FORMATS]); return rows; } +// How many duplicate-matchable items (model files, archives, models inside +// archives) each model holds: { modelId: n }. +async function matchableCounts(modelIds) { + if (!modelIds.length) return {}; + const { rows } = await db.query(` + SELECT m.id, + (SELECT COUNT(*) FROM files f WHERE f.model_id = m.id AND f.format = ANY($2)) + + (SELECT COUNT(*) FROM archive_entries e JOIN files f ON f.id = e.file_id + WHERE f.model_id = m.id AND e.format = ANY($3)) AS n + FROM models m WHERE m.id = ANY($1::int[])`, + [modelIds, [...MODEL_FORMATS, ...ARCHIVE_FORMATS], MODEL_FORMATS]); + return Object.fromEntries(rows.map((r) => [r.id, Number(r.n)])); +} + // Keep one model of a duplicate group: fold the others' tags, collections and // notes into it, then move the others to the trash. async function mergeAndTrash(keepId, otherIds) { @@ -202,6 +269,8 @@ async function dismissedDuplicateSignatures() { async function addFiles(modelId, files) { await db.transaction(async (client) => { for (const file of files) await insertFile(client, modelId, file); + // New files may give the background worker something to make a thumbnail from. + await client.query('UPDATE models SET thumbnail_status = NULL WHERE id = $1', [modelId]); }); return getModel(modelId); } @@ -407,7 +476,8 @@ module.exports = { getAllModels, getModel, getModelsByIds, findModelBySource, createModel, updateModel, deleteModel, addFiles, updateFileConversion, trashModels, restoreModel, trashedModelIds, countTrashed, - duplicateCandidateFiles, mergeAndTrash, dismissDuplicateGroup, dismissedDuplicateSignatures, + duplicateCandidateFiles, matchableCounts, mergeAndTrash, dismissDuplicateGroup, dismissedDuplicateSignatures, + getFile, getArchiveEntries, getArchiveEntry, replaceArchiveEntries, getAllTags, getAllProjects, getProject, createProject, updateProject, deleteProject, getSetting, setSetting, diff --git a/lib/thumbnails.js b/lib/thumbnails.js new file mode 100644 index 0000000..d980ab1 --- /dev/null +++ b/lib/thumbnails.js @@ -0,0 +1,104 @@ +// Background thumbnails for models that don't have one. Tries, in order: an +// embedded 3MF preview, a picture uploaded with the model, a 3MF preview or +// picture inside one of its archives, then a server-side render of an +// STL/OBJ/3MF (or one inside an archive). Models whose only 3D files are +// FBX/STEP/3DM are left to the browser, which can render those. + +const fs = require('fs'); +const path = require('path'); +const db = require('./db'); +const archives = require('./archives'); +const fileProcessor = require('./fileProcessor'); +const { runScript } = require('./python'); + +const THUMBNAIL_DIR = './thumbnails'; +const PICTURES = ['png', 'jpg', 'jpeg', 'gif', 'webp']; +const RENDERABLE = ['3mf', 'stl', 'obj']; // 3MF first: slicer projects usually embed a preview +const MAX_ATTEMPTS = 3; // per kind of source, so a huge pack doesn't stall the queue +const BATCH = 5; + +// Models still waiting: no thumbnail, not given up on, and archives already +// indexed (so what's inside them can be considered). +const PENDING = ` + m.thumbnail_path IS NULL AND m.thumbnail_status IS NULL AND m.deleted_at IS NULL + AND NOT EXISTS (SELECT 1 FROM files f WHERE f.model_id = m.id AND f.format = ANY($1) AND f.archive_status IS NULL)`; + +async function render(input, base) { + const out = path.join(THUMBNAIL_DIR, `${base}.png`); + fs.mkdirSync(THUMBNAIL_DIR, { recursive: true }); + const { err } = await runScript('render_thumbnail.py', [input, path.resolve(out)], + { timeout: 180000, isBackground: true }); + if (err && err.code === 'ENOENT') throw err; + return !err && fs.existsSync(out) ? out : null; +} + +// Returns a path under thumbnails/ or null. +async function thumbnailFor(modelId) { + const { rows: files } = await db.query( + 'SELECT id, file_path, format FROM files WHERE model_id = $1 ORDER BY id', [modelId]); + const base = `model-${modelId}-${Date.now()}`; + + for (const f of files.filter((x) => x.format === '3mf').slice(0, MAX_ATTEMPTS)) { + const out = fileProcessor.extract3mfThumbnail(path.resolve(f.file_path), base); + if (out) return out; + } + const picture = files.find((f) => PICTURES.includes(f.format)); + if (picture) return fileProcessor.copyAsThumbnail(path.resolve(picture.file_path), base); + + const archiveFiles = files.filter((f) => fileProcessor.ARCHIVE_FORMATS.includes(f.format)); + const { rows: entries } = archiveFiles.length + ? await db.query('SELECT id, file_id, path, format FROM archive_entries WHERE file_id = ANY($1::int[]) ORDER BY path', + [archiveFiles.map((f) => f.id)]) + : { rows: [] }; + const extract = (entry) => archives.extractEntry(archiveFiles.find((f) => f.id === entry.file_id), entry); + + for (const e of entries.filter((x) => x.format === '3mf').slice(0, MAX_ATTEMPTS)) { + const out = fileProcessor.extract3mfThumbnail(await extract(e), base); + if (out) return out; + } + const entryPicture = entries.find((e) => PICTURES.includes(e.format)); + if (entryPicture) return fileProcessor.copyAsThumbnail(await extract(entryPicture), base); + + const sources = [ + ...RENDERABLE.flatMap((fmt) => files.filter((f) => f.format === fmt).map((f) => () => path.resolve(f.file_path))), + ...RENDERABLE.flatMap((fmt) => entries.filter((e) => e.format === fmt).map((e) => () => extract(e))), + ]; + for (const source of sources.slice(0, MAX_ATTEMPTS)) { + const out = await render(await source(), base); + if (out) return out; + } + return null; +} + +// One batch; returns whether there was anything to do. +async function generate() { + const { rows } = await db.query( + `SELECT m.id FROM models m WHERE ${PENDING} ORDER BY m.id LIMIT $2`, [fileProcessor.ARCHIVE_FORMATS, BATCH]); + for (const { id } of rows) { + let out = null; + let status = 'none'; + try { + out = await thumbnailFor(id); + } catch (err) { + console.warn(`Thumbnail failed for model ${id}:`, err.message); + status = 'error'; + } + if (out) { + // Don't overwrite a thumbnail set meanwhile (e.g. rendered by a browser). + const { rowCount } = await db.query( + 'UPDATE models SET thumbnail_path = $2 WHERE id = $1 AND thumbnail_path IS NULL', [id, out]); + if (rowCount === 0) fs.rmSync(out, { force: true }); + } else { + await db.query('UPDATE models SET thumbnail_status = $2 WHERE id = $1', [id, status]); + } + } + return rows.length > 0; +} + +async function pendingCount() { + const { rows } = await db.query(`SELECT COUNT(*)::int AS n FROM models m WHERE ${PENDING}`, + [fileProcessor.ARCHIVE_FORMATS]); + return rows[0].n; +} + +module.exports = { generate, pendingCount }; diff --git a/package-lock.json b/package-lock.json index f1e5660..c829c8d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "7z-wasm": "^1.2.0", "adm-zip": "^0.5.17", "bcryptjs": "^3.0.3", "cookie-parser": "^1.4.7", @@ -18,12 +19,42 @@ "multer": "^1.4.5-lts.1", "node-stl": "^0.7.0", "nodemailer": "^9.0.1", - "pg": "^8.22.0" + "occt-import-js": "^0.0.23", + "pg": "^8.22.0", + "rhino3dm": "^8.32.2", + "tar": "^7.5.22" }, "devDependencies": { "nodemon": "^3.0.2" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/7z-wasm": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/7z-wasm/-/7z-wasm-1.2.0.tgz", + "integrity": "sha512-AZjtWXFleAe5nQ27JYISs70cICjO0zZr4LU45I9pXKcrhG0qiEWgq3Vjwr9JHHihFLuqgc7uSHYVn/z3vXCjdw==", + "license": "SEE LICENSE IN License.txt", + "dependencies": { + "readline-sync": "^1.4.10" + }, + "bin": { + "7z-wasm": "bin/cli" + }, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -240,6 +271,15 @@ "fsevents": "~2.3.2" } }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/concat-stream": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", @@ -945,6 +985,27 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/mkdirp": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", @@ -1091,6 +1152,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/occt-import-js": { + "version": "0.0.23", + "resolved": "https://registry.npmjs.org/occt-import-js/-/occt-import-js-0.0.23.tgz", + "integrity": "sha512-RFfYQXYFX5C1mB1Aywm0ShcUKzXOr/VzTnlzhBSDJOR6YCAPt1HYCzeXWg1vwwjn/cUxwqRNhhtf1dlewoZYCQ==", + "license": "LGPL-2.1" + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -1358,6 +1425,21 @@ "node": ">=8.10.0" } }, + "node_modules/readline-sync": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz", + "integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==", + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/rhino3dm": { + "version": "8.32.2", + "resolved": "https://registry.npmjs.org/rhino3dm/-/rhino3dm-8.32.2.tgz", + "integrity": "sha512-Q3/5iHIKJeoE5doXrRy7C1vJer4F8uMYe8K6B8VQHqhGj5BCS1L3EkqNpKyUTERoHyKh7rMmiiZpw9iEUkCnPQ==", + "license": "MIT" + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1586,6 +1668,22 @@ "node": ">=4" } }, + "node_modules/tar": { + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -1685,6 +1783,15 @@ "engines": { "node": ">=0.4" } + }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } } } } diff --git a/package.json b/package.json index 91e1354..e348bd7 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "author": "", "license": "MIT", "dependencies": { + "7z-wasm": "^1.2.0", "adm-zip": "^0.5.17", "bcryptjs": "^3.0.3", "cookie-parser": "^1.4.7", @@ -26,7 +27,10 @@ "multer": "^1.4.5-lts.1", "node-stl": "^0.7.0", "nodemailer": "^9.0.1", - "pg": "^8.22.0" + "occt-import-js": "^0.0.23", + "pg": "^8.22.0", + "rhino3dm": "^8.32.2", + "tar": "^7.5.22" }, "devDependencies": { "nodemon": "^3.0.2" diff --git a/public/app.js b/public/app.js index ef0f2a5..70e3cb8 100644 --- a/public/app.js +++ b/public/app.js @@ -163,6 +163,7 @@ async function loadData() { renderModels(); updateCounts(); generateMissingThumbnails(); // fire-and-forget; updates cards as they finish + watchBackgroundWork(); // First-run onboarding: prompt for import sources once. if (config.setupComplete === false && !setupModal.classList.contains('active')) { @@ -173,24 +174,48 @@ async function loadData() { } } +// While the server is still fingerprinting or making thumbnails, check back +// every few seconds and reload once its progress changes, so new thumbnails +// and duplicates show up without a manual refresh. +let backgroundPoll = null; +function watchBackgroundWork() { + clearTimeout(backgroundPoll); + const pendingOf = (s) => (s.fingerprintsPending || 0) + (s.thumbnailsPending || 0); + const pending = pendingOf(stats); + if (!pending) return; + backgroundPoll = setTimeout(async () => { + try { + const next = await (await fetch('/api/library/stats')).json(); + if (pendingOf(next) !== pending || next.duplicateGroups !== stats.duplicateGroups) { + await loadData(); // re-arms the watch + return; + } + stats = next; + watchBackgroundWork(); + } catch { /* offline: try again on the next reload */ } + }, 5000); +} + async function loadTrash() { const res = await fetch('/api/models?view=trash'); trashModels = res.ok ? await res.json() : []; } -// Render thumbnails in the browser for models that don't have one yet, then -// persist them. Runs one at a time to avoid thrashing the GPU / WebGL contexts. +// The server makes STL/OBJ/3MF thumbnails in the background; the browser +// renders the formats only it can read, then saves them. One at a time, to +// avoid thrashing the GPU / WebGL contexts. +const BROWSER_THUMBNAIL_FORMATS = ['fbx', '3dm', 'step', 'stp']; const thumbAttempted = new Set(); async function generateMissingThumbnails() { if (!window.ModelViewer) return; for (const m of models) { if (m.thumbnailUrl || thumbAttempted.has(m.id)) continue; - const file = (m.files || []).find((f) => window.ModelViewer.RENDERABLE.includes(f.format)); + const file = (m.files || []).find((f) => BROWSER_THUMBNAIL_FORMATS.includes(f.format)); if (!file) continue; thumbAttempted.add(m.id); try { - const src = viewerSourceFor(file.id, file.format); - const image = await window.ModelViewer.snapshot(src.url, src.format, 512); + const src = viewerSourceFor({ kind: 'file', id: file.id, format: file.format }); + const image = await window.ModelViewer.snapshot(src.url, src.format, 512, { resolve: resolverFor(m) }); if (!image) continue; const res = await fetch(`/api/models/${m.id}/thumbnail`, { method: 'POST', @@ -484,7 +509,7 @@ function duplicateGroupHtml(g) { function duplicateCardHtml(g, m) { const recommended = m.id === g.recommendedId; - const unique = m.files.length - m.matchedFiles; + const unique = m.matchableFiles - m.matchedFiles; return `
${thumbHtml(m)}
@@ -492,7 +517,7 @@ function duplicateCardHtml(g, m) { ${recommended ? 'Recommended keep' : ''}
${escapeHtml(m.name)}
${escapeHtml(sourceLabel(sourceKey(m)))} • ${formatDate(m.date_added)}
-
${m.matchedFiles} of ${m.files.length} files match${unique ? ` • ${unique} unique` : ''}
+
${m.matchedFiles} of ${m.matchableFiles} files match${unique ? ` • ${unique} unique` : ''}
`; @@ -505,7 +530,7 @@ async function handleKeepDuplicate(signature, keepId) { const others = group.models.filter((m) => m.id !== keepId); let message = `Keep "${keep.name}" and move ${others.length} other model${others.length === 1 ? '' : 's'} to the trash?\n\n` + 'Their tags, collections and notes are merged into the one you keep.'; - const withUnique = others.filter((m) => m.files.length > m.matchedFiles); + const withUnique = others.filter((m) => m.matchableFiles > m.matchedFiles); if (withUnique.length) { message += `\n\nHeads up: ${withUnique.map((m) => `"${m.name}"`).join(', ')} also contain files that aren't duplicated — they go to the trash too.`; } @@ -700,6 +725,29 @@ function setupEventListeners() { }); document.getElementById('detailNotes').addEventListener('input', queueNotesSave); document.getElementById('detailNotes').addEventListener('blur', flushNotes); + document.getElementById('detailFileTabs').addEventListener('click', (e) => { + const tab = e.target.closest('.file-tab'); + const item = tab && detailItems.find((i) => itemKey(i) === tab.dataset.key); + if (!item) return; + renderViewerTabs(itemKey(item)); + loadViewer(item); + }); + document.getElementById('detailPlates').addEventListener('click', (e) => { + const chip = e.target.closest('.plate-chip'); + const item = chip && detailItems.find((i) => itemKey(i) === activeItemKey); + if (!item) return; + item.plate = chip.dataset.plate === 'all' ? null : Number(chip.dataset.plate); + loadViewer(item); + }); + document.getElementById('detailFileInfo').addEventListener('click', (e) => { + const btn = e.target.closest('[data-view-entry]'); + if (!btn) return; + const archiveId = Number(btn.dataset.archive); + const entry = (archiveEntryCache.get(archiveId) || []).find((x) => String(x.id) === btn.dataset.viewEntry); + if (!entry) return; + openViewerItem({ kind: 'entry', id: entry.id, format: entry.format, name: entry.path.split('/').pop(), archiveId }); + document.getElementById('detailViewer').scrollIntoView({ block: 'nearest', behavior: 'smooth' }); + }); setupTagAutocomplete(document.getElementById('fileTags')); setupTagAutocomplete(document.getElementById('editTags')); @@ -924,38 +972,126 @@ function openModelDetail(modelId) {
${escapeHtml(f.originalName)} (${formatFileSize(f.size)}${f.volume ? `, ${Number(f.volume).toFixed(2)} cm³` : ''}) download -
`).join(''); + + ${ARCHIVE_FORMATS.includes(f.format) ? `
Reading archive…
` : ''}`).join(''); + model.files.filter((f) => ARCHIVE_FORMATS.includes(f.format)).forEach((f) => loadArchiveEntries(model.id, f.id)); } - // Viewer tabs - const renderable = model.files.filter((f) => window.ModelViewer.RENDERABLE.includes(f.format)); - const tabs = document.getElementById('detailFileTabs'); - tabs.innerHTML = renderable.map((f, i) => ` - `).join(''); - tabs.querySelectorAll('.file-tab').forEach((tab) => { - tab.addEventListener('click', () => { - tabs.querySelectorAll('.file-tab').forEach((t) => t.classList.remove('active')); - tab.classList.add('active'); - loadViewer(tab.dataset.fileId, tab.dataset.format); - }); - }); + renderProjectPanels(null); + + // Viewer tabs: 3D files first, then pictures/PDFs/text (entries inside + // archives get a tab when opened from the file list). + const { RENDERABLE, DOCUMENT_FORMATS } = window.ModelViewer; + detailItems = [ + ...model.files.filter((f) => RENDERABLE.includes(f.format)), + ...model.files.filter((f) => DOCUMENT_FORMATS.includes(f.format) && f.format !== 'mtl'), + ].map((f) => ({ kind: 'file', id: f.id, format: f.format, name: f.originalName })); + renderViewerTabs(detailItems.length ? itemKey(detailItems[0]) : null); fileDetailModal.classList.add('active'); currentDetailFile = null; - if (renderable.length) { - loadViewer(renderable[0].id, renderable[0].format); + if (detailItems.length) { + loadViewer(detailItems[0]); } else { + // A model that's only archives previews a file from inside once they're listed. + const hasArchive = model.files.some((f) => ARCHIVE_FORMATS.includes(f.format)); document.getElementById('regenThumbBtn').style.display = 'none'; - window.ModelViewer.render(document.getElementById('detailViewer'), '', 'none', { fallbackImage: currentDetailThumb }); + window.ModelViewer.render(document.getElementById('detailViewer'), '', 'none', { + fallbackImage: currentDetailThumb, + message: hasArchive ? 'Opening a file from the archive…' : 'Nothing here can be previewed yet — add files below.', + }); } } -// Maps a file to what the viewer should load: 3MF goes through the server's -// GLB conversion; everything else is served raw. -function viewerSourceFor(fileId, format) { - return format === '3mf' - ? { url: `/api/files/${fileId}/glb`, format: 'glb' } - : { url: `/api/files/${fileId}/raw`, format }; +// ---------------------------------------------------------------- viewer items +// An item is something the viewer can show: { kind: 'file' | 'entry', id, format, name, archiveId? }. +const ARCHIVE_FORMATS = ['zip', '7z', 'rar', 'tar', 'tgz']; +let detailItems = []; +let activeItemKey = null; +const archiveEntryCache = new Map(); // archive file id -> entries +const itemKey = (item) => `${item.kind}:${item.id}`; + +function renderViewerTabs(activeKey) { + activeItemKey = activeKey; + document.getElementById('detailFileTabs').innerHTML = detailItems.map((item) => ` + `).join(''); +} + +function openViewerItem(item) { + if (!detailItems.some((i) => itemKey(i) === itemKey(item))) detailItems.push(item); + renderViewerTabs(itemKey(item)); + loadViewer(item); +} + +// What the viewer should load: 3MF goes through the server's GLB conversion, +// documents are served inline, everything else raw. +function viewerSourceFor(item) { + const base = `/api/${item.kind === 'entry' ? 'entries' : 'files'}/${item.id}`; + if (item.format === '3mf') return { url: `${base}/glb${item.plate ? `?plate=${item.plate}` : ''}`, format: 'glb' }; + if (window.ModelViewer.DOCUMENT_FORMATS.includes(item.format)) return { url: `${base}/raw?inline=1`, format: item.format }; + return { url: `${base}/raw`, format: item.format }; +} + +// Finds files shipped alongside a model by name — OBJ materials, FBX/MTL +// textures. Entries of the same archive win, then the model's own files. +function resolverFor(model, archiveEntries = null) { + const byName = new Map(); + const key = (name) => String(name).split(/[\\/]/).pop().toLowerCase(); + const add = (name, url) => { if (!byName.has(key(name))) byName.set(key(name), url); }; + (archiveEntries || []).forEach((e) => add(e.path, `/api/entries/${e.id}/raw`)); + ((model && model.files) || []).forEach((f) => add(f.originalName, `/api/files/${f.id}/raw`)); + return (name) => byName.get(key(name)) || null; +} + +// For a model with nothing to preview directly: open the most useful file +// inside its archive — a slicer project, then other 3D files, then pictures/PDFs. +const ENTRY_PREVIEW_ORDER = ['3mf', 'stl', 'obj', 'step', 'stp', 'fbx', '3dm', 'jpg', 'jpeg', 'png', 'webp', 'gif', 'pdf']; +function previewFirstEntry(archiveId, entries) { + const rank = (e) => (ENTRY_PREVIEW_ORDER.includes(e.format) ? ENTRY_PREVIEW_ORDER.indexOf(e.format) : Infinity); + const best = [...entries].sort((a, b) => rank(a) - rank(b))[0]; + if (!best || rank(best) === Infinity) { + window.ModelViewer.render(document.getElementById('detailViewer'), '', 'none', + { fallbackImage: currentDetailThumb, message: 'Nothing inside this archive can be previewed.' }); + return; + } + openViewerItem({ kind: 'entry', id: best.id, format: best.format, name: best.path.split('/').pop(), archiveId }); +} + +async function loadArchiveEntries(modelId, fileId) { + const container = () => (fileDetailModal.dataset.currentModelId === String(modelId) + ? document.querySelector(`[data-entries-for="${fileId}"]`) : null); + try { + const res = await fetch(`/api/files/${fileId}/entries`); + const data = await res.json(); + if (!res.ok) throw new Error(data.error); + if (!container()) return; + if (data.status === 'pending') { + setTimeout(() => { if (container()) loadArchiveEntries(modelId, fileId); }, 2000); + return; + } + if (data.status === 'error') { + container().innerHTML = `
Couldn't read this archive: ${escapeHtml(data.error || 'unknown error')}
`; + return; + } + archiveEntryCache.set(fileId, data.entries); + const { PREVIEWABLE } = window.ModelViewer; + container().innerHTML = `
${data.entries.length} file${data.entries.length === 1 ? '' : 's'} inside
` + + data.entries.map((e) => { + const name = e.path.split('/').pop(); + const folder = e.path.replace(/^\.\//, '').slice(0, -name.length); + return ` +
+ ${escapeHtml(name)}${folder ? `${escapeHtml(folder)}` : ''} ${formatFileSize(e.size)} + + ${PREVIEWABLE.includes(e.format) && e.format !== 'mtl' ? `` : ''} + download + +
`; + }).join(''); + if (!detailItems.length) previewFirstEntry(fileId, data.entries); + } catch (err) { + if (container()) container().innerHTML = `
Couldn't list this archive: ${escapeHtml(err.message)}
`; + } } function renderDetailLicense(model) { @@ -1068,11 +1204,66 @@ async function handleRefreshMetadata() { } } -function loadViewer(fileId, format) { - const src = viewerSourceFor(fileId, format); - currentDetailFile = { id: fileId, viewerUrl: src.url, viewerFormat: src.format }; - document.getElementById('regenThumbBtn').style.display = ''; - window.ModelViewer.render(document.getElementById('detailViewer'), src.url, src.format, { fallbackImage: currentDetailThumb }); +async function loadViewer(item) { + if (item.format === '3mf') { + await loadProjectInfo(item); + if (activeItemKey !== itemKey(item)) return; // another tab was opened meanwhile + } + renderProjectPanels(item); + const src = viewerSourceFor(item); + const model = findModel(fileDetailModal.dataset.currentModelId); + const resolve = resolverFor(model, item.kind === 'entry' ? archiveEntryCache.get(item.archiveId) : null); + const is3d = window.ModelViewer.MODEL_FORMATS.includes(src.format); + currentDetailFile = is3d ? { viewerUrl: src.url, viewerFormat: src.format, resolve } : null; + document.getElementById('regenThumbBtn').style.display = is3d ? '' : 'none'; + window.ModelViewer.render(document.getElementById('detailViewer'), src.url, src.format, + { fallbackImage: currentDetailThumb, resolve }); +} + +// Plates and filaments of a 3MF slicer project, fetched once per viewer item. +// Projects with several plates open on their first plate, like the slicer. +function loadProjectInfo(item) { + if (!item.projectPromise) { + item.projectPromise = fetch(`/api/${item.kind === 'entry' ? 'entries' : 'files'}/${item.id}/project`) + .then((res) => (res.ok ? res.json() : null)) + .catch(() => null) + .then((project) => { + item.project = project || { plates: [], filaments: [] }; + if (item.plate === undefined && item.project.plates.length > 1) item.plate = item.project.plates[0].index; + }); + } + return item.projectPromise; +} + +const HEX_COLOUR = /^#[0-9a-f]{6}([0-9a-f]{2})?$/i; + +function renderProjectPanels(item) { + const project = item && item.format === '3mf' ? item.project : null; + const platesEl = document.getElementById('detailPlates'); + if (!project || project.plates.length < 2) { + platesEl.hidden = true; + platesEl.innerHTML = ''; + } else { + platesEl.hidden = false; + platesEl.innerHTML = project.plates.map((p) => ` + `).join('') + + ``; + } + + const filaments = ((project && project.filaments) || []) + .map((f, i) => ({ ...f, slot: i + 1 })) + .filter((f) => HEX_COLOUR.test(f.colour || '')); + document.getElementById('detailFilamentsSection').hidden = filaments.length === 0; + document.getElementById('detailFilaments').innerHTML = filaments.map((f) => ` +
+ + Slot ${f.slot}${f.type ? ` · ${escapeHtml(f.type)}` : ''} + ${f.colour.slice(0, 7).toUpperCase()} +
`).join(''); } function updateCardThumbnail(modelId, thumbnailUrl) { @@ -1089,7 +1280,8 @@ async function handleRegenThumbnail() { btn.disabled = true; btn.textContent = 'Rendering…'; try { - const image = await window.ModelViewer.snapshot(currentDetailFile.viewerUrl, currentDetailFile.viewerFormat, 512); + const image = await window.ModelViewer.snapshot(currentDetailFile.viewerUrl, currentDetailFile.viewerFormat, 512, + { resolve: currentDetailFile.resolve }); const res = await fetch(`/api/models/${modelId}/thumbnail`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, diff --git a/public/index.html b/public/index.html index 92fd9c0..bbf2369 100644 --- a/public/index.html +++ b/public/index.html @@ -329,8 +329,8 @@
- - + +
@@ -409,6 +409,7 @@
+
@@ -442,13 +443,17 @@

Print Settings

+

Files

+ accept=".stl,.3mf,.obj,.fbx,.step,.stp,.3dm,.gcode,.gco,.zip,.7z,.rar,.tar,.tgz,.gz,.jpg,.jpeg,.png,.gif,.webp,.bmp,.svg,.pdf,.txt,.md,.mtl">
diff --git a/public/styles.css b/public/styles.css index dbf8fd0..4058bc2 100644 --- a/public/styles.css +++ b/public/styles.css @@ -205,7 +205,8 @@ label { display: block; font-size: 13px; font-weight: 500; color: var(--text-mut .form-actions { display: flex; justify-content: flex-end; gap: 10px; margin-top: 22px; } /* ---- model detail ---- */ -.file-detail-content { display: grid; grid-template-columns: 1.1fr 1fr; gap: 26px; } +/* minmax(0, …) keeps a long file name from widening its column and crushing the viewer */ +.file-detail-content { display: grid; grid-template-columns: minmax(0, 1.1fr) minmax(0, 1fr); gap: 26px; } .file-detail-preview { display: flex; flex-direction: column; gap: 10px; } .model-viewer { width: 100%; height: 380px; background: radial-gradient(circle at 50% 40%, #232730 0%, #131519 100%); border: 1px solid var(--border); border-radius: var(--radius); display: flex; align-items: center; justify-content: center; } .model-viewer canvas { display: block; border-radius: var(--radius); } @@ -214,8 +215,34 @@ label { display: block; font-size: 13px; font-weight: 500; color: var(--text-mut .viewer-fallback img { max-width: 100%; max-height: 320px; object-fit: contain; border-radius: var(--radius-sm); } .viewer-fallback span { color: var(--text-dim); font-size: 12px; text-align: center; } +.viewer-image { max-width: 100%; max-height: 100%; object-fit: contain; } +.viewer-pdf, .viewer-text { align-self: stretch; width: 100%; height: 100%; border: 0; border-radius: var(--radius); } +.viewer-pdf { background: #fff; } +.viewer-text { margin: 0; padding: 14px 16px; overflow: auto; font-family: var(--mono); font-size: 12px; line-height: 1.55; color: var(--text); white-space: pre-wrap; word-break: break-word; text-align: left; } + +.info-section .entry-list { margin: 2px 0 8px 10px; padding-left: 12px; border-left: 1px solid var(--border); max-height: 260px; overflow-y: auto; } +.info-section .entry-head { font-size: 12px; color: var(--text-dim); margin: 6px 0 2px; } +.info-section .entry-row { display: flex; justify-content: space-between; align-items: center; gap: 10px; padding: 3px 0; font-size: 13px; color: var(--text-muted); } +.entry-row > span:first-child { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; } +.entry-row em { color: var(--text-dim); font-style: normal; font-family: var(--mono); font-size: 11px; margin-left: 4px; } +.entry-row .entry-folder { color: var(--text-dim); margin-left: 4px; } +.row-actions { display: flex; gap: 12px; flex-shrink: 0; align-items: center; } +.link-btn { background: none; border: none; color: var(--accent); font: inherit; cursor: pointer; padding: 0; } +.link-btn:hover { text-decoration: underline; } + +.plate-chips { display: flex; flex-wrap: wrap; gap: 6px; } +.plate-chip { display: flex; align-items: center; gap: 6px; min-height: 30px; border: 1px solid var(--border); background: var(--surface-2); color: var(--text-muted); padding: 2px 10px 2px 3px; border-radius: var(--radius-sm); font-family: inherit; font-size: 12px; cursor: pointer; } +.plate-chip:not(:has(img)) { padding-left: 10px; } +.plate-chip img { width: 28px; height: 28px; object-fit: cover; border-radius: 4px; background: var(--surface-3); } +.plate-chip:hover { color: var(--text); border-color: var(--border-strong); } +.plate-chip.active { border-color: var(--accent); color: var(--accent); background: var(--accent-soft); } +.info-section .filament-list { display: flex; flex-direction: column; gap: 6px; } +.info-section .filament { display: flex; align-items: center; gap: 8px; font-size: 13px; } +.filament em { font-style: normal; font-family: var(--mono); font-size: 11px; color: var(--text-dim); } +.swatch { width: 16px; height: 16px; border-radius: 50%; border: 1px solid rgba(255, 255, 255, 0.25); flex-shrink: 0; } + .file-tabs { display: flex; flex-wrap: wrap; gap: 6px; } -.file-tab { border: 1px solid var(--border); background: var(--surface-2); color: var(--text-muted); padding: 5px 11px; border-radius: var(--radius-sm); font-size: 12px; cursor: pointer; font-family: inherit; } +.file-tab { border: 1px solid var(--border); background: var(--surface-2); color: var(--text-muted); padding: 5px 11px; border-radius: var(--radius-sm); font-size: 12px; cursor: pointer; font-family: inherit; max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .file-tab:hover { color: var(--text); } .file-tab.active { background: var(--accent); color: var(--accent-text); border-color: var(--accent); } @@ -228,6 +255,7 @@ label { display: block; font-size: 13px; font-weight: 500; color: var(--text-mut .file-row { display: flex; justify-content: space-between; align-items: center; gap: 10px; padding: 7px 0; border-bottom: 1px solid var(--border); font-size: 14px; color: var(--text-muted); } .file-row:last-child { border-bottom: none; } .file-row em { color: var(--text-dim); font-style: normal; font-family: var(--mono); font-size: 12px; } +.file-row > span:first-child { min-width: 0; overflow-wrap: anywhere; } /* ---- import / settings ---- */ .import-status { margin-top: 14px; color: var(--text-muted); font-size: 14px; min-height: 20px; } diff --git a/public/viewer.js b/public/viewer.js index 922973e..b677797 100644 --- a/public/viewer.js +++ b/public/viewer.js @@ -1,16 +1,27 @@ -// In-browser 3D preview for STL / 3MF / OBJ files, loaded as an ES module. +// In-browser preview for 3D models (STL, OBJ + MTL, 3MF via GLB, FBX, STEP, 3DM) +// and for the pictures, PDFs and text files that come with them. import * as THREE from 'three'; import { OrbitControls } from 'three/addons/controls/OrbitControls.js'; import { STLLoader } from 'three/addons/loaders/STLLoader.js'; import { OBJLoader } from 'three/addons/loaders/OBJLoader.js'; -import { ThreeMFLoader } from 'three/addons/loaders/3MFLoader.js'; +import { MTLLoader } from 'three/addons/loaders/MTLLoader.js'; +import { FBXLoader } from 'three/addons/loaders/FBXLoader.js'; import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; +import { Rhino3dmLoader } from 'three/addons/loaders/3DMLoader.js'; + +// Formats the 3D renderer draws directly (3MF arrives as a server-converted GLB). +const MODEL_FORMATS = ['stl', 'obj', 'glb', 'fbx', '3dm', 'step', 'stp']; +const IMAGE_FORMATS = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'svg']; +const TEXT_FORMATS = ['txt', 'md', 'mtl']; +const DOCUMENT_FORMATS = [...IMAGE_FORMATS, 'pdf', ...TEXT_FORMATS]; +// What the app offers a preview for. +const RENDERABLE = ['stl', '3mf', 'obj', 'fbx', '3dm', 'step', 'stp']; +const PREVIEWABLE = [...RENDERABLE, ...DOCUMENT_FORMATS]; +// Formats authored Z-up; turned to sit upright in three.js's Y-up world. +const Z_UP = ['stl', 'glb', '3dm', 'step', 'stp']; -// Formats the renderer can draw directly. 3MF is rendered via server-converted -// GLB (see RENDERABLE for what the app treats as previewable input). -const SUPPORTED = ['stl', 'obj', 'glb']; -const RENDERABLE = ['stl', '3mf', 'obj']; let active = null; // { renderer, controls, rafId } +let renderToken = 0; // bumps on every render, so a slow earlier load can't draw over a newer one function dispose() { if (!active) return; @@ -26,8 +37,7 @@ function message(container, text) { container.innerHTML = `
${text}
`; } -// When 3D rendering isn't possible (e.g. Bambu/MakerWorld 3MF project files that -// the loader can't parse), show the model's preview image instead of an error. +// When 3D rendering isn't possible, show the model's preview image instead of an error. function showFallback(container, image, text) { if (image) { container.innerHTML = @@ -37,33 +47,172 @@ function showFallback(container, image, text) { } } -async function loadObject(url, format) { +async function fetchOk(url) { + const res = await fetch(url); + if (!res.ok) throw new Error(`HTTP ${res.status} loading ${url}`); + return res; +} + +const basename = (p) => String(p).split(/[\\/]/).pop(); + +function defaultMaterial(color) { + return new THREE.MeshStandardMaterial({ color: color || 0xc4cad6, metalness: 0.15, roughness: 0.65 }); +} + +// A loading manager that points material and texture references ("wood.png", +// "../tex/wood.png") at the files shipped alongside the model. `own` is the +// URL being loaded itself, which is left alone. +function managerFor(resolve, own) { + const manager = new THREE.LoadingManager(); + if (resolve) { + manager.setURLModifier((url) => { + if (url === own || url.startsWith('blob:') || url.startsWith('data:')) return url; + return resolve(decodeURIComponent(basename(url.split('?')[0]))) || url; + }); + } + return manager; +} + +async function loadObj(url, resolve) { + const text = await (await fetchOk(url)).text(); + const loader = new OBJLoader(); + const mtllib = /^mtllib\s+(.+)$/m.exec(text); + const mtlUrl = mtllib && resolve ? resolve(basename(mtllib[1].trim())) : null; + let textured = false; + if (mtlUrl) { + try { + const materials = await new MTLLoader(managerFor(resolve, mtlUrl)).loadAsync(mtlUrl); + materials.preload(); + loader.setMaterials(materials); + textured = true; + } catch { /* fall back to plain grey */ } + } + const object = loader.parse(text); + if (!textured) object.traverse((c) => { if (c.isMesh) c.material = defaultMaterial(); }); + return object; +} + +// OpenCascade (WASM) is large, so it's fetched the first time a STEP file is opened. +let occtPromise = null; +function occt() { + if (!occtPromise) { + occtPromise = new Promise((resolve, reject) => { + const script = document.createElement('script'); + script.src = '/vendor/occt/occt-import-js.js'; + script.onload = () => resolve(window.occtimportjs({ locateFile: (name) => `/vendor/occt/${name}` })); + script.onerror = () => { occtPromise = null; reject(new Error('Could not load the STEP reader')); }; + document.head.appendChild(script); + }); + } + return occtPromise; +} + +async function loadStep(url) { + const [reader, buffer] = await Promise.all([occt(), fetchOk(url).then((r) => r.arrayBuffer())]); + const result = reader.ReadStepFile(new Uint8Array(buffer), null); + if (!result.success || !result.meshes.length) throw new Error('No geometry in this STEP file'); + const group = new THREE.Group(); + for (const m of result.meshes) { + const geometry = new THREE.BufferGeometry(); + geometry.setAttribute('position', new THREE.Float32BufferAttribute(m.attributes.position.array, 3)); + if (m.attributes.normal) geometry.setAttribute('normal', new THREE.Float32BufferAttribute(m.attributes.normal.array, 3)); + geometry.setIndex(Array.from(m.index.array)); + if (!m.attributes.normal) geometry.computeVertexNormals(); + const color = m.color ? new THREE.Color(m.color[0], m.color[1], m.color[2]) : null; + group.add(new THREE.Mesh(geometry, defaultMaterial(color))); + } + return group; +} + +async function load3dm(url) { + const loader = new Rhino3dmLoader().setLibraryPath('/vendor/rhino3dm/'); + try { + return await loader.loadAsync(url); + } finally { + loader.dispose(); + } +} + +async function loadObject(url, format, opts = {}) { + let object; if (format === 'stl') { const geometry = await new STLLoader().loadAsync(url); geometry.computeVertexNormals(); - const material = new THREE.MeshStandardMaterial({ color: 0xc4cad6, metalness: 0.15, roughness: 0.65 }); - const mesh = new THREE.Mesh(geometry, material); - mesh.rotation.x = -Math.PI / 2; // STL is typically Z-up; sit it flat - return mesh; + object = new THREE.Mesh(geometry, defaultMaterial()); + } else if (format === 'obj') { + object = await loadObj(url, opts.resolve); + } else if (format === 'fbx') { + object = await new FBXLoader(managerFor(opts.resolve, url)).loadAsync(url); + } else if (format === 'glb') { + object = (await new GLTFLoader().loadAsync(url)).scene; + } else if (format === '3dm') { + object = await load3dm(url); + } else if (format === 'step' || format === 'stp') { + object = await loadStep(url); + } else { + throw new Error(`Unsupported format ${format}`); } - if (format === 'obj') return new OBJLoader().loadAsync(url); - if (format === '3mf') return new ThreeMFLoader().loadAsync(url); - if (format === 'glb') { - const gltf = await new GLTFLoader().loadAsync(url); - gltf.scene.rotation.x = -Math.PI / 2; // 3MF/trimesh is Z-up; sit it upright - return gltf.scene; + if (Z_UP.includes(format)) object.rotation.x = -Math.PI / 2; + return object; +} + +async function renderDocument(container, url, format, token) { + if (IMAGE_FORMATS.includes(format)) { + const img = document.createElement('img'); + img.className = 'viewer-image'; + img.alt = ''; + img.src = url; + container.appendChild(img); + } else if (format === 'pdf') { + const frame = document.createElement('iframe'); + frame.className = 'viewer-pdf'; + frame.title = 'PDF preview'; + frame.src = url; + container.appendChild(frame); + } else { + let text; + try { + text = await (await fetchOk(url)).text(); + } catch { + text = 'Could not load this file.'; + } + if (token !== renderToken) return; + const pre = document.createElement('pre'); + pre.className = 'viewer-text'; + pre.textContent = text.length > 200000 ? `${text.slice(0, 200000)}\n…` : text; + container.appendChild(pre); } - throw new Error(`Unsupported format ${format}`); } async function render(container, url, format, opts = {}) { dispose(); + const token = ++renderToken; container.innerHTML = ''; - if (!SUPPORTED.includes(format)) { - showFallback(container, opts.fallbackImage, `No 3D preview for .${format} files.`); + if (DOCUMENT_FORMATS.includes(format)) { + await renderDocument(container, url, format, token); + return; + } + if (!MODEL_FORMATS.includes(format)) { + showFallback(container, opts.fallbackImage, opts.message || `No 3D preview for .${format} files.`); return; } + message(container, 'Loading preview…'); + let object; + try { + object = await loadObject(url, format, opts); + } catch (err) { + console.warn('Preview failed:', err); + if (token !== renderToken) return; + const note = format === 'glb' + ? 'Interactive 3D isn’t available for this 3MF.' + : 'This file couldn’t be rendered in 3D.'; + showFallback(container, opts.fallbackImage, note); + return; + } + if (token !== renderToken) return; // another file was opened meanwhile + + container.innerHTML = ''; const width = container.clientWidth || 600; const height = container.clientHeight || 420; @@ -81,20 +230,6 @@ async function render(container, url, format, opts = {}) { const controls = new OrbitControls(camera, renderer.domElement); controls.enableDamping = true; - let object; - try { - object = await loadObject(url, format); - } catch (err) { - renderer.dispose(); - renderer.forceContextLoss(); - if (renderer.domElement.parentNode) renderer.domElement.parentNode.removeChild(renderer.domElement); - const note = format === '3mf' - ? 'Interactive 3D isn’t supported for this 3MF (Bambu/MakerWorld project file).' - : 'This file couldn’t be rendered in 3D.'; - showFallback(container, opts.fallbackImage, note); - return; - } - // Center the object and frame the camera around it. const box = new THREE.Box3().setFromObject(object); const size = box.getSize(new THREE.Vector3()); @@ -123,8 +258,8 @@ async function render(container, url, format, opts = {}) { // Render a model once to an offscreen canvas and return a PNG data URL. // Used to generate card thumbnails. Transparent background so it composites // over the card surface. -async function snapshot(url, format, size = 512) { - if (!SUPPORTED.includes(format)) return null; +async function snapshot(url, format, size = 512, opts = {}) { + if (!MODEL_FORMATS.includes(format)) return null; const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true, preserveDrawingBuffer: true }); renderer.setSize(size, size); @@ -137,7 +272,7 @@ async function snapshot(url, format, size = 512) { const fill = new THREE.DirectionalLight(0xffffff, 0.35); fill.position.set(-1, 0.5, -1); scene.add(fill); try { - const object = await loadObject(url, format); + const object = await loadObject(url, format, opts); const box = new THREE.Box3().setFromObject(object); const dim = box.getSize(new THREE.Vector3()); const center = box.getCenter(new THREE.Vector3()); @@ -159,4 +294,4 @@ async function snapshot(url, format, size = 512) { } } -window.ModelViewer = { render, dispose, snapshot, SUPPORTED, RENDERABLE }; +window.ModelViewer = { render, dispose, snapshot, MODEL_FORMATS, DOCUMENT_FORMATS, RENDERABLE, PREVIEWABLE }; diff --git a/server.js b/server.js index 8f266e6..d110fca 100644 --- a/server.js +++ b/server.js @@ -21,6 +21,8 @@ const auth = require('./lib/auth'); const mailer = require('./lib/mailer'); const fingerprint = require('./lib/fingerprint'); const duplicates = require('./lib/duplicates'); +const archives = require('./lib/archives'); +const thumbnails = require('./lib/thumbnails'); const app = express(); const PORT = process.env.PORT || 3000; @@ -31,6 +33,9 @@ app.use(express.json({ limit: '8mb' })); // generous for base64 thumbnail PNGs app.use(cookieParser()); app.use(express.static('public')); app.use('/thumbnails', express.static('thumbnails')); +// In-browser readers for STEP (OpenCascade) and Rhino 3DM, served from node_modules. +app.use('/vendor/occt', express.static(path.join(__dirname, 'node_modules/occt-import-js/dist'))); +app.use('/vendor/rhino3dm', express.static(path.join(__dirname, 'node_modules/rhino3dm'))); // Gate the API: everything under /api requires auth except the bootstrap config // and the auth endpoints themselves. @@ -57,11 +62,14 @@ function describeFile(filePath, fileName, originalName) { return { fileName, filePath, originalName, format, size, ...meta }; } -// Choose a thumbnail for a model from its files (embedded 3MF preview, if any). +// Choose a thumbnail for a model from its files: an embedded 3MF preview, else +// a picture that came with the model. function pickThumbnail(files) { const threeMf = files.find((f) => f.format === '3mf'); - if (!threeMf) return null; - return fileProcessor.extract3mfThumbnail(threeMf.filePath, path.parse(threeMf.fileName).name); + const embedded = threeMf && fileProcessor.extract3mfThumbnail(threeMf.filePath, path.parse(threeMf.fileName).name); + if (embedded) return embedded; + const image = files.find((f) => ['png', 'jpg', 'jpeg', 'gif', 'webp'].includes(f.format)); + return image ? fileProcessor.copyAsThumbnail(image.filePath, path.parse(image.fileName).name) : null; } // Transform a stored model into the public shape (thumbnail path -> URL). @@ -82,7 +90,7 @@ const upload = multer({ storage, fileFilter: (req, file, cb) => { if (fileProcessor.isAllowed(file.originalname)) cb(null, true); - else cb(new Error(`Unsupported file type. Allowed: ${fileProcessor.MODEL_FORMATS.join(', ')}`)); + else cb(new Error(`Unsupported file type. Allowed: ${fileProcessor.ALLOWED_FORMATS.join(', ')}`)); }, }); @@ -474,6 +482,8 @@ app.delete('/api/trash', async (req, res, next) => { async function purgeModel(id) { const removed = await store.deleteModel(id); if (!removed) return; + removed.fileIds.forEach(archives.removeExtracted); + removeDerivedFiles(removed.fileIds, removed.entryIds); for (const p of [...removed.filePaths, ...removed.glbPaths, removed.thumbnailPath]) { if (!p) continue; try { fs.unlinkSync(p); } catch (err) { @@ -534,45 +544,179 @@ app.get('/api/models/:id/download', async (req, res, next) => { } catch (e) { next(e); } }); -// Serve a raw model file (used by the in-browser 3D viewer). +// Send a stored file as a download, or with ?inline=1 for display in the page +// (images, PDFs, text). SVGs are sandboxed so scripts inside them can't run. +function sendStored(req, res, absPath, name) { + res.set('X-Content-Type-Options', 'nosniff'); + const ext = path.extname(name).toLowerCase(); + if (ext === '.svg') res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; sandbox"); + if (req.query.inline !== '1') return res.download(absPath, name); + if (['.md', '.mtl', '.txt'].includes(ext)) res.type('text/plain'); + res.sendFile(absPath); +} + +// Serve a stored file (downloads, and the in-browser viewers). app.get('/api/files/:id/raw', async (req, res, next) => { try { - const { rows } = await db.query('SELECT file_path, original_name FROM files WHERE id = $1', [req.params.id]); - if (rows.length === 0) return res.status(404).json({ error: 'File not found' }); - const abs = path.resolve(rows[0].file_path); + const file = await store.getFile(req.params.id); + if (!file) return res.status(404).json({ error: 'File not found' }); + const abs = path.resolve(file.file_path); if (!fs.existsSync(abs)) return res.status(404).json({ error: 'File missing on disk' }); - res.download(abs, rows[0].original_name); + sendStored(req, res, abs, file.original_name); + } catch (e) { next(e); } +}); + +// What's inside an archive file (indexed in the background). +app.get('/api/files/:id/entries', async (req, res, next) => { + try { + const file = await store.getFile(req.params.id); + if (!file) return res.status(404).json({ error: 'File not found' }); + if (!fileProcessor.ARCHIVE_FORMATS.includes(file.format)) return res.status(400).json({ error: 'Not an archive' }); + res.json({ + status: file.archive_status || 'pending', + error: file.archive_error, + entries: await store.getArchiveEntries(file.id), + }); + } catch (e) { next(e); } +}); + +// Serve a file from inside an archive (extracted on first use, then cached). +app.get('/api/entries/:id/raw', async (req, res, next) => { + try { + const entry = await store.getArchiveEntry(req.params.id); + if (!entry) return res.status(404).json({ error: 'Entry not found' }); + const abs = await archives.extractEntry(entry.archive, entry); + sendStored(req, res, abs, path.posix.basename(entry.path)); } catch (e) { next(e); } }); // Serve a 3MF as GLB, converting on first request (cached thereafter). This lets // the browser render Bambu/MakerWorld 3MFs and backfills volume/bbox. -const glbConversions = new Map(); // fileId -> in-flight Promise +// ?plate=N converts just that plate of a slicer project. +const glbConversions = new Map(); // output path -> in-flight Promise + +// Converts into a temporary file and renames it into place, so a half-written +// GLB is never served. +function convertOnce(inputPath, outPath, plate = null) { + if (!glbConversions.has(outPath)) { + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + const tmp = `${outPath}.part`; + glbConversions.set(outPath, converter.convert3mfToGlb(inputPath, tmp, { plate }) + .then((result) => { fs.renameSync(tmp, outPath); return result; }) + .catch((err) => { + if (/no plate \d+/.test(err.message)) err.status = 404; + throw err; + }) + .finally(() => { fs.rmSync(tmp, { force: true }); glbConversions.delete(outPath); })); + } + return glbConversions.get(outPath); +} + +// Bump when the converter's output changes (e.g. filament colours), so previews +// cached by an older version get rebuilt instead of served. +const GLB_VERSION = 2; +const glbName = (base, plate) => `${base}${plate ? `-plate${plate}` : ''}-v${GLB_VERSION}.glb`; + +function plateParam(req) { + const n = Number(req.query.plate); + return Number.isInteger(n) && n > 0 ? n : null; +} app.get('/api/files/:id/glb', async (req, res, next) => { try { - const { rows } = await db.query('SELECT id, file_path, format, glb_path FROM files WHERE id = $1', [req.params.id]); - if (rows.length === 0) return res.status(404).json({ error: 'File not found' }); - const file = rows[0]; + const file = await store.getFile(req.params.id); + if (!file) return res.status(404).json({ error: 'File not found' }); if (file.format !== '3mf') return res.status(415).json({ error: 'GLB conversion is only for 3MF files' }); + const plate = plateParam(req); + if (plate) { + const platePath = path.resolve('./glb', glbName(file.id, plate)); + if (!fs.existsSync(platePath)) await convertOnce(path.resolve(file.file_path), platePath, plate); + return res.sendFile(platePath); + } let glbPath = file.glb_path; - if (!glbPath || !fs.existsSync(glbPath)) { - if (!glbConversions.has(file.id)) { - glbConversions.set(file.id, convertFileToGlb(file).finally(() => glbConversions.delete(file.id))); - } - glbPath = await glbConversions.get(file.id); + if (!glbPath || path.basename(glbPath) !== glbName(file.id) || !fs.existsSync(glbPath)) { + if (glbPath) fs.rmSync(glbPath, { force: true }); // made by an older converter + glbPath = path.join('./glb', glbName(file.id)); + const result = await convertOnce(path.resolve(file.file_path), path.resolve(glbPath)); + await store.updateFileConversion(file.id, { glb_path: glbPath, volume: result.volume_cm3, bbox: result.bbox }); } res.sendFile(path.resolve(glbPath)); } catch (e) { next(e); } }); -async function convertFileToGlb(file) { - if (!fs.existsSync('./glb')) fs.mkdirSync('./glb', { recursive: true }); - const out = path.join('./glb', `${file.id}.glb`); - const result = await converter.convert3mfToGlb(path.resolve(file.file_path), path.resolve(out)); - await store.updateFileConversion(file.id, { glb_path: out, volume: result.volume_cm3, bbox: result.bbox }); - return out; +app.get('/api/entries/:id/glb', async (req, res, next) => { + try { + const entry = await store.getArchiveEntry(req.params.id); + if (!entry) return res.status(404).json({ error: 'Entry not found' }); + if (entry.format !== '3mf') return res.status(415).json({ error: 'GLB conversion is only for 3MF files' }); + const plate = plateParam(req); + const glbPath = path.resolve('./glb', glbName(`entry-${entry.id}`, plate)); + if (!fs.existsSync(glbPath)) await convertOnce(await archives.extractEntry(entry.archive, entry), glbPath, plate); + res.sendFile(glbPath); + } catch (e) { next(e); } +}); + +// Plates and filaments of a 3MF slicer project (Bambu Studio / OrcaSlicer), +// read once per server run. Plate previews land in thumbnails/plates/. +const PLATE_IMAGE_DIR = './thumbnails/plates'; +const projectInfoCache = new Map(); // "-" -> Promise + +async function threeMfPath(kind, id) { + const unsupported = () => Object.assign(new Error('Only 3MF files have plates'), { status: 415 }); + if (kind === 'files') { + const file = await store.getFile(id); + if (!file) return null; + if (file.format !== '3mf') throw unsupported(); + return path.resolve(file.file_path); + } + const entry = await store.getArchiveEntry(id); + if (!entry) return null; + if (entry.format !== '3mf') throw unsupported(); + return archives.extractEntry(entry.archive, entry); +} + +for (const kind of ['files', 'entries']) { + app.get(`/api/${kind}/:id/project`, async (req, res, next) => { + try { + const key = `${kind}-${Number(req.params.id)}`; + if (!projectInfoCache.has(key)) { + const abs = await threeMfPath(kind, req.params.id); + if (!abs) return res.status(404).json({ error: 'Not found' }); + const imagesDir = path.resolve(PLATE_IMAGE_DIR, key); + const job = converter.readProjectInfo(abs, imagesDir).then((info) => ({ + ...info, + plates: info.plates.map((p) => { + const image = [`${p.index}-small.png`, `${p.index}.png`].find((n) => fs.existsSync(path.join(imagesDir, n))); + return { ...p, thumbnailUrl: image ? `/thumbnails/plates/${key}/${image}` : null }; + }), + })); + projectInfoCache.set(key, job); + job.catch(() => projectInfoCache.delete(key)); + } + res.json(await projectInfoCache.get(key)); + } catch (e) { next(e); } + }); +} + +// Removes cached previews (GLBs incl. per-plate ones) and plate images for +// deleted files and archive entries. +function removeDerivedFiles(fileIds, entryIds) { + const owned = new Set([...fileIds.map(String), ...entryIds.map((id) => `entry-${id}`)]); + let names = []; + try { names = fs.readdirSync('./glb'); } catch { /* no previews yet */ } + for (const name of names) { + const m = /^(.+?)(?:-plate\d+)?(?:-v\d+)?\.glb$/.exec(name); + if (m && owned.has(m[1])) fs.rmSync(path.join('./glb', name), { force: true }); + } + for (const id of fileIds) { + fs.rmSync(path.join(PLATE_IMAGE_DIR, `files-${id}`), { recursive: true, force: true }); + projectInfoCache.delete(`files-${id}`); + } + for (const id of entryIds) { + fs.rmSync(path.join(PLATE_IMAGE_DIR, `entries-${id}`), { recursive: true, force: true }); + projectInfoCache.delete(`entries-${id}`); + } } // ----------------------------------------------------------------- import @@ -865,16 +1009,17 @@ async function findDuplicateGroups() { app.get('/api/library/stats', async (req, res, next) => { try { - const [trash, groups, pending] = await Promise.all([ - store.countTrashed(), findDuplicateGroups(), fingerprint.pendingCount()]); - res.json({ trash, duplicateGroups: groups.length, fingerprintsPending: pending }); + const [trash, groups, pending, thumbnailsPending] = await Promise.all([ + store.countTrashed(), findDuplicateGroups(), fingerprint.pendingCount(), thumbnails.pendingCount()]); + res.json({ trash, duplicateGroups: groups.length, fingerprintsPending: pending, thumbnailsPending }); } catch (e) { next(e); } }); app.get('/api/duplicates', async (req, res, next) => { try { const [groups, pending] = await Promise.all([findDuplicateGroups(), fingerprint.pendingCount()]); - const found = await store.getModelsByIds(groups.flatMap((g) => g.modelIds)); + const ids = groups.flatMap((g) => g.modelIds); + const [found, counts] = await Promise.all([store.getModelsByIds(ids), store.matchableCounts(ids)]); const byId = new Map(found.map((m) => [m.id, m])); const result = groups .map((g) => ({ ...g, models: g.modelIds.map((id) => byId.get(id)).filter(Boolean) })) @@ -883,7 +1028,11 @@ app.get('/api/duplicates', async (req, res, next) => { signature: duplicates.signature(g.modelIds), exact: g.exact, recommendedId: duplicates.recommendKeep(g.models), - models: g.models.map((m) => ({ ...publicModel(m), matchedFiles: g.matchedFiles[m.id] })), + models: g.models.map((m) => ({ + ...publicModel(m), + matchedFiles: g.matchedFiles[m.id], + matchableFiles: counts[m.id] ?? m.files.length, + })), })); res.json({ pending, groups: result }); } catch (e) { next(e); } diff --git a/test/archives.test.js b/test/archives.test.js new file mode 100644 index 0000000..a9093df --- /dev/null +++ b/test/archives.test.js @@ -0,0 +1,47 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const crypto = require('crypto'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +process.env.EXTRACT_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'printhive-extract-')); +const archives = require('../lib/archives'); + +const fixture = (name) => path.join(__dirname, 'fixtures', name); +const sha = (p) => crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex'); +const clean = (p) => p.replace(/^\.\//, ''); + +// Each pack holds models/cube.stl, models/inner.3mf and readme.txt, plus macOS +// junk (._ files, __MACOSX, .DS_Store); the zip also has a ../ path. +const EXPECTED = ['models/cube.stl', 'models/inner.3mf', 'readme.txt']; + +test.after(() => fs.rmSync(process.env.EXTRACT_DIR, { recursive: true, force: true })); + +for (const [name, format] of [['pack.zip', 'zip'], ['pack.7z', '7z'], ['pack.tar.gz', 'tgz']]) { + const file = { id: `test-${format}`, file_path: fixture(name), format }; + + test(`${name}: lists real files, skipping folders, junk and unsafe paths`, async () => { + const entries = await archives.listEntries(file); + assert.deepEqual(entries.map((e) => clean(e.path)).sort(), EXPECTED); + const cube = entries.find((e) => clean(e.path) === 'models/cube.stl'); + assert.equal(cube.size, fs.statSync(fixture('cube.stl')).size); + }); + + test(`${name}: extracts one entry byte-for-byte, and caches it`, async () => { + const entries = await archives.listEntries(file); + const cube = { ...entries.find((e) => clean(e.path) === 'models/cube.stl'), id: 1 }; + const abs = await archives.extractEntry(file, cube); + assert.equal(sha(abs), sha(fixture('cube.stl'))); + assert.equal(await archives.extractEntry(file, cube), abs); + }); +} + +test('a missing entry fails clearly instead of returning a path', async () => { + const file = { id: 'test-missing', file_path: fixture('pack.zip'), format: 'zip' }; + await assert.rejects(archives.extractEntry(file, { id: 2, path: 'models/nope.stl' }), /Could not extract/); +}); + +test('an unreadable archive rejects', async () => { + await assert.rejects(archives.listEntries({ file_path: fixture('cube.stl'), format: 'zip' })); +}); diff --git a/test/fixtures/pack.7z b/test/fixtures/pack.7z new file mode 100644 index 0000000..9721826 Binary files /dev/null and b/test/fixtures/pack.7z differ diff --git a/test/fixtures/pack.tar.gz b/test/fixtures/pack.tar.gz new file mode 100644 index 0000000..59869e8 Binary files /dev/null and b/test/fixtures/pack.tar.gz differ diff --git a/test/fixtures/pack.zip b/test/fixtures/pack.zip new file mode 100644 index 0000000..f69c910 Binary files /dev/null and b/test/fixtures/pack.zip differ diff --git a/test/fixtures/plates.3mf b/test/fixtures/plates.3mf new file mode 100644 index 0000000..5bd71b6 Binary files /dev/null and b/test/fixtures/plates.3mf differ diff --git a/test/project.test.js b/test/project.test.js new file mode 100644 index 0000000..78f268b --- /dev/null +++ b/test/project.test.js @@ -0,0 +1,81 @@ +// Slicer-project handling of 3MF files: plates, filament colours and part types. +// plates.3mf is laid out like a Bambu Studio project: +// plate 1 "Left": object 10 = a 20 mm cube (part 1, filament slot 2) +// + a negative part (part 2) that must never be drawn +// plate 2 "Right": object 20 = a 10x10x40 tower (part 3, no slot of its own: +// uses its object's slot 3), placed at x = 300 +// Filaments: #FF0000 PLA, #00FF00 PETG, #0000FF TPU. Only plate 1 has a preview image. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { execFileSync, spawnSync } = require('child_process'); +const { PYTHON } = require('../lib/python'); + +const TOOLS = path.join(__dirname, '..', 'tools'); +const FIXTURE = path.join(__dirname, 'fixtures', 'plates.3mf'); +const skip = !fs.existsSync(PYTHON) && 'Python venv not installed'; + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'printhive-project-')); +test.after(() => fs.rmSync(tmp, { recursive: true, force: true })); + +const lastJson = (stdout) => JSON.parse(stdout.trim().split('\n').pop()); + +// Converts (optionally one plate) and returns { summary, colours } where colours +// are the base colours of the meshes drawn, one per placed node. +function convert(plate) { + const out = path.join(tmp, `plate-${plate || 'all'}.glb`); + const args = [path.join(TOOLS, 'convert_3mf.py'), FIXTURE, out, ...(plate ? ['--plate', String(plate)] : [])]; + const summary = lastJson(execFileSync(PYTHON, args, { encoding: 'utf8' })); + const glb = fs.readFileSync(out); + const gltf = JSON.parse(glb.subarray(20, 20 + glb.readUInt32LE(12)).toString('utf8')); + const colours = gltf.nodes.filter((n) => n.mesh !== undefined).map((n) => { + const material = gltf.materials[gltf.meshes[n.mesh].primitives[0].material]; + return material.pbrMetallicRoughness.baseColorFactor.slice(0, 3).map((c) => Math.round(c * 255)); + }); + return { summary, colours }; +} + +test('project info lists plates, their previews and the filaments', { skip }, () => { + const images = path.join(tmp, 'images'); + const info = lastJson(execFileSync(PYTHON, [path.join(TOOLS, 'threemf_info.py'), FIXTURE, '--images', images], { encoding: 'utf8' })); + assert.deepEqual(info.plates, [ + { index: 1, name: 'Left', objectCount: 1, thumbnail: true }, + { index: 2, name: 'Right', objectCount: 1, thumbnail: false }, + ]); + assert.deepEqual(info.filaments.map((f) => [f.colour, f.type]), [['#FF0000', 'PLA'], ['#00FF00', 'PETG'], ['#0000FF', 'TPU']]); + assert.ok(fs.existsSync(path.join(images, '1.png'))); + assert.ok(!fs.existsSync(path.join(images, '2.png'))); +}); + +test('a plate converts alone, coloured by its part’s filament, without negative parts', { skip }, () => { + const { summary, colours } = convert(1); + assert.equal(summary.faces, 12); + assert.deepEqual([summary.bbox.x, summary.bbox.y, summary.bbox.z].map(Math.round), [20, 20, 20]); + assert.deepEqual(colours, [[0, 255, 0]]); +}); + +test('a part without its own filament uses its object’s', { skip }, () => { + const { summary, colours } = convert(2); + assert.equal(summary.faces, 12); + assert.deepEqual(colours, [[0, 0, 255]]); +}); + +test('all plates convert together', { skip }, () => { + const { summary, colours } = convert(null); + assert.equal(summary.faces, 24); + assert.deepEqual([summary.bbox.x, summary.bbox.y, summary.bbox.z].map(Math.round), [315, 20, 40]); + assert.deepEqual(colours.sort(), [[0, 0, 255], [0, 255, 0]]); +}); + +test('asking for a plate that does not exist fails with a clear error', { skip }, () => { + const run = spawnSync(PYTHON, [path.join(TOOLS, 'convert_3mf.py'), FIXTURE, path.join(tmp, 'none.glb'), '--plate', '5'], { encoding: 'utf8' }); + assert.equal(run.status, 1); + assert.match(lastJson(run.stdout).error, /no plate 5/); +}); + +test('fingerprints leave out negative parts', { skip }, () => { + const out = execFileSync(PYTHON, [path.join(TOOLS, 'fingerprint.py'), FIXTURE], { encoding: 'utf8' }); + assert.equal(lastJson(out).fp.faces, 24); +}); diff --git a/test/thumbnail.test.js b/test/thumbnail.test.js new file mode 100644 index 0000000..f3d4e89 --- /dev/null +++ b/test/thumbnail.test.js @@ -0,0 +1,79 @@ +// Server-side thumbnail rendering (tools/render_thumbnail.py). +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { execFileSync, spawnSync } = require('child_process'); +const { PYTHON } = require('../lib/python'); + +const RENDER = path.join(__dirname, '..', 'tools', 'render_thumbnail.py'); +const fixture = (name) => path.join(__dirname, 'fixtures', name); +const skip = !fs.existsSync(PYTHON) && 'Python venv not installed'; + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'printhive-thumb-')); +test.after(() => fs.rmSync(tmp, { recursive: true, force: true })); + +const lastJson = (stdout) => JSON.parse(stdout.trim().split('\n').pop()); + +// Size, how much of the image is covered, where, and how green/blue the model is. +function pixelStats(png) { + const script = ` +import json, sys +from PIL import Image +im = Image.open(sys.argv[1]).convert("RGBA") +px = [p for p in im.getdata() if p[3] > 0] +opaque = Image.new("L", im.size); opaque.putdata([255 if p[3] > 0 else 0 for p in im.getdata()]) +share = lambda test: sum(1 for p in px if test(p)) / max(len(px), 1) +from collections import Counter +top3 = sum(n for _, n in Counter(p[:3] for p in px if p[3] == 255).most_common(3)) / max(len(px), 1) +print(json.dumps({"size": im.size, "coverage": len(px) / (im.size[0] * im.size[1]), "bbox": opaque.getbbox(), "top3": top3, + "green": share(lambda p: p[1] > p[0] + 40 and p[1] > p[2] + 40), + "blue": share(lambda p: p[2] > p[0] + 40 and p[2] > p[1] + 40)}))`; + return JSON.parse(execFileSync(PYTHON, ['-c', script, png], { encoding: 'utf8' })); +} + +test('an STL renders to a 512px PNG, fully in frame', { skip }, () => { + const out = path.join(tmp, 'cube.png'); + const result = lastJson(execFileSync(PYTHON, [RENDER, fixture('cube.stl'), out], { encoding: 'utf8' })); + assert.equal(result.faces, 12); + const stats = pixelStats(out); + assert.deepEqual(stats.size, [512, 512]); + assert.ok(stats.coverage > 0.1 && stats.coverage < 0.9, `coverage ${stats.coverage}`); + const [left, top, right, bottom] = stats.bbox; + assert.ok(left > 2 && top > 2 && right < 510 && bottom < 510, `bbox ${stats.bbox}`); +}); + +// A cube shows three faces, each one flat shade; hidden faces painted over +// visible ones (or inside-out winding) would add more shades. +for (const inverted of [false, true]) { + test(`a cube${inverted ? ' with inside-out faces' : ''} shows just its three visible faces`, { skip }, () => { + let input = fixture('cube.stl'); + if (inverted) { + input = path.join(tmp, 'inverted.stl'); + execFileSync(PYTHON, ['-c', ` +import sys, trimesh +m = trimesh.load(sys.argv[1], force="mesh"); m.invert(); m.export(sys.argv[2])`, fixture('cube.stl'), input]); + } + const out = path.join(tmp, `cube-${inverted}.png`); + execFileSync(PYTHON, [RENDER, input, out], { encoding: 'utf8' }); + const { top3 } = pixelStats(out); + assert.ok(top3 > 0.9, `three most common colours cover ${top3}`); + }); +} + +test('a multi-plate 3MF renders its first plate in filament colours', { skip }, () => { + const out = path.join(tmp, 'plates.png'); + lastJson(execFileSync(PYTHON, [RENDER, fixture('plates.3mf'), out], { encoding: 'utf8' })); + const stats = pixelStats(out); + assert.ok(stats.green > 0.5, `green share ${stats.green}`); // plate 1's cube prints in #00FF00 + assert.ok(stats.blue < 0.01, `blue share ${stats.blue}`); // plate 2's blue tower isn't drawn +}); + +test('a file with no geometry fails with a JSON error', { skip }, () => { + const broken = path.join(tmp, 'broken.stl'); + fs.writeFileSync(broken, 'this is not a mesh'); + const run = spawnSync(PYTHON, [RENDER, broken, path.join(tmp, 'x.png')], { encoding: 'utf8' }); + assert.equal(run.status, 1); + assert.match(lastJson(run.stdout).error, /could not render thumbnail/); +}); diff --git a/tools/convert_3mf.py b/tools/convert_3mf.py index fb0f90a..7a98bff 100644 --- a/tools/convert_3mf.py +++ b/tools/convert_3mf.py @@ -1,25 +1,43 @@ #!/usr/bin/env python3 """Convert a 3MF (incl. Bambu/MakerWorld production-extension files) to GLB. -Usage: convert_3mf.py -Prints a JSON summary {glb, volume_cm3, bbox, faces} to stdout. +Usage: convert_3mf.py [--plate N] +Prints a JSON summary {glb, plate, volume_cm3, bbox, faces} to stdout. +Negative/modifier/support parts are left out, parts are coloured by their +filament when the project says so, and --plate keeps just that plate. """ -import sys +import argparse import json import trimesh +from trimesh.visual import TextureVisuals +from trimesh.visual.material import PBRMaterial import threemf_stream -def main(): - if len(sys.argv) != 3: - print(json.dumps({"error": "usage: convert_3mf.py "})) - return 2 +def mesh_in(vertices, faces, colour): + mesh = trimesh.Trimesh(vertices=vertices, faces=faces, process=False) + if colour: + rgb = [int(colour[i:i + 2], 16) for i in (1, 3, 5)] + mesh.visual = TextureVisuals(material=PBRMaterial( + baseColorFactor=rgb + [255], metallicFactor=0.1, roughnessFactor=0.7)) + return mesh + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input") + parser.add_argument("output") + parser.add_argument("--plate", type=int) + args = parser.parse_args() - in_path, out_path = sys.argv[1], sys.argv[2] try: - meshes, placements = threemf_stream.load(in_path) + meshes, placements = threemf_stream.load(args.input) + project = threemf_stream.read_project(args.input) + placements = threemf_stream.printable(project, placements) + if args.plate is not None: + placements = threemf_stream.on_plate(project, placements, args.plate) except Exception as exc: print(json.dumps({"error": f"could not read 3MF: {str(exc)[:200]}"})) return 1 @@ -27,24 +45,26 @@ def main(): print(json.dumps({"error": "no geometry found in 3MF"})) return 1 - # Each object is stored once; repeated placements become instances of it. + # Each (object, colour) is stored once; repeated placements become instances of it. scene = trimesh.Scene() added = set() - for i, (key, transform) in enumerate(placements): + for i, p in enumerate(placements): + colour = threemf_stream.filament_colour(project, p) + geom = f"{p.key}|{colour or ''}" node = f"part{i}" - if key in added: - scene.graph.update(frame_from=scene.graph.base_frame, frame_to=node, matrix=transform, geometry=key) + if geom in added: + scene.graph.update(frame_from=scene.graph.base_frame, frame_to=node, matrix=p.transform, geometry=geom) else: - v, f = meshes[key] - scene.add_geometry(trimesh.Trimesh(vertices=v, faces=f, process=False), - node_name=node, geom_name=key, transform=transform) - added.add(key) - scene.export(out_path, file_type="glb") + v, f = meshes[p.key] + scene.add_geometry(mesh_in(v, f, colour), node_name=node, geom_name=geom, transform=p.transform) + added.add(geom) + scene.export(args.output, file_type="glb") stats = threemf_stream.measure(meshes, placements) ext = stats["extents"] print(json.dumps({ - "glb": out_path, + "glb": args.output, + "plate": args.plate, "bbox": {"x": round(ext[0], 3), "y": round(ext[1], 3), "z": round(ext[2], 3)}, "faces": stats["faces"], # mm^3 -> cm^3 to match the rest of the app @@ -54,4 +74,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main()) diff --git a/tools/fingerprint.py b/tools/fingerprint.py index edac29f..acfe8d8 100644 --- a/tools/fingerprint.py +++ b/tools/fingerprint.py @@ -15,7 +15,11 @@ import threemf_stream def measure(path): if path.lower().endswith(".3mf"): - stats = threemf_stream.measure(*threemf_stream.load(path)) + # Leave out negative/modifier/support parts, so a project matches the + # model it prints (e.g. its STL export). + meshes, placements = threemf_stream.load(path) + placements = threemf_stream.printable(threemf_stream.read_project(path), placements) + stats = threemf_stream.measure(meshes, placements) if not stats["faces"]: raise ValueError("no geometry found") return stats diff --git a/tools/render_thumbnail.py b/tools/render_thumbnail.py new file mode 100644 index 0000000..d2eb5fb --- /dev/null +++ b/tools/render_thumbnail.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Render a model's card thumbnail without a GPU. + +Usage: render_thumbnail.py [--size 512] +Prints {png, faces} (or {error}) as JSON. + +3MF projects show their first plate with parts in their filament colours. Big +meshes are simplified first, then triangles are painted back to front with +Pillow, which is plenty for a card image. +""" +import argparse +import json + +import fast_simplification +import numpy as np +import trimesh +from PIL import Image, ImageDraw + +import threemf_stream + +MAX_FACES = 150_000 +GREY = (196, 202, 214) +SUPERSAMPLE = 2 + + +def simplify(vertices, faces, ratio): + if ratio >= 1.0 or len(faces) <= 400: + return vertices, faces + v, f = fast_simplification.simplify( + np.asarray(vertices, dtype=np.float32), np.asarray(faces, dtype=np.int32), target_reduction=1 - ratio) + return np.asarray(v, dtype=np.float64), np.asarray(f) + + +def hex_rgb(colour): + return tuple(int(colour[i:i + 2], 16) for i in (1, 3, 5)) if colour else GREY + + +def load_3mf(path): + meshes, placements = threemf_stream.load(path) + project = threemf_stream.read_project(path) + placements = threemf_stream.printable(project, placements) + if len(project["plates"]) > 1: + placements = threemf_stream.on_plate(project, placements, project["plates"][0]["index"]) or placements + if not placements: + raise ValueError("no geometry found") + + # Simplify each distinct part once, by the same ratio, so instances stay cheap. + total = sum(len(meshes[p.key][1]) for p in placements) + ratio = min(1.0, MAX_FACES / total) + small = {} + vs, fs, cs, offset = [], [], [], 0 + for p in placements: + if p.key not in small: + small[p.key] = simplify(*meshes[p.key], ratio) + v, f = small[p.key] + t = p.transform + if np.linalg.det(t[:3, :3]) < 0: + f = f[:, ::-1] # a mirrored placement flips winding; flip it back so faces point outward + vs.append(v @ t[:3, :3].T + t[:3, 3]) + fs.append(f + offset) + cs.append(np.tile(hex_rgb(threemf_stream.filament_colour(project, p)), (len(f), 1))) + offset += len(v) + return np.vstack(vs), np.vstack(fs), np.vstack(cs) + + +def load_mesh(path): + mesh = trimesh.load(path, force="mesh") + if mesh is None or len(mesh.faces) == 0: + raise ValueError("no geometry found") + v, f = simplify(np.asarray(mesh.vertices), np.asarray(mesh.faces), min(1.0, MAX_FACES / len(mesh.faces))) + if path.lower().endswith(".obj"): + v = np.column_stack([v[:, 0], -v[:, 2], v[:, 1]]) # OBJ is Y-up; everything else here is Z-up + return v, f, np.tile(GREY, (len(f), 1)) + + +def render(vertices, faces, colours, size): + # Three-quarter view from the front right, a little above. + az, el = np.radians(-40), np.radians(30) + forward = -np.array([np.cos(el) * np.sin(az), -np.cos(el) * np.cos(az), np.sin(el)]) + right = np.cross(forward, [0, 0, 1]) + right /= np.linalg.norm(right) + up = np.cross(right, forward) + + a, b, c = vertices[faces[:, 0]], vertices[faces[:, 1]], vertices[faces[:, 2]] + normals = np.cross(b - a, c - a) + # A mesh exported inside-out has negative volume: turn its normals around. + if np.einsum("ij,ij->i", a, np.cross(b, c)).sum() < 0: + normals = -normals + normals /= np.linalg.norm(normals, axis=1, keepdims=True) + 1e-12 + + # Only faces pointing at the camera are drawn. Painting back faces too lets a + # large far triangle land on top of a near one when sorted by centre depth. + facing = normals @ forward < 0 + if facing.any(): + a, b, c, normals, colours = a[facing], b[facing], c[facing], normals[facing], colours[facing] + + light = -forward + 0.6 * up - 0.4 * right + light /= np.linalg.norm(light) + shade = 0.32 + 0.68 * np.clip(normals @ light, 0, 1) + fills = np.clip(colours * shade[:, None], 0, 255).astype(np.uint8) + + xs = np.stack([a @ right, b @ right, c @ right], axis=1) + ys = np.stack([a @ up, b @ up, c @ up], axis=1) + lo = np.array([xs.min(), ys.min()]) + span = np.array([xs.max(), ys.max()]) - lo + width = size * SUPERSAMPLE + scale = width * 0.86 / max(span.max(), 1e-9) + offset = (width - span * scale) / 2 + px = (xs - lo[0]) * scale + offset[0] + py = width - ((ys - lo[1]) * scale + offset[1]) + + image = Image.new("RGBA", (width, width), (0, 0, 0, 0)) + draw = ImageDraw.Draw(image) + depth = ((a + b + c) / 3) @ forward + for i in np.argsort(-depth): # far to near + fill = tuple(int(x) for x in fills[i]) + (255,) + draw.polygon([(px[i, 0], py[i, 0]), (px[i, 1], py[i, 1]), (px[i, 2], py[i, 2])], fill=fill, outline=fill) + return image.resize((size, size), Image.LANCZOS) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input") + parser.add_argument("output") + parser.add_argument("--size", type=int, default=512) + args = parser.parse_args() + try: + load = load_3mf if args.input.lower().endswith(".3mf") else load_mesh + vertices, faces, colours = load(args.input) + render(vertices, faces, colours, args.size).save(args.output, "PNG") + except Exception as exc: + print(json.dumps({"error": f"could not render thumbnail: {str(exc)[:200]}"})) + return 1 + print(json.dumps({"png": args.output, "faces": int(len(faces))})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/requirements.txt b/tools/requirements.txt index 5af82ec..2369c07 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -2,3 +2,5 @@ trimesh numpy networkx lxml +pillow +fast-simplification diff --git a/tools/threemf_info.py b/tools/threemf_info.py new file mode 100644 index 0000000..583559d --- /dev/null +++ b/tools/threemf_info.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Print a 3MF project's plates and filaments as JSON, without loading geometry. + +Usage: threemf_info.py [--images DIR] +With --images, each plate's preview is written to DIR/.png and DIR/-small.png. +""" +import argparse +import json +import os +import zipfile + +import threemf_stream + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input") + parser.add_argument("--images") + args = parser.parse_args() + + try: + project = threemf_stream.read_project(args.input) + if args.images and project["plates"]: + os.makedirs(args.images, exist_ok=True) + with zipfile.ZipFile(args.input) as z: + for plate in project["plates"]: + for name, suffix in ((plate["thumbnail"], ""), (plate["thumbnail_small"], "-small")): + if name: + with open(os.path.join(args.images, f"{plate['index']}{suffix}.png"), "wb") as out: + out.write(z.read(name)) + except Exception as exc: + print(json.dumps({"error": f"could not read 3MF: {str(exc)[:200]}"})) + return 1 + + print(json.dumps({ + "plates": [{"index": p["index"], "name": p["name"], "objectCount": len(p["objects"]), + "thumbnail": bool(p["thumbnail"] or p["thumbnail_small"])} for p in project["plates"]], + "filaments": project["filaments"], + })) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/threemf_stream.py b/tools/threemf_stream.py index 4a2d11a..a7e3808 100644 --- a/tools/threemf_stream.py +++ b/tools/threemf_stream.py @@ -1,4 +1,4 @@ -"""Streaming 3MF reader shared by convert_3mf.py and fingerprint.py. +"""Streaming 3MF reader shared by convert_3mf.py, fingerprint.py and threemf_info.py. trimesh's 3MF loader re-reads a whole production-extension part file (every mesh in it) for each component that points into that file. Bambu Studio @@ -8,10 +8,14 @@ the geometry multiplies until the process runs out of memory. This reader parses each part file once, element by element, resolves each component to the one object it names, and returns every object's mesh once plus the transforms where it is placed — so parts reused across plates share -their geometry. +their geometry. It also reads Bambu Studio / OrcaSlicer project metadata +(plates, filaments, part types) when the file has it. """ +import json +import re import zipfile from array import array +from collections import namedtuple import numpy as np from lxml import etree @@ -19,6 +23,13 @@ from lxml import etree UNIT_MM = {"micron": 0.001, "millimeter": 1.0, "centimeter": 10.0, "inch": 25.4, "foot": 304.8, "meter": 1000.0} PARSER = dict(huge_tree=True, resolve_entities=False, no_network=True) MAX_DEPTH = 32 +# Slicer part types that shape or steer printing but aren't printed themselves. +HIDDEN_SUBTYPES = {"negative_part", "modifier_part", "support_blocker", "support_enforcer"} + +# One placed copy of an object's mesh. `item` is the object id of the build item +# it belongs to and `instance` which occurrence of that id it is; `part` is the +# component directly under that object (how slicer metadata names parts). +Placement = namedtuple("Placement", "key transform item instance part") def _local(name): @@ -33,6 +44,13 @@ def _path_attr(el): return None +def _int(value): + try: + return int(value) + except (TypeError, ValueError): + return None + + def _matrix(text): m = np.eye(4) if text: @@ -98,7 +116,7 @@ def load(path): """Returns (meshes, placements). meshes: {key: (vertices, faces)} — each object once, in file units. - placements: [(key, 4x4 transform)] — one per placed instance, mapping to millimetres. + placements: [Placement] — one per placed instance; transforms map to millimetres. """ with zipfile.ZipFile(path) as z: names = set(z.namelist()) @@ -118,7 +136,7 @@ def load(path): scale = np.diag([UNIT_MM.get(unit or "millimeter", 1.0)] * 3 + [1.0]) meshes, placements = {}, [] - def walk(name, oid, transform, depth): + def walk(name, oid, transform, depth, item, instance, part_id): if depth > MAX_DEPTH: raise ValueError("3MF components nest too deeply") obj = part(name)[0].get(oid) @@ -128,23 +146,117 @@ def load(path): if len(f): key = f"{name}#{oid}" meshes.setdefault(key, (v, f)) - placements.append((key, transform)) + placements.append(Placement(key, transform, item, instance, part_id)) for cpath, coid, m in comps: - walk(cpath.lstrip("/") if cpath else name, coid, transform @ m, depth + 1) + walk(cpath.lstrip("/") if cpath else name, coid, transform @ m, depth + 1, + item, instance, part_id if depth else coid) + seen = {} for oid, m in items: - walk(root, oid, scale @ m, 0) + instance = seen.get(oid, 0) + seen[oid] = instance + 1 + walk(root, oid, scale @ m, 0, oid, instance, None) return meshes, placements +def read_project(path): + """Slicer project metadata (Bambu Studio / OrcaSlicer), empty when absent: + + {plates: [{index, name, objects: [(object_id, instance)], thumbnail, thumbnail_small}], + filaments: [{colour, type}], + objects: {object_id: {name, extruder, parts: {part_id: {extruder, subtype}}}}} + """ + project = {"plates": [], "filaments": [], "objects": {}} + with zipfile.ZipFile(path) as z: + names = set(z.namelist()) + + if "Metadata/project_settings.config" in names: + try: + settings = json.loads(z.read("Metadata/project_settings.config")) + colours = settings.get("filament_colour") or [] + types = settings.get("filament_type") or [] + project["filaments"] = [ + {"colour": c, "type": types[i] if i < len(types) else None} for i, c in enumerate(colours)] + except (ValueError, UnicodeDecodeError, AttributeError): + pass + + if "Metadata/model_settings.config" in names: + try: + root = etree.fromstring(z.read("Metadata/model_settings.config"), etree.XMLParser(**PARSER)) + except etree.XMLSyntaxError: + return project + + def meta(el): + return {m.get("key"): m.get("value") for m in el.findall("metadata") if m.get("key")} + + for obj in root.findall("object"): + info = meta(obj) + project["objects"][obj.get("id")] = { + "name": info.get("name"), + "extruder": _int(info.get("extruder")), + "parts": { + p.get("id"): {"extruder": _int(meta(p).get("extruder")), "subtype": p.get("subtype") or "normal_part"} + for p in obj.findall("part") + }, + } + for plate in root.findall("plate"): + info = meta(plate) + index = _int(info.get("plater_id")) or len(project["plates"]) + 1 + large = info.get("thumbnail_file") or f"Metadata/plate_{index}.png" + small = f"Metadata/plate_{index}_small.png" + project["plates"].append({ + "index": index, + "name": info.get("plater_name") or "", + "objects": [(meta(i).get("object_id"), _int(meta(i).get("instance_id")) or 0) + for i in plate.findall("model_instance")], + "thumbnail": large if large in names else None, + "thumbnail_small": small if small in names else None, + }) + return project + + +def printable(project, placements): + """Drops negative, modifier and support parts (known from slicer metadata).""" + def visible(p): + obj = project["objects"].get(p.item) + part = obj and obj["parts"].get(p.part) + return not part or part["subtype"] not in HIDDEN_SUBTYPES + return [p for p in placements if visible(p)] + + +def on_plate(project, placements, plate_index): + plate = next((pl for pl in project["plates"] if pl["index"] == plate_index), None) + if plate is None: + raise ValueError(f"no plate {plate_index} in this project") + wanted = set(plate["objects"]) + return [p for p in placements if (p.item, p.instance) in wanted] + + +def filament_colour(project, placement): + """The '#RRGGBB' colour a placed part prints in, or None if unknown. + A part's extruder of 0 (or none) means it uses its object's extruder.""" + obj = project["objects"].get(placement.item) + if not obj: + return None + part = obj["parts"].get(placement.part) or {} + extruder = part.get("extruder") or obj["extruder"] + filaments = project["filaments"] + if extruder and 1 <= extruder <= len(filaments): + colour = filaments[extruder - 1]["colour"] + if isinstance(colour, str) and re.fullmatch(r"#[0-9A-Fa-f]{6}([0-9A-Fa-f]{2})?", colour): + return colour[:7] + return None + + def measure(meshes, placements): """Faces, surface area (mm^2), volume (mm^3) and extents (mm) of everything placed, as if concatenated into one mesh — computed per instance to keep memory flat.""" faces, area, volume = 0, 0.0, 0.0 lo, hi = np.full(3, np.inf), np.full(3, -np.inf) - for key, t in placements: - v, f = meshes[key] + for p in placements: + v, f = meshes[p.key] + t = p.transform w = v @ t[:3, :3].T + t[:3, 3] a, b, c = w[f[:, 0]], w[f[:, 1]], w[f[:, 2]] cross = np.cross(b - a, c - a)