Browse archives, preview companion files, and view FBX/STEP/3DM/OBJ+MTL
Archives - ZIP, 7Z, RAR and TAR/.tar.gz uploads are indexed in the background into archive_entries (7-Zip compiled to WASM in a worker thread; node-tar for tar). macOS/Windows clutter and paths that could escape the extraction folder are skipped. - Models inside archives are extracted once per archive to hash and geometry-fingerprint them, so duplicates are found inside packs too. Pictures, PDFs and readmes are listed but never matched. - New routes list an archive's entries and serve/convert a single entry, extracted on first use and cached under extracted/. Viewer - Model detail lists what's inside each archive with view/download links; opening an entry adds a viewer tab. - New 3D formats: FBX (with sibling textures), OBJ with its MTL and textures, STEP/STP via occt-import-js, and Rhino 3DM via rhino3dm — the WASM readers are served from node_modules, loaded on first use. - Pictures, PDFs and text files (jpg/png/gif/webp/bmp/svg, pdf, txt/md) can be uploaded and preview in the viewer; a model without an embedded 3MF preview uses its first picture as the card thumbnail. - Inline file responses send nosniff, and SVGs are sandboxed. - GLB conversions write to a temp file and rename, so a half-written preview is never served; a later tab can't be overdrawn by a slow load. - Better 3MF card image: the project thumbnail or plate 1 render, not a slicer pick map. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dlawler489
parent
502eefdec2
commit
d4b7e29348
19 changed files with 1045 additions and 157 deletions
|
|
@ -14,3 +14,4 @@ npm-debug.log*
|
|||
Dockerfile
|
||||
.dockerignore
|
||||
docker-compose.yml
|
||||
extracted
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -9,3 +9,4 @@ thumbnails/
|
|||
glb/
|
||||
tools/venv/
|
||||
__pycache__/
|
||||
extracted/
|
||||
|
|
|
|||
|
|
@ -124,6 +124,27 @@ ALTER TABLE models ADD COLUMN IF NOT EXISTS notes TEXT NOT NULL DEFAULT '';
|
|||
ALTER TABLE models ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ;
|
||||
CREATE INDEX IF NOT EXISTS idx_models_deleted ON models(deleted_at);
|
||||
|
||||
-- Archive files (zip/7z/rar/tar) are indexed in the background:
|
||||
-- null = not yet, ok | error (with archive_error).
|
||||
ALTER TABLE files ADD COLUMN IF NOT EXISTS archive_status TEXT;
|
||||
ALTER TABLE files ADD COLUMN IF NOT EXISTS archive_error TEXT;
|
||||
|
||||
-- Files inside archives, so they can be browsed, previewed and matched as duplicates.
|
||||
CREATE TABLE IF NOT EXISTS archive_entries (
|
||||
id SERIAL PRIMARY KEY,
|
||||
file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
|
||||
path TEXT NOT NULL, -- path inside the archive, as the archive stores it
|
||||
format TEXT NOT NULL,
|
||||
size BIGINT NOT NULL DEFAULT 0,
|
||||
sha256 TEXT,
|
||||
geometry_hash TEXT,
|
||||
geometry_status TEXT, -- same states as files.geometry_status
|
||||
fingerprinted_at TIMESTAMPTZ,
|
||||
UNIQUE (file_id, path)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_entries_sha256 ON archive_entries(sha256);
|
||||
CREATE INDEX IF NOT EXISTS idx_entries_geometry ON archive_entries(geometry_hash);
|
||||
|
||||
-- Duplicate groups marked "not duplicates" (signature = sorted model ids).
|
||||
CREATE TABLE IF NOT EXISTS duplicate_dismissals (
|
||||
signature TEXT PRIMARY KEY,
|
||||
|
|
|
|||
82
lib/archiveWorker.js
Normal file
82
lib/archiveWorker.js
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
// Worker thread running 7-Zip (compiled to WASM) for ZIP/7Z/RAR archives.
|
||||
// 7-Zip calls are synchronous, so they run here to keep the server responsive.
|
||||
|
||||
const { parentPort } = require('worker_threads');
|
||||
const SevenZip = require('7z-wasm');
|
||||
|
||||
let sevenZip = null;
|
||||
const output = [];
|
||||
|
||||
// The host filesystem is mounted at /host inside the WASM filesystem.
|
||||
const host = (p) => `/host${p}`;
|
||||
|
||||
async function instance() {
|
||||
if (!sevenZip) {
|
||||
sevenZip = await SevenZip({ print: (l) => output.push(l), printErr: (l) => output.push(l) });
|
||||
sevenZip.FS.mkdir('/host');
|
||||
sevenZip.FS.mount(sevenZip.NODEFS, { root: '/' }, '/host');
|
||||
}
|
||||
return sevenZip;
|
||||
}
|
||||
|
||||
async function run(args) {
|
||||
const sz = await instance();
|
||||
output.length = 0;
|
||||
let code;
|
||||
try {
|
||||
code = sz.callMain(args);
|
||||
} catch (err) {
|
||||
code = err && typeof err.status === 'number' ? err.status : 2;
|
||||
}
|
||||
return { code, lines: output.slice() };
|
||||
}
|
||||
|
||||
function errorText(lines) {
|
||||
const line = lines.find((l) => /error|wrong password|can ?not open|unsupported|headers/i.test(l));
|
||||
return line ? line.trim() : null;
|
||||
}
|
||||
|
||||
// Parses `7zz l -slt` into [{ path, size }] for files (folders are dropped).
|
||||
function parseListing(lines) {
|
||||
const start = lines.findIndex((l) => l.startsWith('----------'));
|
||||
if (start === -1) return null;
|
||||
const entries = [];
|
||||
let cur = {};
|
||||
const flush = () => {
|
||||
const isFolder = cur.Folder === '+' || (cur.Attributes || '').startsWith('D');
|
||||
if (cur.Path !== undefined && !isFolder) entries.push({ path: cur.Path, size: Number(cur.Size) || 0 });
|
||||
cur = {};
|
||||
};
|
||||
for (const line of lines.slice(start + 1)) {
|
||||
if (!line.trim()) { flush(); continue; }
|
||||
const i = line.indexOf(' = ');
|
||||
if (i > 0) cur[line.slice(0, i)] = line.slice(i + 3);
|
||||
}
|
||||
flush();
|
||||
return entries;
|
||||
}
|
||||
|
||||
async function list(archive) {
|
||||
// A dummy password makes encrypted archives fail fast instead of prompting.
|
||||
const { code, lines } = await run(['l', '-slt', '-pprinthive', host(archive)]);
|
||||
const entries = parseListing(lines);
|
||||
if (code !== 0 || !entries) throw new Error(errorText(lines) || `7-Zip could not read the archive (exit ${code})`);
|
||||
return entries;
|
||||
}
|
||||
|
||||
// Extracts the named entries (keeping their folders) under destDir. Callers
|
||||
// check which files actually arrived.
|
||||
async function extract(archive, entries, destDir) {
|
||||
const { code, lines } = await run(
|
||||
['x', host(archive), `-o${host(destDir)}`, '-y', '-spd', '-bsp0', '-pprinthive', '--', ...entries]);
|
||||
return { code, error: code === 0 ? null : errorText(lines) };
|
||||
}
|
||||
|
||||
parentPort.on('message', async ({ id, op, archive, entries, destDir }) => {
|
||||
try {
|
||||
const result = op === 'list' ? await list(archive) : await extract(archive, entries, destDir);
|
||||
parentPort.postMessage({ id, result });
|
||||
} catch (err) {
|
||||
parentPort.postMessage({ id, error: err.message });
|
||||
}
|
||||
});
|
||||
133
lib/archives.js
Normal file
133
lib/archives.js
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
// Lists and extracts files inside archives. ZIP/7Z/RAR go through 7-Zip (WASM,
|
||||
// in a worker thread); TAR and .tar.gz go through node-tar. Extracted entries
|
||||
// are cached under EXTRACT_DIR/<file id>/ and can always be re-extracted.
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { Worker } = require('worker_threads');
|
||||
const tar = require('tar');
|
||||
|
||||
const EXTRACT_DIR = path.resolve(process.env.EXTRACT_DIR || './extracted');
|
||||
const TAR_FILE_TYPES = ['File', 'OldFile', 'ContiguousFile'];
|
||||
|
||||
let worker = null;
|
||||
let nextId = 1;
|
||||
const pending = new Map();
|
||||
|
||||
function failPending(err) {
|
||||
for (const p of pending.values()) p.reject(err);
|
||||
pending.clear();
|
||||
worker = null;
|
||||
}
|
||||
|
||||
// The worker keeps the process alive only while a request is outstanding, so
|
||||
// an idle worker never stops Node from exiting.
|
||||
function sevenZip(message) {
|
||||
if (!worker) {
|
||||
worker = new Worker(path.join(__dirname, 'archiveWorker.js'));
|
||||
worker.on('message', ({ id, result, error }) => {
|
||||
const p = pending.get(id);
|
||||
pending.delete(id);
|
||||
if (pending.size === 0 && worker) worker.unref();
|
||||
if (p) error ? p.reject(new Error(error)) : p.resolve(result);
|
||||
});
|
||||
worker.on('error', failPending);
|
||||
worker.on('exit', () => failPending(new Error('Archive worker stopped')));
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = nextId++;
|
||||
pending.set(id, { resolve, reject });
|
||||
worker.ref();
|
||||
worker.postMessage({ id, ...message });
|
||||
});
|
||||
}
|
||||
|
||||
const isTar = (format) => format === 'tar' || format === 'tgz';
|
||||
const clean = (p) => p.replace(/^\.\//, '');
|
||||
|
||||
// Clutter archivers add (macOS resource forks, Finder/Explorer files).
|
||||
function isJunk(entryPath) {
|
||||
const parts = clean(entryPath).split('/');
|
||||
const name = parts[parts.length - 1];
|
||||
return parts.includes('__MACOSX') || name.startsWith('._') || name === '.DS_Store' || name === 'Thumbs.db';
|
||||
}
|
||||
|
||||
// Paths that could land outside the extraction folder are never used.
|
||||
function isSafe(entryPath) {
|
||||
return Boolean(entryPath) && !path.isAbsolute(entryPath) && !entryPath.split(/[\\/]/).includes('..');
|
||||
}
|
||||
|
||||
// file: { file_path, format }. Returns [{ path, size }] of real files.
|
||||
async function listEntries(file) {
|
||||
const abs = path.resolve(file.file_path);
|
||||
let raw;
|
||||
if (isTar(file.format)) {
|
||||
raw = [];
|
||||
await tar.t({
|
||||
file: abs,
|
||||
onReadEntry: (e) => { if (TAR_FILE_TYPES.includes(e.type)) raw.push({ path: e.path, size: e.size }); },
|
||||
});
|
||||
} else {
|
||||
raw = await sevenZip({ op: 'list', archive: abs });
|
||||
}
|
||||
return raw.filter((e) => isSafe(e.path) && !isJunk(e.path));
|
||||
}
|
||||
|
||||
function tempDir() {
|
||||
fs.mkdirSync(EXTRACT_DIR, { recursive: true });
|
||||
return fs.mkdtempSync(path.join(EXTRACT_DIR, '.tmp-'));
|
||||
}
|
||||
|
||||
// Extracts entries into destDir. Returns Map(entry path -> absolute path) for
|
||||
// the ones that came out; archive-level failures throw.
|
||||
async function extractMany(file, entryPaths, destDir) {
|
||||
const abs = path.resolve(file.file_path);
|
||||
const wanted = new Set(entryPaths.filter(isSafe));
|
||||
if (isTar(file.format)) {
|
||||
await tar.x({ file: abs, cwd: destDir, filter: (p) => wanted.has(p) });
|
||||
} else {
|
||||
const { code, error } = await sevenZip({ op: 'extract', archive: abs, entries: [...wanted], destDir });
|
||||
if (code !== 0 && error) console.warn(`7-Zip reported a problem extracting from ${abs}: ${error}`);
|
||||
}
|
||||
const out = new Map();
|
||||
for (const p of wanted) {
|
||||
const target = path.resolve(destDir, p);
|
||||
if (target.startsWith(destDir + path.sep) && fs.existsSync(target)) out.set(p, target);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function entryCachePath(fileId, entry) {
|
||||
return path.join(EXTRACT_DIR, String(fileId), `${entry.id}${path.extname(entry.path).toLowerCase()}`);
|
||||
}
|
||||
|
||||
const inflight = new Map();
|
||||
|
||||
// file: { id, file_path, format }; entry: { id, path }. Returns the absolute
|
||||
// path of a cached copy, extracting it on first use.
|
||||
async function extractEntry(file, entry) {
|
||||
const dest = entryCachePath(file.id, entry);
|
||||
if (fs.existsSync(dest)) return dest;
|
||||
if (!inflight.has(dest)) {
|
||||
const job = (async () => {
|
||||
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
||||
const tmp = tempDir();
|
||||
try {
|
||||
const got = (await extractMany(file, [entry.path], tmp)).get(entry.path);
|
||||
if (!got) throw new Error(`Could not extract ${clean(entry.path)} from the archive`);
|
||||
fs.renameSync(got, dest);
|
||||
} finally {
|
||||
fs.rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
})().finally(() => inflight.delete(dest));
|
||||
inflight.set(dest, job);
|
||||
}
|
||||
await inflight.get(dest);
|
||||
return dest;
|
||||
}
|
||||
|
||||
function removeExtracted(fileId) {
|
||||
fs.rmSync(path.join(EXTRACT_DIR, String(fileId)), { recursive: true, force: true });
|
||||
}
|
||||
|
||||
module.exports = { listEntries, extractMany, extractEntry, removeExtracted, tempDir, isJunk };
|
||||
|
|
@ -5,18 +5,25 @@ const AdmZip = require('adm-zip');
|
|||
|
||||
const THUMBNAIL_DIR = './thumbnails';
|
||||
|
||||
const MODEL_FORMATS = ['stl', '3mf', 'obj', 'step', 'stp', 'gcode', 'gco', 'zip'];
|
||||
const MODEL_FORMATS = ['stl', '3mf', 'obj', 'fbx', 'step', 'stp', '3dm', 'gcode', 'gco'];
|
||||
const ARCHIVE_FORMATS = ['zip', '7z', 'rar', 'tar', 'tgz'];
|
||||
const IMAGE_FORMATS = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'svg'];
|
||||
// Kept with a model and viewable, but never matched as duplicates.
|
||||
const COMPANION_FORMATS = [...IMAGE_FORMATS, 'pdf', 'txt', 'md', 'mtl'];
|
||||
const ALLOWED_FORMATS = [...MODEL_FORMATS, ...ARCHIVE_FORMATS, ...COMPANION_FORMATS];
|
||||
|
||||
function ensureThumbnailDir() {
|
||||
if (!fs.existsSync(THUMBNAIL_DIR)) fs.mkdirSync(THUMBNAIL_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
function formatOf(filename) {
|
||||
return path.extname(filename).slice(1).toLowerCase();
|
||||
const lower = filename.toLowerCase();
|
||||
if (lower.endsWith('.tar.gz')) return 'tgz';
|
||||
return path.extname(lower).slice(1);
|
||||
}
|
||||
|
||||
function isAllowed(filename) {
|
||||
return MODEL_FORMATS.includes(formatOf(filename));
|
||||
return ALLOWED_FORMATS.includes(formatOf(filename));
|
||||
}
|
||||
|
||||
// Geometry metadata. Only STL is computed natively today; others return nulls
|
||||
|
|
@ -37,16 +44,28 @@ function extractMetadata(filePath, format) {
|
|||
}
|
||||
}
|
||||
|
||||
// Which embedded PNG makes the best card image: the project thumbnail, then the
|
||||
// first plate render. Slicer pick maps, top views and small/unlit variants last.
|
||||
function thumbnailRank(name) {
|
||||
if (name === 'metadata/thumbnail.png') return 0;
|
||||
if (/metadata\/plate_1\.png$/.test(name)) return 1;
|
||||
if (/metadata\/plate_\d+\.png$/.test(name)) return 2;
|
||||
if (/(pick_|top_|no_light|_small)/.test(name)) return 9;
|
||||
return 5;
|
||||
}
|
||||
|
||||
// A .3mf is a zip; slicers embed a PNG preview under /Metadata/. If present,
|
||||
// pull it out as the model thumbnail. Returns a path under thumbnails/ or null.
|
||||
function extract3mfThumbnail(filePath, baseName) {
|
||||
try {
|
||||
ensureThumbnailDir();
|
||||
const zip = new AdmZip(filePath);
|
||||
const entry = zip.getEntries().find((e) => {
|
||||
const entry = zip.getEntries()
|
||||
.filter((e) => {
|
||||
const n = e.entryName.toLowerCase();
|
||||
return n.endsWith('.png') && (n.includes('thumbnail') || n.includes('/metadata/'));
|
||||
});
|
||||
return n.endsWith('.png') && (n.includes('thumbnail') || n.includes('metadata/'));
|
||||
})
|
||||
.sort((a, b) => thumbnailRank(a.entryName.toLowerCase()) - thumbnailRank(b.entryName.toLowerCase()))[0];
|
||||
if (!entry) return null;
|
||||
const out = path.join(THUMBNAIL_DIR, `${baseName}.png`);
|
||||
fs.writeFileSync(out, entry.getData());
|
||||
|
|
@ -57,4 +76,21 @@ function extract3mfThumbnail(filePath, baseName) {
|
|||
}
|
||||
}
|
||||
|
||||
module.exports = { MODEL_FORMATS, formatOf, isAllowed, extractMetadata, extract3mfThumbnail };
|
||||
// Copy a picture that came with a model into thumbnails/ (a copy, so replacing
|
||||
// the thumbnail later never deletes the model's own file).
|
||||
function copyAsThumbnail(filePath, baseName) {
|
||||
try {
|
||||
ensureThumbnailDir();
|
||||
const out = path.join(THUMBNAIL_DIR, `${baseName}-image${path.extname(filePath).toLowerCase()}`);
|
||||
fs.copyFileSync(filePath, out);
|
||||
return out;
|
||||
} catch (err) {
|
||||
console.error('Image thumbnail copy failed:', err.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
MODEL_FORMATS, ARCHIVE_FORMATS, IMAGE_FORMATS, COMPANION_FORMATS, ALLOWED_FORMATS,
|
||||
formatOf, isAllowed, extractMetadata, extract3mfThumbnail, copyAsThumbnail,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,16 +1,21 @@
|
|||
// Background worker that fingerprints stored files for the duplicate finder: a
|
||||
// SHA-256 of the bytes for every file, plus a geometry fingerprint for mesh
|
||||
// Background worker that indexes and fingerprints stored files for the
|
||||
// duplicate finder: a SHA-256 of the bytes, plus a geometry fingerprint for mesh
|
||||
// formats so the same model still matches after being renamed, re-exported
|
||||
// (ASCII/binary STL) or converted (STL -> 3MF).
|
||||
// (ASCII/binary STL) or converted (STL -> 3MF). Archives are listed, and the
|
||||
// models inside them are fingerprinted too.
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const db = require('./db');
|
||||
const store = require('./store');
|
||||
const archives = require('./archives');
|
||||
const { MODEL_FORMATS, ARCHIVE_FORMATS, formatOf } = require('./fileProcessor');
|
||||
const { PYTHON, runScript } = require('./python');
|
||||
|
||||
const GEOMETRY_FORMATS = ['stl', 'obj', '3mf'];
|
||||
const BATCH = 10;
|
||||
const TABLES = ['files', 'archive_entries'];
|
||||
|
||||
let pythonAvailable = true;
|
||||
let running = false;
|
||||
|
|
@ -58,31 +63,43 @@ async function measure(paths) {
|
|||
return results;
|
||||
}
|
||||
|
||||
async function fingerprintGeometry(rows) {
|
||||
const abs = (row) => path.resolve(row.file_path);
|
||||
// rows: [{ id, format, abs }] from `table` ('files' or 'archive_entries').
|
||||
async function fingerprintGeometry(rows, table) {
|
||||
if (!TABLES.includes(table)) throw new Error(`Unknown table ${table}`);
|
||||
// 3MF projects can be big, so each gets its own run; that keeps a queued
|
||||
// preview from waiting behind a whole batch of them.
|
||||
const chunks = [rows.filter((r) => r.format !== '3mf'), ...rows.filter((r) => r.format === '3mf').map((r) => [r])];
|
||||
for (const chunk of chunks.filter((c) => c.length)) {
|
||||
const results = await measure(chunk.map(abs));
|
||||
const results = await measure(chunk.map((r) => r.abs));
|
||||
for (const row of chunk) {
|
||||
let r = results.get(abs(row));
|
||||
let r = results.get(row.abs);
|
||||
// A batch that died part-way: retry the missing file on its own.
|
||||
if (!r && chunk.length > 1) r = (await measure([abs(row)])).get(abs(row));
|
||||
if (!r && chunk.length > 1) r = (await measure([row.abs])).get(row.abs);
|
||||
const key = r && r.fp ? geometryKey(r.fp) : null;
|
||||
await db.query('UPDATE files SET geometry_hash = $2, geometry_status = $3 WHERE id = $1',
|
||||
await db.query(`UPDATE ${table} SET geometry_hash = $2, geometry_status = $3 WHERE id = $1`,
|
||||
[row.id, key, key ? 'ok' : 'error']);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function run() {
|
||||
for (;;) {
|
||||
// Runs fn; if Python turns out to be missing, geometry fingerprints are switched
|
||||
// off for this process (content hashes still run).
|
||||
async function withPython(fn) {
|
||||
try {
|
||||
await fn();
|
||||
} catch (err) {
|
||||
if (err.code !== 'ENOENT') throw err;
|
||||
pythonAvailable = false;
|
||||
console.warn(`Geometry fingerprints disabled: no Python at ${PYTHON} (content hashes still run).`);
|
||||
}
|
||||
}
|
||||
|
||||
async function fingerprintFiles() {
|
||||
const { rows } = await db.query(
|
||||
`SELECT id, file_path, format, fingerprinted_at FROM files
|
||||
WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean)
|
||||
ORDER BY id LIMIT $2`, [pythonAvailable, BATCH]);
|
||||
if (rows.length === 0) return;
|
||||
if (rows.length === 0) return false;
|
||||
|
||||
const geometry = [];
|
||||
for (const row of rows) {
|
||||
|
|
@ -99,23 +116,100 @@ async function run() {
|
|||
if (!GEOMETRY_FORMATS.includes(row.format)) {
|
||||
await db.query(`UPDATE files SET geometry_status = 'na' WHERE id = $1`, [row.id]);
|
||||
} else if (pythonAvailable) {
|
||||
geometry.push(row);
|
||||
geometry.push({ id: row.id, format: row.format, abs });
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`Fingerprint failed for file ${row.id}:`, err.message);
|
||||
await db.query(`UPDATE files SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [row.id]);
|
||||
}
|
||||
}
|
||||
if (geometry.length) await withPython(() => fingerprintGeometry(geometry, 'files'));
|
||||
return true;
|
||||
}
|
||||
|
||||
if (geometry.length) {
|
||||
async function indexArchives() {
|
||||
const { rows } = await db.query(
|
||||
`SELECT id, file_path, format FROM files
|
||||
WHERE format = ANY($1) AND archive_status IS NULL ORDER BY id LIMIT 5`, [ARCHIVE_FORMATS]);
|
||||
for (const file of rows) {
|
||||
try {
|
||||
await fingerprintGeometry(geometry);
|
||||
const entries = await archives.listEntries(file);
|
||||
await store.replaceArchiveEntries(file.id,
|
||||
entries.map((e) => ({ path: e.path, size: e.size, format: formatOf(e.path) })));
|
||||
await db.query(`UPDATE files SET archive_status = 'ok', archive_error = NULL WHERE id = $1`, [file.id]);
|
||||
} catch (err) {
|
||||
if (err.code !== 'ENOENT') throw err;
|
||||
pythonAvailable = false;
|
||||
console.warn(`Geometry fingerprints disabled: no Python at ${PYTHON} (content hashes still run).`);
|
||||
console.warn(`Could not read archive (file ${file.id}):`, err.message);
|
||||
await db.query(`UPDATE files SET archive_status = 'error', archive_error = $2 WHERE id = $1`,
|
||||
[file.id, err.message.slice(0, 300)]);
|
||||
}
|
||||
}
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
const ENTRY_PENDING = 'e.fingerprinted_at IS NULL OR (e.geometry_status IS NULL AND $1::boolean)';
|
||||
|
||||
// Fingerprints the pending entries of one archive, extracting its models in a
|
||||
// single pass (cheaper than one extraction per entry, especially for solid 7z).
|
||||
async function fingerprintEntries() {
|
||||
const { rows: [file] } = await db.query(
|
||||
`SELECT f.id, f.file_path, f.format FROM files f
|
||||
WHERE EXISTS (SELECT 1 FROM archive_entries e WHERE e.file_id = f.id AND (${ENTRY_PENDING}))
|
||||
ORDER BY f.id LIMIT 1`, [pythonAvailable]);
|
||||
if (!file) return false;
|
||||
const { rows: entries } = await db.query(
|
||||
`SELECT e.id, e.path, e.format, e.fingerprinted_at FROM archive_entries e
|
||||
WHERE e.file_id = $2 AND (${ENTRY_PENDING}) ORDER BY e.id LIMIT 500`, [pythonAvailable, file.id]);
|
||||
|
||||
// Pictures, documents and nested archives are listed but not matched as duplicates.
|
||||
const others = entries.filter((e) => !MODEL_FORMATS.includes(e.format));
|
||||
if (others.length) {
|
||||
await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'na' WHERE id = ANY($1::int[])`,
|
||||
[others.map((e) => e.id)]);
|
||||
}
|
||||
const models = entries.filter((e) => MODEL_FORMATS.includes(e.format));
|
||||
if (!models.length) return true;
|
||||
|
||||
const tmp = archives.tempDir();
|
||||
try {
|
||||
let extracted = new Map();
|
||||
try {
|
||||
extracted = await archives.extractMany(file, models.map((e) => e.path), tmp);
|
||||
} catch (err) {
|
||||
console.warn(`Could not extract from archive (file ${file.id}):`, err.message);
|
||||
}
|
||||
const geometry = [];
|
||||
for (const entry of models) {
|
||||
const abs = extracted.get(entry.path);
|
||||
try {
|
||||
if (!abs) {
|
||||
await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [entry.id]);
|
||||
continue;
|
||||
}
|
||||
if (!entry.fingerprinted_at) {
|
||||
await db.query('UPDATE archive_entries SET sha256 = $2, fingerprinted_at = now() WHERE id = $1',
|
||||
[entry.id, await sha256File(abs)]);
|
||||
}
|
||||
if (!GEOMETRY_FORMATS.includes(entry.format)) {
|
||||
await db.query(`UPDATE archive_entries SET geometry_status = 'na' WHERE id = $1`, [entry.id]);
|
||||
} else if (pythonAvailable) {
|
||||
geometry.push({ id: entry.id, format: entry.format, abs });
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`Fingerprint failed for archive entry ${entry.id}:`, err.message);
|
||||
await db.query(`UPDATE archive_entries SET fingerprinted_at = now(), geometry_status = 'error' WHERE id = $1`, [entry.id]);
|
||||
}
|
||||
}
|
||||
if (geometry.length) await withPython(() => fingerprintGeometry(geometry, 'archive_entries'));
|
||||
} finally {
|
||||
fs.rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function run() {
|
||||
for (;;) {
|
||||
const worked = [await fingerprintFiles(), await indexArchives(), await fingerprintEntries()];
|
||||
if (!worked.some(Boolean)) return;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -133,9 +227,12 @@ function kick() {
|
|||
|
||||
async function pendingCount() {
|
||||
const { rows } = await db.query(
|
||||
`SELECT COUNT(*)::int AS n FROM files
|
||||
WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean)`, [pythonAvailable]);
|
||||
return rows[0].n;
|
||||
`SELECT (SELECT COUNT(*) FROM files
|
||||
WHERE fingerprinted_at IS NULL OR (geometry_status IS NULL AND $1::boolean))
|
||||
+ (SELECT COUNT(*) FROM files WHERE format = ANY($2) AND archive_status IS NULL)
|
||||
+ (SELECT COUNT(*) FROM archive_entries e WHERE ${ENTRY_PENDING}) AS n`,
|
||||
[pythonAvailable, ARCHIVE_FORMATS]);
|
||||
return Number(rows[0].n);
|
||||
}
|
||||
|
||||
module.exports = { kick, pendingCount, geometryKey, sha256File };
|
||||
|
|
|
|||
81
lib/store.js
81
lib/store.js
|
|
@ -1,4 +1,5 @@
|
|||
const db = require('./db');
|
||||
const { MODEL_FORMATS, ARCHIVE_FORMATS } = require('./fileProcessor');
|
||||
|
||||
// ---- shared SELECT that hydrates a model with tags, projects, files, designer ----
|
||||
const MODEL_SELECT = `
|
||||
|
|
@ -25,7 +26,8 @@ const MODEL_SELECT = `
|
|||
SELECT json_agg(json_build_object(
|
||||
'id', fl.id, 'originalName', fl.original_name, 'fileName', fl.file_name,
|
||||
'format', fl.format, 'size', fl.size, 'volume', fl.volume,
|
||||
'bbox', fl.bbox, 'facets', fl.facets) ORDER BY fl.id) AS files
|
||||
'bbox', fl.bbox, 'facets', fl.facets, 'archiveStatus', fl.archive_status,
|
||||
'entryCount', (SELECT COUNT(*)::int FROM archive_entries ae WHERE ae.file_id = fl.id)) ORDER BY fl.id) AS files
|
||||
FROM files fl WHERE fl.model_id = m.id
|
||||
) f ON true
|
||||
`;
|
||||
|
|
@ -116,17 +118,60 @@ async function updateModel(id, updates) {
|
|||
|
||||
// Returns the on-disk file rows so the caller can unlink them.
|
||||
async function deleteModel(id) {
|
||||
const { rows } = await db.query('SELECT file_path, glb_path FROM files WHERE model_id = $1', [id]);
|
||||
const { rows } = await db.query('SELECT id, file_path, glb_path FROM files WHERE model_id = $1', [id]);
|
||||
const entries = await db.query(
|
||||
'SELECT e.id FROM archive_entries e JOIN files f ON f.id = e.file_id WHERE f.model_id = $1', [id]);
|
||||
const model = await db.query('SELECT thumbnail_path FROM models WHERE id = $1', [id]);
|
||||
const result = await db.query('DELETE FROM models WHERE id = $1 RETURNING id', [id]);
|
||||
if (result.rowCount === 0) return null;
|
||||
return {
|
||||
fileIds: rows.map((r) => r.id),
|
||||
filePaths: rows.map((r) => r.file_path),
|
||||
glbPaths: rows.map((r) => r.glb_path).filter(Boolean),
|
||||
glbPaths: [
|
||||
...rows.map((r) => r.glb_path).filter(Boolean),
|
||||
...entries.rows.map((e) => `glb/entry-${e.id}.glb`),
|
||||
],
|
||||
thumbnailPath: model.rows[0] && model.rows[0].thumbnail_path,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- files / archive entries
|
||||
async function getFile(id) {
|
||||
const { rows } = await db.query('SELECT * FROM files WHERE id = $1', [id]);
|
||||
return rows[0] || null;
|
||||
}
|
||||
|
||||
async function getArchiveEntries(fileId) {
|
||||
const { rows } = await db.query(
|
||||
'SELECT id, path, format, size FROM archive_entries WHERE file_id = $1 ORDER BY path', [fileId]);
|
||||
return rows.map((r) => ({ ...r, size: Number(r.size) }));
|
||||
}
|
||||
|
||||
// An entry plus the archive it lives in ({ id, file_path, format }).
|
||||
async function getArchiveEntry(id) {
|
||||
const { rows } = await db.query(
|
||||
`SELECT e.id, e.file_id, e.path, e.format, e.size, f.file_path, f.format AS archive_format
|
||||
FROM archive_entries e JOIN files f ON f.id = e.file_id WHERE e.id = $1`, [id]);
|
||||
const r = rows[0];
|
||||
if (!r) return null;
|
||||
return {
|
||||
id: r.id, file_id: r.file_id, path: r.path, format: r.format, size: Number(r.size),
|
||||
archive: { id: r.file_id, file_path: r.file_path, format: r.archive_format },
|
||||
};
|
||||
}
|
||||
|
||||
async function replaceArchiveEntries(fileId, entries) {
|
||||
await db.transaction(async (client) => {
|
||||
await client.query('DELETE FROM archive_entries WHERE file_id = $1', [fileId]);
|
||||
if (!entries.length) return;
|
||||
await client.query(
|
||||
`INSERT INTO archive_entries (file_id, path, format, size)
|
||||
SELECT $1::int, * FROM unnest($2::text[], $3::text[], $4::bigint[])
|
||||
ON CONFLICT (file_id, path) DO NOTHING`,
|
||||
[fileId, entries.map((e) => e.path), entries.map((e) => e.format), entries.map((e) => e.size)]);
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- trash
|
||||
async function trashModels(ids) {
|
||||
const { rowCount } = await db.query(
|
||||
|
|
@ -153,21 +198,42 @@ async function countTrashed() {
|
|||
// ---------------------------------------------------------------- duplicates
|
||||
// Files of non-trashed models whose content hash or geometry fingerprint also
|
||||
// appears in another non-trashed model — the input to the duplicate finder.
|
||||
// Ids are prefixed ('f12' for a file, 'e7' for an archive entry) so both kinds
|
||||
// can be matched together. Pictures and documents never count.
|
||||
async function duplicateCandidateFiles() {
|
||||
const { rows } = await db.query(`
|
||||
WITH live AS (
|
||||
SELECT f.id, f.model_id, f.sha256, f.geometry_hash
|
||||
SELECT 'f' || f.id AS id, f.model_id, f.sha256, f.geometry_hash
|
||||
FROM files f JOIN models m ON m.id = f.model_id
|
||||
WHERE m.deleted_at IS NULL AND f.size > 0
|
||||
WHERE m.deleted_at IS NULL AND f.size > 0 AND f.format = ANY($1)
|
||||
UNION ALL
|
||||
SELECT 'e' || e.id, f.model_id, e.sha256, e.geometry_hash
|
||||
FROM archive_entries e JOIN files f ON f.id = e.file_id JOIN models m ON m.id = f.model_id
|
||||
WHERE m.deleted_at IS NULL AND e.size > 0 AND e.format = ANY($2)
|
||||
)
|
||||
SELECT * FROM live
|
||||
WHERE sha256 IN (SELECT sha256 FROM live WHERE sha256 IS NOT NULL
|
||||
GROUP BY sha256 HAVING COUNT(DISTINCT model_id) > 1)
|
||||
OR geometry_hash IN (SELECT geometry_hash FROM live WHERE geometry_hash IS NOT NULL
|
||||
GROUP BY geometry_hash HAVING COUNT(DISTINCT model_id) > 1)`);
|
||||
GROUP BY geometry_hash HAVING COUNT(DISTINCT model_id) > 1)`,
|
||||
[[...MODEL_FORMATS, ...ARCHIVE_FORMATS], MODEL_FORMATS]);
|
||||
return rows;
|
||||
}
|
||||
|
||||
// How many duplicate-matchable items (model files, archives, models inside
|
||||
// archives) each model holds: { modelId: n }.
|
||||
async function matchableCounts(modelIds) {
|
||||
if (!modelIds.length) return {};
|
||||
const { rows } = await db.query(`
|
||||
SELECT m.id,
|
||||
(SELECT COUNT(*) FROM files f WHERE f.model_id = m.id AND f.format = ANY($2))
|
||||
+ (SELECT COUNT(*) FROM archive_entries e JOIN files f ON f.id = e.file_id
|
||||
WHERE f.model_id = m.id AND e.format = ANY($3)) AS n
|
||||
FROM models m WHERE m.id = ANY($1::int[])`,
|
||||
[modelIds, [...MODEL_FORMATS, ...ARCHIVE_FORMATS], MODEL_FORMATS]);
|
||||
return Object.fromEntries(rows.map((r) => [r.id, Number(r.n)]));
|
||||
}
|
||||
|
||||
// Keep one model of a duplicate group: fold the others' tags, collections and
|
||||
// notes into it, then move the others to the trash.
|
||||
async function mergeAndTrash(keepId, otherIds) {
|
||||
|
|
@ -407,7 +473,8 @@ module.exports = {
|
|||
getAllModels, getModel, getModelsByIds, findModelBySource, createModel, updateModel, deleteModel, addFiles,
|
||||
updateFileConversion,
|
||||
trashModels, restoreModel, trashedModelIds, countTrashed,
|
||||
duplicateCandidateFiles, mergeAndTrash, dismissDuplicateGroup, dismissedDuplicateSignatures,
|
||||
duplicateCandidateFiles, matchableCounts, mergeAndTrash, dismissDuplicateGroup, dismissedDuplicateSignatures,
|
||||
getFile, getArchiveEntries, getArchiveEntry, replaceArchiveEntries,
|
||||
getAllTags,
|
||||
getAllProjects, getProject, createProject, updateProject, deleteProject,
|
||||
getSetting, setSetting,
|
||||
|
|
|
|||
109
package-lock.json
generated
109
package-lock.json
generated
|
|
@ -9,6 +9,7 @@
|
|||
"version": "1.0.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"7z-wasm": "^1.2.0",
|
||||
"adm-zip": "^0.5.17",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"cookie-parser": "^1.4.7",
|
||||
|
|
@ -18,12 +19,42 @@
|
|||
"multer": "^1.4.5-lts.1",
|
||||
"node-stl": "^0.7.0",
|
||||
"nodemailer": "^9.0.1",
|
||||
"pg": "^8.22.0"
|
||||
"occt-import-js": "^0.0.23",
|
||||
"pg": "^8.22.0",
|
||||
"rhino3dm": "^8.32.2",
|
||||
"tar": "^7.5.22"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@isaacs/fs-minipass": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
|
||||
"integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"minipass": "^7.0.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/7z-wasm": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/7z-wasm/-/7z-wasm-1.2.0.tgz",
|
||||
"integrity": "sha512-AZjtWXFleAe5nQ27JYISs70cICjO0zZr4LU45I9pXKcrhG0qiEWgq3Vjwr9JHHihFLuqgc7uSHYVn/z3vXCjdw==",
|
||||
"license": "SEE LICENSE IN License.txt",
|
||||
"dependencies": {
|
||||
"readline-sync": "^1.4.10"
|
||||
},
|
||||
"bin": {
|
||||
"7z-wasm": "bin/cli"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/accepts": {
|
||||
"version": "1.3.8",
|
||||
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
|
||||
|
|
@ -240,6 +271,15 @@
|
|||
"fsevents": "~2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/chownr": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
|
||||
"integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
|
||||
"license": "BlueOak-1.0.0",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/concat-stream": {
|
||||
"version": "1.6.2",
|
||||
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
|
||||
|
|
@ -945,6 +985,27 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/minipass": {
|
||||
"version": "7.1.3",
|
||||
"resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz",
|
||||
"integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==",
|
||||
"license": "BlueOak-1.0.0",
|
||||
"engines": {
|
||||
"node": ">=16 || 14 >=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/minizlib": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz",
|
||||
"integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"minipass": "^7.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 18"
|
||||
}
|
||||
},
|
||||
"node_modules/mkdirp": {
|
||||
"version": "0.5.6",
|
||||
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
|
||||
|
|
@ -1091,6 +1152,12 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/occt-import-js": {
|
||||
"version": "0.0.23",
|
||||
"resolved": "https://registry.npmjs.org/occt-import-js/-/occt-import-js-0.0.23.tgz",
|
||||
"integrity": "sha512-RFfYQXYFX5C1mB1Aywm0ShcUKzXOr/VzTnlzhBSDJOR6YCAPt1HYCzeXWg1vwwjn/cUxwqRNhhtf1dlewoZYCQ==",
|
||||
"license": "LGPL-2.1"
|
||||
},
|
||||
"node_modules/on-finished": {
|
||||
"version": "2.4.1",
|
||||
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
|
||||
|
|
@ -1358,6 +1425,21 @@
|
|||
"node": ">=8.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/readline-sync": {
|
||||
"version": "1.4.10",
|
||||
"resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz",
|
||||
"integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/rhino3dm": {
|
||||
"version": "8.32.2",
|
||||
"resolved": "https://registry.npmjs.org/rhino3dm/-/rhino3dm-8.32.2.tgz",
|
||||
"integrity": "sha512-Q3/5iHIKJeoE5doXrRy7C1vJer4F8uMYe8K6B8VQHqhGj5BCS1L3EkqNpKyUTERoHyKh7rMmiiZpw9iEUkCnPQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
|
|
@ -1586,6 +1668,22 @@
|
|||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/tar": {
|
||||
"version": "7.5.22",
|
||||
"resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz",
|
||||
"integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==",
|
||||
"license": "BlueOak-1.0.0",
|
||||
"dependencies": {
|
||||
"@isaacs/fs-minipass": "^4.0.0",
|
||||
"chownr": "^3.0.0",
|
||||
"minipass": "^7.1.2",
|
||||
"minizlib": "^3.1.0",
|
||||
"yallist": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/to-regex-range": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
|
||||
|
|
@ -1685,6 +1783,15 @@
|
|||
"engines": {
|
||||
"node": ">=0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/yallist": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz",
|
||||
"integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==",
|
||||
"license": "BlueOak-1.0.0",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
"author": "",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"7z-wasm": "^1.2.0",
|
||||
"adm-zip": "^0.5.17",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"cookie-parser": "^1.4.7",
|
||||
|
|
@ -26,7 +27,10 @@
|
|||
"multer": "^1.4.5-lts.1",
|
||||
"node-stl": "^0.7.0",
|
||||
"nodemailer": "^9.0.1",
|
||||
"pg": "^8.22.0"
|
||||
"occt-import-js": "^0.0.23",
|
||||
"pg": "^8.22.0",
|
||||
"rhino3dm": "^8.32.2",
|
||||
"tar": "^7.5.22"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.0.2"
|
||||
|
|
|
|||
145
public/app.js
145
public/app.js
|
|
@ -189,8 +189,8 @@ async function generateMissingThumbnails() {
|
|||
if (!file) continue;
|
||||
thumbAttempted.add(m.id);
|
||||
try {
|
||||
const src = viewerSourceFor(file.id, file.format);
|
||||
const image = await window.ModelViewer.snapshot(src.url, src.format, 512);
|
||||
const src = viewerSourceFor({ kind: 'file', id: file.id, format: file.format });
|
||||
const image = await window.ModelViewer.snapshot(src.url, src.format, 512, { resolve: resolverFor(m) });
|
||||
if (!image) continue;
|
||||
const res = await fetch(`/api/models/${m.id}/thumbnail`, {
|
||||
method: 'POST',
|
||||
|
|
@ -484,7 +484,7 @@ function duplicateGroupHtml(g) {
|
|||
|
||||
function duplicateCardHtml(g, m) {
|
||||
const recommended = m.id === g.recommendedId;
|
||||
const unique = m.files.length - m.matchedFiles;
|
||||
const unique = m.matchableFiles - m.matchedFiles;
|
||||
return `
|
||||
<div class="dup-card${recommended ? ' recommended' : ''}" data-model-id="${m.id}">
|
||||
<div class="file-thumbnail">${thumbHtml(m)}</div>
|
||||
|
|
@ -492,7 +492,7 @@ function duplicateCardHtml(g, m) {
|
|||
${recommended ? '<span class="rec-badge">Recommended keep</span>' : ''}
|
||||
<div class="file-name" title="${escapeHtml(m.name)}">${escapeHtml(m.name)}</div>
|
||||
<div class="file-meta">${escapeHtml(sourceLabel(sourceKey(m)))} • ${formatDate(m.date_added)}</div>
|
||||
<div class="file-meta">${m.matchedFiles} of ${m.files.length} files match${unique ? ` • <span class="warn">${unique} unique</span>` : ''}</div>
|
||||
<div class="file-meta">${m.matchedFiles} of ${m.matchableFiles} files match${unique ? ` • <span class="warn">${unique} unique</span>` : ''}</div>
|
||||
<button class="btn btn-primary btn-sm" data-action="keep">Keep this one</button>
|
||||
</div>
|
||||
</div>`;
|
||||
|
|
@ -505,7 +505,7 @@ async function handleKeepDuplicate(signature, keepId) {
|
|||
const others = group.models.filter((m) => m.id !== keepId);
|
||||
let message = `Keep "${keep.name}" and move ${others.length} other model${others.length === 1 ? '' : 's'} to the trash?\n\n`
|
||||
+ 'Their tags, collections and notes are merged into the one you keep.';
|
||||
const withUnique = others.filter((m) => m.files.length > m.matchedFiles);
|
||||
const withUnique = others.filter((m) => m.matchableFiles > m.matchedFiles);
|
||||
if (withUnique.length) {
|
||||
message += `\n\nHeads up: ${withUnique.map((m) => `"${m.name}"`).join(', ')} also contain files that aren't duplicated — they go to the trash too.`;
|
||||
}
|
||||
|
|
@ -700,6 +700,22 @@ function setupEventListeners() {
|
|||
});
|
||||
document.getElementById('detailNotes').addEventListener('input', queueNotesSave);
|
||||
document.getElementById('detailNotes').addEventListener('blur', flushNotes);
|
||||
document.getElementById('detailFileTabs').addEventListener('click', (e) => {
|
||||
const tab = e.target.closest('.file-tab');
|
||||
const item = tab && detailItems.find((i) => itemKey(i) === tab.dataset.key);
|
||||
if (!item) return;
|
||||
renderViewerTabs(itemKey(item));
|
||||
loadViewer(item);
|
||||
});
|
||||
document.getElementById('detailFileInfo').addEventListener('click', (e) => {
|
||||
const btn = e.target.closest('[data-view-entry]');
|
||||
if (!btn) return;
|
||||
const archiveId = Number(btn.dataset.archive);
|
||||
const entry = (archiveEntryCache.get(archiveId) || []).find((x) => String(x.id) === btn.dataset.viewEntry);
|
||||
if (!entry) return;
|
||||
openViewerItem({ kind: 'entry', id: entry.id, format: entry.format, name: entry.path.split('/').pop(), archiveId });
|
||||
document.getElementById('detailViewer').scrollIntoView({ block: 'nearest', behavior: 'smooth' });
|
||||
});
|
||||
|
||||
setupTagAutocomplete(document.getElementById('fileTags'));
|
||||
setupTagAutocomplete(document.getElementById('editTags'));
|
||||
|
|
@ -924,38 +940,98 @@ function openModelDetail(modelId) {
|
|||
<div class="file-row">
|
||||
<span>${escapeHtml(f.originalName)} <em>(${formatFileSize(f.size)}${f.volume ? `, ${Number(f.volume).toFixed(2)} cm³` : ''})</em></span>
|
||||
<a href="/api/files/${f.id}/raw" download>download</a>
|
||||
</div>`).join('');
|
||||
</div>
|
||||
${ARCHIVE_FORMATS.includes(f.format) ? `<div class="entry-list" data-entries-for="${f.id}"><div class="entry-head">Reading archive…</div></div>` : ''}`).join('');
|
||||
model.files.filter((f) => ARCHIVE_FORMATS.includes(f.format)).forEach((f) => loadArchiveEntries(model.id, f.id));
|
||||
}
|
||||
|
||||
// Viewer tabs
|
||||
const renderable = model.files.filter((f) => window.ModelViewer.RENDERABLE.includes(f.format));
|
||||
const tabs = document.getElementById('detailFileTabs');
|
||||
tabs.innerHTML = renderable.map((f, i) => `
|
||||
<button class="file-tab${i === 0 ? ' active' : ''}" data-file-id="${f.id}" data-format="${f.format}">${escapeHtml(f.originalName)}</button>`).join('');
|
||||
tabs.querySelectorAll('.file-tab').forEach((tab) => {
|
||||
tab.addEventListener('click', () => {
|
||||
tabs.querySelectorAll('.file-tab').forEach((t) => t.classList.remove('active'));
|
||||
tab.classList.add('active');
|
||||
loadViewer(tab.dataset.fileId, tab.dataset.format);
|
||||
});
|
||||
});
|
||||
// Viewer tabs: 3D files first, then pictures/PDFs/text (entries inside
|
||||
// archives get a tab when opened from the file list).
|
||||
const { RENDERABLE, DOCUMENT_FORMATS } = window.ModelViewer;
|
||||
detailItems = [
|
||||
...model.files.filter((f) => RENDERABLE.includes(f.format)),
|
||||
...model.files.filter((f) => DOCUMENT_FORMATS.includes(f.format) && f.format !== 'mtl'),
|
||||
].map((f) => ({ kind: 'file', id: f.id, format: f.format, name: f.originalName }));
|
||||
renderViewerTabs(detailItems.length ? itemKey(detailItems[0]) : null);
|
||||
|
||||
fileDetailModal.classList.add('active');
|
||||
currentDetailFile = null;
|
||||
if (renderable.length) {
|
||||
loadViewer(renderable[0].id, renderable[0].format);
|
||||
if (detailItems.length) {
|
||||
loadViewer(detailItems[0]);
|
||||
} else {
|
||||
document.getElementById('regenThumbBtn').style.display = 'none';
|
||||
window.ModelViewer.render(document.getElementById('detailViewer'), '', 'none', { fallbackImage: currentDetailThumb });
|
||||
}
|
||||
}
|
||||
|
||||
// Maps a file to what the viewer should load: 3MF goes through the server's
|
||||
// GLB conversion; everything else is served raw.
|
||||
function viewerSourceFor(fileId, format) {
|
||||
return format === '3mf'
|
||||
? { url: `/api/files/${fileId}/glb`, format: 'glb' }
|
||||
: { url: `/api/files/${fileId}/raw`, format };
|
||||
// ---------------------------------------------------------------- viewer items
|
||||
// An item is something the viewer can show: { kind: 'file' | 'entry', id, format, name, archiveId? }.
|
||||
const ARCHIVE_FORMATS = ['zip', '7z', 'rar', 'tar', 'tgz'];
|
||||
let detailItems = [];
|
||||
const archiveEntryCache = new Map(); // archive file id -> entries
|
||||
const itemKey = (item) => `${item.kind}:${item.id}`;
|
||||
|
||||
function renderViewerTabs(activeKey) {
|
||||
document.getElementById('detailFileTabs').innerHTML = detailItems.map((item) => `
|
||||
<button class="file-tab${itemKey(item) === activeKey ? ' active' : ''}" data-key="${itemKey(item)}">${escapeHtml(item.name)}</button>`).join('');
|
||||
}
|
||||
|
||||
function openViewerItem(item) {
|
||||
if (!detailItems.some((i) => itemKey(i) === itemKey(item))) detailItems.push(item);
|
||||
renderViewerTabs(itemKey(item));
|
||||
loadViewer(item);
|
||||
}
|
||||
|
||||
// What the viewer should load: 3MF goes through the server's GLB conversion,
|
||||
// documents are served inline, everything else raw.
|
||||
function viewerSourceFor(item) {
|
||||
const base = `/api/${item.kind === 'entry' ? 'entries' : 'files'}/${item.id}`;
|
||||
if (item.format === '3mf') return { url: `${base}/glb`, format: 'glb' };
|
||||
if (window.ModelViewer.DOCUMENT_FORMATS.includes(item.format)) return { url: `${base}/raw?inline=1`, format: item.format };
|
||||
return { url: `${base}/raw`, format: item.format };
|
||||
}
|
||||
|
||||
// Finds files shipped alongside a model by name — OBJ materials, FBX/MTL
|
||||
// textures. Entries of the same archive win, then the model's own files.
|
||||
function resolverFor(model, archiveEntries = null) {
|
||||
const byName = new Map();
|
||||
const key = (name) => String(name).split(/[\\/]/).pop().toLowerCase();
|
||||
const add = (name, url) => { if (!byName.has(key(name))) byName.set(key(name), url); };
|
||||
(archiveEntries || []).forEach((e) => add(e.path, `/api/entries/${e.id}/raw`));
|
||||
((model && model.files) || []).forEach((f) => add(f.originalName, `/api/files/${f.id}/raw`));
|
||||
return (name) => byName.get(key(name)) || null;
|
||||
}
|
||||
|
||||
async function loadArchiveEntries(modelId, fileId) {
|
||||
const container = () => (fileDetailModal.dataset.currentModelId === String(modelId)
|
||||
? document.querySelector(`[data-entries-for="${fileId}"]`) : null);
|
||||
try {
|
||||
const res = await fetch(`/api/files/${fileId}/entries`);
|
||||
const data = await res.json();
|
||||
if (!res.ok) throw new Error(data.error);
|
||||
if (!container()) return;
|
||||
if (data.status === 'pending') {
|
||||
setTimeout(() => { if (container()) loadArchiveEntries(modelId, fileId); }, 2000);
|
||||
return;
|
||||
}
|
||||
if (data.status === 'error') {
|
||||
container().innerHTML = `<div class="entry-head">Couldn't read this archive: ${escapeHtml(data.error || 'unknown error')}</div>`;
|
||||
return;
|
||||
}
|
||||
archiveEntryCache.set(fileId, data.entries);
|
||||
const { PREVIEWABLE } = window.ModelViewer;
|
||||
container().innerHTML = `<div class="entry-head">${data.entries.length} file${data.entries.length === 1 ? '' : 's'} inside</div>`
|
||||
+ data.entries.map((e) => `
|
||||
<div class="entry-row">
|
||||
<span title="${escapeHtml(e.path)}">${escapeHtml(e.path.replace(/^\.\//, ''))} <em>${formatFileSize(e.size)}</em></span>
|
||||
<span class="row-actions">
|
||||
${PREVIEWABLE.includes(e.format) && e.format !== 'mtl' ? `<button type="button" class="link-btn" data-view-entry="${e.id}" data-archive="${fileId}">view</button>` : ''}
|
||||
<a href="/api/entries/${e.id}/raw" download>download</a>
|
||||
</span>
|
||||
</div>`).join('');
|
||||
} catch (err) {
|
||||
if (container()) container().innerHTML = `<div class="entry-head">Couldn't list this archive: ${escapeHtml(err.message)}</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
function renderDetailLicense(model) {
|
||||
|
|
@ -1068,11 +1144,15 @@ async function handleRefreshMetadata() {
|
|||
}
|
||||
}
|
||||
|
||||
function loadViewer(fileId, format) {
|
||||
const src = viewerSourceFor(fileId, format);
|
||||
currentDetailFile = { id: fileId, viewerUrl: src.url, viewerFormat: src.format };
|
||||
document.getElementById('regenThumbBtn').style.display = '';
|
||||
window.ModelViewer.render(document.getElementById('detailViewer'), src.url, src.format, { fallbackImage: currentDetailThumb });
|
||||
function loadViewer(item) {
|
||||
const src = viewerSourceFor(item);
|
||||
const model = findModel(fileDetailModal.dataset.currentModelId);
|
||||
const resolve = resolverFor(model, item.kind === 'entry' ? archiveEntryCache.get(item.archiveId) : null);
|
||||
const is3d = window.ModelViewer.MODEL_FORMATS.includes(src.format);
|
||||
currentDetailFile = is3d ? { viewerUrl: src.url, viewerFormat: src.format, resolve } : null;
|
||||
document.getElementById('regenThumbBtn').style.display = is3d ? '' : 'none';
|
||||
window.ModelViewer.render(document.getElementById('detailViewer'), src.url, src.format,
|
||||
{ fallbackImage: currentDetailThumb, resolve });
|
||||
}
|
||||
|
||||
function updateCardThumbnail(modelId, thumbnailUrl) {
|
||||
|
|
@ -1089,7 +1169,8 @@ async function handleRegenThumbnail() {
|
|||
btn.disabled = true;
|
||||
btn.textContent = 'Rendering…';
|
||||
try {
|
||||
const image = await window.ModelViewer.snapshot(currentDetailFile.viewerUrl, currentDetailFile.viewerFormat, 512);
|
||||
const image = await window.ModelViewer.snapshot(currentDetailFile.viewerUrl, currentDetailFile.viewerFormat, 512,
|
||||
{ resolve: currentDetailFile.resolve });
|
||||
const res = await fetch(`/api/models/${modelId}/thumbnail`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
|
|
|
|||
|
|
@ -329,8 +329,8 @@
|
|||
<div class="modal-body">
|
||||
<form id="uploadForm">
|
||||
<div class="form-group">
|
||||
<label for="fileInput">Files * <span class="form-hint-inline">(stl, 3mf, obj, step, gcode, zip — multiple allowed)</span></label>
|
||||
<input type="file" id="fileInput" accept=".stl,.3mf,.obj,.step,.stp,.gcode,.gco,.zip" multiple required>
|
||||
<label for="fileInput">Files * <span class="form-hint-inline">(models, archives, pictures, PDFs & readmes — multiple allowed)</span></label>
|
||||
<input type="file" id="fileInput" accept=".stl,.3mf,.obj,.fbx,.step,.stp,.3dm,.gcode,.gco,.zip,.7z,.rar,.tar,.tgz,.gz,.jpg,.jpeg,.png,.gif,.webp,.bmp,.svg,.pdf,.txt,.md,.mtl" multiple required>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
|
|
@ -448,7 +448,7 @@
|
|||
<button type="button" class="btn btn-secondary btn-sm" id="downloadFilesBtn" style="margin-top:10px; display:none"></button>
|
||||
</div>
|
||||
<input type="file" id="addFilesInput" multiple style="display:none"
|
||||
accept=".stl,.3mf,.obj,.step,.stp,.gcode,.gco,.zip">
|
||||
accept=".stl,.3mf,.obj,.fbx,.step,.stp,.3dm,.gcode,.gco,.zip,.7z,.rar,.tar,.tgz,.gz,.jpg,.jpeg,.png,.gif,.webp,.bmp,.svg,.pdf,.txt,.md,.mtl">
|
||||
<div class="form-actions" id="detailActions">
|
||||
<button class="btn btn-danger" id="deleteFileBtn">Move to trash</button>
|
||||
<button class="btn btn-secondary" id="addFilesBtn">Add files</button>
|
||||
|
|
|
|||
|
|
@ -214,6 +214,20 @@ label { display: block; font-size: 13px; font-weight: 500; color: var(--text-mut
|
|||
.viewer-fallback img { max-width: 100%; max-height: 320px; object-fit: contain; border-radius: var(--radius-sm); }
|
||||
.viewer-fallback span { color: var(--text-dim); font-size: 12px; text-align: center; }
|
||||
|
||||
.viewer-image { max-width: 100%; max-height: 100%; object-fit: contain; }
|
||||
.viewer-pdf, .viewer-text { align-self: stretch; width: 100%; height: 100%; border: 0; border-radius: var(--radius); }
|
||||
.viewer-pdf { background: #fff; }
|
||||
.viewer-text { margin: 0; padding: 14px 16px; overflow: auto; font-family: var(--mono); font-size: 12px; line-height: 1.55; color: var(--text); white-space: pre-wrap; word-break: break-word; text-align: left; }
|
||||
|
||||
.info-section .entry-list { margin: 2px 0 8px 10px; padding-left: 12px; border-left: 1px solid var(--border); max-height: 260px; overflow-y: auto; }
|
||||
.info-section .entry-head { font-size: 12px; color: var(--text-dim); margin: 6px 0 2px; }
|
||||
.info-section .entry-row { display: flex; justify-content: space-between; align-items: center; gap: 10px; padding: 3px 0; font-size: 13px; color: var(--text-muted); }
|
||||
.entry-row > span:first-child { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; }
|
||||
.entry-row em { color: var(--text-dim); font-style: normal; font-family: var(--mono); font-size: 11px; margin-left: 4px; }
|
||||
.row-actions { display: flex; gap: 12px; flex-shrink: 0; align-items: center; }
|
||||
.link-btn { background: none; border: none; color: var(--accent); font: inherit; cursor: pointer; padding: 0; }
|
||||
.link-btn:hover { text-decoration: underline; }
|
||||
|
||||
.file-tabs { display: flex; flex-wrap: wrap; gap: 6px; }
|
||||
.file-tab { border: 1px solid var(--border); background: var(--surface-2); color: var(--text-muted); padding: 5px 11px; border-radius: var(--radius-sm); font-size: 12px; cursor: pointer; font-family: inherit; }
|
||||
.file-tab:hover { color: var(--text); }
|
||||
|
|
|
|||
215
public/viewer.js
215
public/viewer.js
|
|
@ -1,16 +1,27 @@
|
|||
// In-browser 3D preview for STL / 3MF / OBJ files, loaded as an ES module.
|
||||
// In-browser preview for 3D models (STL, OBJ + MTL, 3MF via GLB, FBX, STEP, 3DM)
|
||||
// and for the pictures, PDFs and text files that come with them.
|
||||
import * as THREE from 'three';
|
||||
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
|
||||
import { STLLoader } from 'three/addons/loaders/STLLoader.js';
|
||||
import { OBJLoader } from 'three/addons/loaders/OBJLoader.js';
|
||||
import { ThreeMFLoader } from 'three/addons/loaders/3MFLoader.js';
|
||||
import { MTLLoader } from 'three/addons/loaders/MTLLoader.js';
|
||||
import { FBXLoader } from 'three/addons/loaders/FBXLoader.js';
|
||||
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
|
||||
import { Rhino3dmLoader } from 'three/addons/loaders/3DMLoader.js';
|
||||
|
||||
// Formats the 3D renderer draws directly (3MF arrives as a server-converted GLB).
|
||||
const MODEL_FORMATS = ['stl', 'obj', 'glb', 'fbx', '3dm', 'step', 'stp'];
|
||||
const IMAGE_FORMATS = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'svg'];
|
||||
const TEXT_FORMATS = ['txt', 'md', 'mtl'];
|
||||
const DOCUMENT_FORMATS = [...IMAGE_FORMATS, 'pdf', ...TEXT_FORMATS];
|
||||
// What the app offers a preview for.
|
||||
const RENDERABLE = ['stl', '3mf', 'obj', 'fbx', '3dm', 'step', 'stp'];
|
||||
const PREVIEWABLE = [...RENDERABLE, ...DOCUMENT_FORMATS];
|
||||
// Formats authored Z-up; turned to sit upright in three.js's Y-up world.
|
||||
const Z_UP = ['stl', 'glb', '3dm', 'step', 'stp'];
|
||||
|
||||
// Formats the renderer can draw directly. 3MF is rendered via server-converted
|
||||
// GLB (see RENDERABLE for what the app treats as previewable input).
|
||||
const SUPPORTED = ['stl', 'obj', 'glb'];
|
||||
const RENDERABLE = ['stl', '3mf', 'obj'];
|
||||
let active = null; // { renderer, controls, rafId }
|
||||
let renderToken = 0; // bumps on every render, so a slow earlier load can't draw over a newer one
|
||||
|
||||
function dispose() {
|
||||
if (!active) return;
|
||||
|
|
@ -26,8 +37,7 @@ function message(container, text) {
|
|||
container.innerHTML = `<div class="viewer-msg">${text}</div>`;
|
||||
}
|
||||
|
||||
// When 3D rendering isn't possible (e.g. Bambu/MakerWorld 3MF project files that
|
||||
// the loader can't parse), show the model's preview image instead of an error.
|
||||
// When 3D rendering isn't possible, show the model's preview image instead of an error.
|
||||
function showFallback(container, image, text) {
|
||||
if (image) {
|
||||
container.innerHTML =
|
||||
|
|
@ -37,33 +47,172 @@ function showFallback(container, image, text) {
|
|||
}
|
||||
}
|
||||
|
||||
async function loadObject(url, format) {
|
||||
async function fetchOk(url) {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status} loading ${url}`);
|
||||
return res;
|
||||
}
|
||||
|
||||
const basename = (p) => String(p).split(/[\\/]/).pop();
|
||||
|
||||
function defaultMaterial(color) {
|
||||
return new THREE.MeshStandardMaterial({ color: color || 0xc4cad6, metalness: 0.15, roughness: 0.65 });
|
||||
}
|
||||
|
||||
// A loading manager that points material and texture references ("wood.png",
|
||||
// "../tex/wood.png") at the files shipped alongside the model. `own` is the
|
||||
// URL being loaded itself, which is left alone.
|
||||
function managerFor(resolve, own) {
|
||||
const manager = new THREE.LoadingManager();
|
||||
if (resolve) {
|
||||
manager.setURLModifier((url) => {
|
||||
if (url === own || url.startsWith('blob:') || url.startsWith('data:')) return url;
|
||||
return resolve(decodeURIComponent(basename(url.split('?')[0]))) || url;
|
||||
});
|
||||
}
|
||||
return manager;
|
||||
}
|
||||
|
||||
async function loadObj(url, resolve) {
|
||||
const text = await (await fetchOk(url)).text();
|
||||
const loader = new OBJLoader();
|
||||
const mtllib = /^mtllib\s+(.+)$/m.exec(text);
|
||||
const mtlUrl = mtllib && resolve ? resolve(basename(mtllib[1].trim())) : null;
|
||||
let textured = false;
|
||||
if (mtlUrl) {
|
||||
try {
|
||||
const materials = await new MTLLoader(managerFor(resolve, mtlUrl)).loadAsync(mtlUrl);
|
||||
materials.preload();
|
||||
loader.setMaterials(materials);
|
||||
textured = true;
|
||||
} catch { /* fall back to plain grey */ }
|
||||
}
|
||||
const object = loader.parse(text);
|
||||
if (!textured) object.traverse((c) => { if (c.isMesh) c.material = defaultMaterial(); });
|
||||
return object;
|
||||
}
|
||||
|
||||
// OpenCascade (WASM) is large, so it's fetched the first time a STEP file is opened.
|
||||
let occtPromise = null;
|
||||
function occt() {
|
||||
if (!occtPromise) {
|
||||
occtPromise = new Promise((resolve, reject) => {
|
||||
const script = document.createElement('script');
|
||||
script.src = '/vendor/occt/occt-import-js.js';
|
||||
script.onload = () => resolve(window.occtimportjs({ locateFile: (name) => `/vendor/occt/${name}` }));
|
||||
script.onerror = () => { occtPromise = null; reject(new Error('Could not load the STEP reader')); };
|
||||
document.head.appendChild(script);
|
||||
});
|
||||
}
|
||||
return occtPromise;
|
||||
}
|
||||
|
||||
async function loadStep(url) {
|
||||
const [reader, buffer] = await Promise.all([occt(), fetchOk(url).then((r) => r.arrayBuffer())]);
|
||||
const result = reader.ReadStepFile(new Uint8Array(buffer), null);
|
||||
if (!result.success || !result.meshes.length) throw new Error('No geometry in this STEP file');
|
||||
const group = new THREE.Group();
|
||||
for (const m of result.meshes) {
|
||||
const geometry = new THREE.BufferGeometry();
|
||||
geometry.setAttribute('position', new THREE.Float32BufferAttribute(m.attributes.position.array, 3));
|
||||
if (m.attributes.normal) geometry.setAttribute('normal', new THREE.Float32BufferAttribute(m.attributes.normal.array, 3));
|
||||
geometry.setIndex(Array.from(m.index.array));
|
||||
if (!m.attributes.normal) geometry.computeVertexNormals();
|
||||
const color = m.color ? new THREE.Color(m.color[0], m.color[1], m.color[2]) : null;
|
||||
group.add(new THREE.Mesh(geometry, defaultMaterial(color)));
|
||||
}
|
||||
return group;
|
||||
}
|
||||
|
||||
async function load3dm(url) {
|
||||
const loader = new Rhino3dmLoader().setLibraryPath('/vendor/rhino3dm/');
|
||||
try {
|
||||
return await loader.loadAsync(url);
|
||||
} finally {
|
||||
loader.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
async function loadObject(url, format, opts = {}) {
|
||||
let object;
|
||||
if (format === 'stl') {
|
||||
const geometry = await new STLLoader().loadAsync(url);
|
||||
geometry.computeVertexNormals();
|
||||
const material = new THREE.MeshStandardMaterial({ color: 0xc4cad6, metalness: 0.15, roughness: 0.65 });
|
||||
const mesh = new THREE.Mesh(geometry, material);
|
||||
mesh.rotation.x = -Math.PI / 2; // STL is typically Z-up; sit it flat
|
||||
return mesh;
|
||||
}
|
||||
if (format === 'obj') return new OBJLoader().loadAsync(url);
|
||||
if (format === '3mf') return new ThreeMFLoader().loadAsync(url);
|
||||
if (format === 'glb') {
|
||||
const gltf = await new GLTFLoader().loadAsync(url);
|
||||
gltf.scene.rotation.x = -Math.PI / 2; // 3MF/trimesh is Z-up; sit it upright
|
||||
return gltf.scene;
|
||||
}
|
||||
object = new THREE.Mesh(geometry, defaultMaterial());
|
||||
} else if (format === 'obj') {
|
||||
object = await loadObj(url, opts.resolve);
|
||||
} else if (format === 'fbx') {
|
||||
object = await new FBXLoader(managerFor(opts.resolve, url)).loadAsync(url);
|
||||
} else if (format === 'glb') {
|
||||
object = (await new GLTFLoader().loadAsync(url)).scene;
|
||||
} else if (format === '3dm') {
|
||||
object = await load3dm(url);
|
||||
} else if (format === 'step' || format === 'stp') {
|
||||
object = await loadStep(url);
|
||||
} else {
|
||||
throw new Error(`Unsupported format ${format}`);
|
||||
}
|
||||
if (Z_UP.includes(format)) object.rotation.x = -Math.PI / 2;
|
||||
return object;
|
||||
}
|
||||
|
||||
async function renderDocument(container, url, format, token) {
|
||||
if (IMAGE_FORMATS.includes(format)) {
|
||||
const img = document.createElement('img');
|
||||
img.className = 'viewer-image';
|
||||
img.alt = '';
|
||||
img.src = url;
|
||||
container.appendChild(img);
|
||||
} else if (format === 'pdf') {
|
||||
const frame = document.createElement('iframe');
|
||||
frame.className = 'viewer-pdf';
|
||||
frame.title = 'PDF preview';
|
||||
frame.src = url;
|
||||
container.appendChild(frame);
|
||||
} else {
|
||||
let text;
|
||||
try {
|
||||
text = await (await fetchOk(url)).text();
|
||||
} catch {
|
||||
text = 'Could not load this file.';
|
||||
}
|
||||
if (token !== renderToken) return;
|
||||
const pre = document.createElement('pre');
|
||||
pre.className = 'viewer-text';
|
||||
pre.textContent = text.length > 200000 ? `${text.slice(0, 200000)}\n…` : text;
|
||||
container.appendChild(pre);
|
||||
}
|
||||
}
|
||||
|
||||
async function render(container, url, format, opts = {}) {
|
||||
dispose();
|
||||
const token = ++renderToken;
|
||||
container.innerHTML = '';
|
||||
if (!SUPPORTED.includes(format)) {
|
||||
if (DOCUMENT_FORMATS.includes(format)) {
|
||||
await renderDocument(container, url, format, token);
|
||||
return;
|
||||
}
|
||||
if (!MODEL_FORMATS.includes(format)) {
|
||||
showFallback(container, opts.fallbackImage, `No 3D preview for .${format} files.`);
|
||||
return;
|
||||
}
|
||||
|
||||
message(container, 'Loading preview…');
|
||||
let object;
|
||||
try {
|
||||
object = await loadObject(url, format, opts);
|
||||
} catch (err) {
|
||||
console.warn('Preview failed:', err);
|
||||
if (token !== renderToken) return;
|
||||
const note = format === 'glb'
|
||||
? 'Interactive 3D isn’t available for this 3MF.'
|
||||
: 'This file couldn’t be rendered in 3D.';
|
||||
showFallback(container, opts.fallbackImage, note);
|
||||
return;
|
||||
}
|
||||
if (token !== renderToken) return; // another file was opened meanwhile
|
||||
|
||||
container.innerHTML = '';
|
||||
const width = container.clientWidth || 600;
|
||||
const height = container.clientHeight || 420;
|
||||
|
||||
|
|
@ -81,20 +230,6 @@ async function render(container, url, format, opts = {}) {
|
|||
const controls = new OrbitControls(camera, renderer.domElement);
|
||||
controls.enableDamping = true;
|
||||
|
||||
let object;
|
||||
try {
|
||||
object = await loadObject(url, format);
|
||||
} catch (err) {
|
||||
renderer.dispose();
|
||||
renderer.forceContextLoss();
|
||||
if (renderer.domElement.parentNode) renderer.domElement.parentNode.removeChild(renderer.domElement);
|
||||
const note = format === '3mf'
|
||||
? 'Interactive 3D isn’t supported for this 3MF (Bambu/MakerWorld project file).'
|
||||
: 'This file couldn’t be rendered in 3D.';
|
||||
showFallback(container, opts.fallbackImage, note);
|
||||
return;
|
||||
}
|
||||
|
||||
// Center the object and frame the camera around it.
|
||||
const box = new THREE.Box3().setFromObject(object);
|
||||
const size = box.getSize(new THREE.Vector3());
|
||||
|
|
@ -123,8 +258,8 @@ async function render(container, url, format, opts = {}) {
|
|||
// Render a model once to an offscreen canvas and return a PNG data URL.
|
||||
// Used to generate card thumbnails. Transparent background so it composites
|
||||
// over the card surface.
|
||||
async function snapshot(url, format, size = 512) {
|
||||
if (!SUPPORTED.includes(format)) return null;
|
||||
async function snapshot(url, format, size = 512, opts = {}) {
|
||||
if (!MODEL_FORMATS.includes(format)) return null;
|
||||
|
||||
const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true, preserveDrawingBuffer: true });
|
||||
renderer.setSize(size, size);
|
||||
|
|
@ -137,7 +272,7 @@ async function snapshot(url, format, size = 512) {
|
|||
const fill = new THREE.DirectionalLight(0xffffff, 0.35); fill.position.set(-1, 0.5, -1); scene.add(fill);
|
||||
|
||||
try {
|
||||
const object = await loadObject(url, format);
|
||||
const object = await loadObject(url, format, opts);
|
||||
const box = new THREE.Box3().setFromObject(object);
|
||||
const dim = box.getSize(new THREE.Vector3());
|
||||
const center = box.getCenter(new THREE.Vector3());
|
||||
|
|
@ -159,4 +294,4 @@ async function snapshot(url, format, size = 512) {
|
|||
}
|
||||
}
|
||||
|
||||
window.ModelViewer = { render, dispose, snapshot, SUPPORTED, RENDERABLE };
|
||||
window.ModelViewer = { render, dispose, snapshot, MODEL_FORMATS, DOCUMENT_FORMATS, RENDERABLE, PREVIEWABLE };
|
||||
|
|
|
|||
114
server.js
114
server.js
|
|
@ -21,6 +21,7 @@ const auth = require('./lib/auth');
|
|||
const mailer = require('./lib/mailer');
|
||||
const fingerprint = require('./lib/fingerprint');
|
||||
const duplicates = require('./lib/duplicates');
|
||||
const archives = require('./lib/archives');
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 3000;
|
||||
|
|
@ -31,6 +32,9 @@ app.use(express.json({ limit: '8mb' })); // generous for base64 thumbnail PNGs
|
|||
app.use(cookieParser());
|
||||
app.use(express.static('public'));
|
||||
app.use('/thumbnails', express.static('thumbnails'));
|
||||
// In-browser readers for STEP (OpenCascade) and Rhino 3DM, served from node_modules.
|
||||
app.use('/vendor/occt', express.static(path.join(__dirname, 'node_modules/occt-import-js/dist')));
|
||||
app.use('/vendor/rhino3dm', express.static(path.join(__dirname, 'node_modules/rhino3dm')));
|
||||
|
||||
// Gate the API: everything under /api requires auth except the bootstrap config
|
||||
// and the auth endpoints themselves.
|
||||
|
|
@ -57,11 +61,14 @@ function describeFile(filePath, fileName, originalName) {
|
|||
return { fileName, filePath, originalName, format, size, ...meta };
|
||||
}
|
||||
|
||||
// Choose a thumbnail for a model from its files (embedded 3MF preview, if any).
|
||||
// Choose a thumbnail for a model from its files: an embedded 3MF preview, else
|
||||
// a picture that came with the model.
|
||||
function pickThumbnail(files) {
|
||||
const threeMf = files.find((f) => f.format === '3mf');
|
||||
if (!threeMf) return null;
|
||||
return fileProcessor.extract3mfThumbnail(threeMf.filePath, path.parse(threeMf.fileName).name);
|
||||
const embedded = threeMf && fileProcessor.extract3mfThumbnail(threeMf.filePath, path.parse(threeMf.fileName).name);
|
||||
if (embedded) return embedded;
|
||||
const image = files.find((f) => ['png', 'jpg', 'jpeg', 'gif', 'webp'].includes(f.format));
|
||||
return image ? fileProcessor.copyAsThumbnail(image.filePath, path.parse(image.fileName).name) : null;
|
||||
}
|
||||
|
||||
// Transform a stored model into the public shape (thumbnail path -> URL).
|
||||
|
|
@ -82,7 +89,7 @@ const upload = multer({
|
|||
storage,
|
||||
fileFilter: (req, file, cb) => {
|
||||
if (fileProcessor.isAllowed(file.originalname)) cb(null, true);
|
||||
else cb(new Error(`Unsupported file type. Allowed: ${fileProcessor.MODEL_FORMATS.join(', ')}`));
|
||||
else cb(new Error(`Unsupported file type. Allowed: ${fileProcessor.ALLOWED_FORMATS.join(', ')}`));
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -474,6 +481,7 @@ app.delete('/api/trash', async (req, res, next) => {
|
|||
async function purgeModel(id) {
|
||||
const removed = await store.deleteModel(id);
|
||||
if (!removed) return;
|
||||
removed.fileIds.forEach(archives.removeExtracted);
|
||||
for (const p of [...removed.filePaths, ...removed.glbPaths, removed.thumbnailPath]) {
|
||||
if (!p) continue;
|
||||
try { fs.unlinkSync(p); } catch (err) {
|
||||
|
|
@ -534,46 +542,95 @@ app.get('/api/models/:id/download', async (req, res, next) => {
|
|||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
// Serve a raw model file (used by the in-browser 3D viewer).
|
||||
// Send a stored file as a download, or with ?inline=1 for display in the page
|
||||
// (images, PDFs, text). SVGs are sandboxed so scripts inside them can't run.
|
||||
function sendStored(req, res, absPath, name) {
|
||||
res.set('X-Content-Type-Options', 'nosniff');
|
||||
const ext = path.extname(name).toLowerCase();
|
||||
if (ext === '.svg') res.set('Content-Security-Policy', "default-src 'none'; style-src 'unsafe-inline'; sandbox");
|
||||
if (req.query.inline !== '1') return res.download(absPath, name);
|
||||
if (['.md', '.mtl', '.txt'].includes(ext)) res.type('text/plain');
|
||||
res.sendFile(absPath);
|
||||
}
|
||||
|
||||
// Serve a stored file (downloads, and the in-browser viewers).
|
||||
app.get('/api/files/:id/raw', async (req, res, next) => {
|
||||
try {
|
||||
const { rows } = await db.query('SELECT file_path, original_name FROM files WHERE id = $1', [req.params.id]);
|
||||
if (rows.length === 0) return res.status(404).json({ error: 'File not found' });
|
||||
const abs = path.resolve(rows[0].file_path);
|
||||
const file = await store.getFile(req.params.id);
|
||||
if (!file) return res.status(404).json({ error: 'File not found' });
|
||||
const abs = path.resolve(file.file_path);
|
||||
if (!fs.existsSync(abs)) return res.status(404).json({ error: 'File missing on disk' });
|
||||
res.download(abs, rows[0].original_name);
|
||||
sendStored(req, res, abs, file.original_name);
|
||||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
// What's inside an archive file (indexed in the background).
|
||||
app.get('/api/files/:id/entries', async (req, res, next) => {
|
||||
try {
|
||||
const file = await store.getFile(req.params.id);
|
||||
if (!file) return res.status(404).json({ error: 'File not found' });
|
||||
if (!fileProcessor.ARCHIVE_FORMATS.includes(file.format)) return res.status(400).json({ error: 'Not an archive' });
|
||||
res.json({
|
||||
status: file.archive_status || 'pending',
|
||||
error: file.archive_error,
|
||||
entries: await store.getArchiveEntries(file.id),
|
||||
});
|
||||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
// Serve a file from inside an archive (extracted on first use, then cached).
|
||||
app.get('/api/entries/:id/raw', async (req, res, next) => {
|
||||
try {
|
||||
const entry = await store.getArchiveEntry(req.params.id);
|
||||
if (!entry) return res.status(404).json({ error: 'Entry not found' });
|
||||
const abs = await archives.extractEntry(entry.archive, entry);
|
||||
sendStored(req, res, abs, path.posix.basename(entry.path));
|
||||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
// Serve a 3MF as GLB, converting on first request (cached thereafter). This lets
|
||||
// the browser render Bambu/MakerWorld 3MFs and backfills volume/bbox.
|
||||
const glbConversions = new Map(); // fileId -> in-flight Promise
|
||||
const glbConversions = new Map(); // output path -> in-flight Promise
|
||||
|
||||
// Converts into a temporary file and renames it into place, so a half-written
|
||||
// GLB is never served.
|
||||
function convertOnce(inputPath, outPath) {
|
||||
if (!glbConversions.has(outPath)) {
|
||||
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
||||
const tmp = `${outPath}.part`;
|
||||
glbConversions.set(outPath, converter.convert3mfToGlb(inputPath, tmp)
|
||||
.then((result) => { fs.renameSync(tmp, outPath); return result; })
|
||||
.finally(() => { fs.rmSync(tmp, { force: true }); glbConversions.delete(outPath); }));
|
||||
}
|
||||
return glbConversions.get(outPath);
|
||||
}
|
||||
|
||||
app.get('/api/files/:id/glb', async (req, res, next) => {
|
||||
try {
|
||||
const { rows } = await db.query('SELECT id, file_path, format, glb_path FROM files WHERE id = $1', [req.params.id]);
|
||||
if (rows.length === 0) return res.status(404).json({ error: 'File not found' });
|
||||
const file = rows[0];
|
||||
const file = await store.getFile(req.params.id);
|
||||
if (!file) return res.status(404).json({ error: 'File not found' });
|
||||
if (file.format !== '3mf') return res.status(415).json({ error: 'GLB conversion is only for 3MF files' });
|
||||
|
||||
let glbPath = file.glb_path;
|
||||
if (!glbPath || !fs.existsSync(glbPath)) {
|
||||
if (!glbConversions.has(file.id)) {
|
||||
glbConversions.set(file.id, convertFileToGlb(file).finally(() => glbConversions.delete(file.id)));
|
||||
}
|
||||
glbPath = await glbConversions.get(file.id);
|
||||
glbPath = path.join('./glb', `${file.id}.glb`);
|
||||
const result = await convertOnce(path.resolve(file.file_path), path.resolve(glbPath));
|
||||
await store.updateFileConversion(file.id, { glb_path: glbPath, volume: result.volume_cm3, bbox: result.bbox });
|
||||
}
|
||||
res.sendFile(path.resolve(glbPath));
|
||||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
async function convertFileToGlb(file) {
|
||||
if (!fs.existsSync('./glb')) fs.mkdirSync('./glb', { recursive: true });
|
||||
const out = path.join('./glb', `${file.id}.glb`);
|
||||
const result = await converter.convert3mfToGlb(path.resolve(file.file_path), path.resolve(out));
|
||||
await store.updateFileConversion(file.id, { glb_path: out, volume: result.volume_cm3, bbox: result.bbox });
|
||||
return out;
|
||||
}
|
||||
app.get('/api/entries/:id/glb', async (req, res, next) => {
|
||||
try {
|
||||
const entry = await store.getArchiveEntry(req.params.id);
|
||||
if (!entry) return res.status(404).json({ error: 'Entry not found' });
|
||||
if (entry.format !== '3mf') return res.status(415).json({ error: 'GLB conversion is only for 3MF files' });
|
||||
const glbPath = path.resolve('./glb', `entry-${entry.id}.glb`);
|
||||
if (!fs.existsSync(glbPath)) await convertOnce(await archives.extractEntry(entry.archive, entry), glbPath);
|
||||
res.sendFile(glbPath);
|
||||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------------- import
|
||||
// Unified import. Thingiverse downloads files automatically (needs a token);
|
||||
|
|
@ -874,7 +931,8 @@ app.get('/api/library/stats', async (req, res, next) => {
|
|||
app.get('/api/duplicates', async (req, res, next) => {
|
||||
try {
|
||||
const [groups, pending] = await Promise.all([findDuplicateGroups(), fingerprint.pendingCount()]);
|
||||
const found = await store.getModelsByIds(groups.flatMap((g) => g.modelIds));
|
||||
const ids = groups.flatMap((g) => g.modelIds);
|
||||
const [found, counts] = await Promise.all([store.getModelsByIds(ids), store.matchableCounts(ids)]);
|
||||
const byId = new Map(found.map((m) => [m.id, m]));
|
||||
const result = groups
|
||||
.map((g) => ({ ...g, models: g.modelIds.map((id) => byId.get(id)).filter(Boolean) }))
|
||||
|
|
@ -883,7 +941,11 @@ app.get('/api/duplicates', async (req, res, next) => {
|
|||
signature: duplicates.signature(g.modelIds),
|
||||
exact: g.exact,
|
||||
recommendedId: duplicates.recommendKeep(g.models),
|
||||
models: g.models.map((m) => ({ ...publicModel(m), matchedFiles: g.matchedFiles[m.id] })),
|
||||
models: g.models.map((m) => ({
|
||||
...publicModel(m),
|
||||
matchedFiles: g.matchedFiles[m.id],
|
||||
matchableFiles: counts[m.id] ?? m.files.length,
|
||||
})),
|
||||
}));
|
||||
res.json({ pending, groups: result });
|
||||
} catch (e) { next(e); }
|
||||
|
|
|
|||
47
test/archives.test.js
Normal file
47
test/archives.test.js
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const crypto = require('crypto');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
|
||||
process.env.EXTRACT_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'printhive-extract-'));
|
||||
const archives = require('../lib/archives');
|
||||
|
||||
const fixture = (name) => path.join(__dirname, 'fixtures', name);
|
||||
const sha = (p) => crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex');
|
||||
const clean = (p) => p.replace(/^\.\//, '');
|
||||
|
||||
// Each pack holds models/cube.stl, models/inner.3mf and readme.txt, plus macOS
|
||||
// junk (._ files, __MACOSX, .DS_Store); the zip also has a ../ path.
|
||||
const EXPECTED = ['models/cube.stl', 'models/inner.3mf', 'readme.txt'];
|
||||
|
||||
test.after(() => fs.rmSync(process.env.EXTRACT_DIR, { recursive: true, force: true }));
|
||||
|
||||
for (const [name, format] of [['pack.zip', 'zip'], ['pack.7z', '7z'], ['pack.tar.gz', 'tgz']]) {
|
||||
const file = { id: `test-${format}`, file_path: fixture(name), format };
|
||||
|
||||
test(`${name}: lists real files, skipping folders, junk and unsafe paths`, async () => {
|
||||
const entries = await archives.listEntries(file);
|
||||
assert.deepEqual(entries.map((e) => clean(e.path)).sort(), EXPECTED);
|
||||
const cube = entries.find((e) => clean(e.path) === 'models/cube.stl');
|
||||
assert.equal(cube.size, fs.statSync(fixture('cube.stl')).size);
|
||||
});
|
||||
|
||||
test(`${name}: extracts one entry byte-for-byte, and caches it`, async () => {
|
||||
const entries = await archives.listEntries(file);
|
||||
const cube = { ...entries.find((e) => clean(e.path) === 'models/cube.stl'), id: 1 };
|
||||
const abs = await archives.extractEntry(file, cube);
|
||||
assert.equal(sha(abs), sha(fixture('cube.stl')));
|
||||
assert.equal(await archives.extractEntry(file, cube), abs);
|
||||
});
|
||||
}
|
||||
|
||||
test('a missing entry fails clearly instead of returning a path', async () => {
|
||||
const file = { id: 'test-missing', file_path: fixture('pack.zip'), format: 'zip' };
|
||||
await assert.rejects(archives.extractEntry(file, { id: 2, path: 'models/nope.stl' }), /Could not extract/);
|
||||
});
|
||||
|
||||
test('an unreadable archive rejects', async () => {
|
||||
await assert.rejects(archives.listEntries({ file_path: fixture('cube.stl'), format: 'zip' }));
|
||||
});
|
||||
BIN
test/fixtures/pack.7z
vendored
Normal file
BIN
test/fixtures/pack.7z
vendored
Normal file
Binary file not shown.
BIN
test/fixtures/pack.tar.gz
vendored
Normal file
BIN
test/fixtures/pack.tar.gz
vendored
Normal file
Binary file not shown.
BIN
test/fixtures/pack.zip
vendored
Normal file
BIN
test/fixtures/pack.zip
vendored
Normal file
Binary file not shown.
Loading…
Reference in a new issue