Compare commits
3 commits
66b62cbb1e
...
18f15cf3b4
| Author | SHA1 | Date | |
|---|---|---|---|
|
18f15cf3b4 | ||
|
|
4c54f76ae3 | ||
|
|
0df38c83c7 |
10 changed files with 789 additions and 107 deletions
|
|
@ -23,6 +23,7 @@ A self-hosted web app for organizing your 3D-printing files (STL, 3MF, OBJ, FBX,
|
|||
- **Notes** — free-form notes per model, saved as you type.
|
||||
- **Search, filters & sorting** — search across names, descriptions, notes, tags, file names and designers; filter by format, source and license; include *or exclude* tags (match any/all); sort and group by source or format. Press <kbd>/</kbd> to search.
|
||||
- **Duplicate finder** — every file gets a SHA-256 content hash plus (for STL/OBJ/3MF) a geometry fingerprint, computed in the background. Models sharing files are grouped as *exact copies* or *same shape* (renamed, re-exported, converted — or mirrored, so check those), with a recommended copy to keep. Keeping one merges the others' tags, collections and notes into it and moves them to the trash; groups can be dismissed as "not duplicates".
|
||||
- **Watch a MakerWorld collection** — paste a collection link (private ones work, using your MakerWorld token) and Printhive checks it on a schedule you choose, importing models added to it into a collection of the same name here, optionally AI-tagging the new ones. Each check reads the whole collection (paging through MakerWorld's own load-more call, up to ~640 models) and imports up to 15 of the new ones, spaced out, returning shortly after if more are waiting; models you remove on MakerWorld stay in your library. Each watch shows when it last ran, what it imported, and why a check failed — with a one-click token update when MakerWorld refuses your token.
|
||||
- **AI tags & descriptions** — bring your own key for DeepSeek (the default), OpenRouter, OpenAI or any OpenAI-compatible server (Ollama, LM Studio). Tag the whole library, only models not yet tagged, or a single model from its page. Each model's name, file names and (optionally) thumbnail are sent; tags are only ever added and descriptions only filled where empty unless you ask. Optionally it also gives models with unclear names (file-name leftovers, typos, site boilerplate) a clearer one; the old name is kept and can be restored with Undo on the model page. It can also suggest collections (reusing yours where they fit): suggestions wait on the AI page, grouped by name, until you create/add or dismiss them. Each run can do any mix of tags & descriptions, names and collections, and a reply that isn't the JSON asked for is retried once. Runs show live progress, tokens and cost, can be stopped and resumed, stop at an optional spending limit, and pause (rather than keep spending) if the server restarts. "Save & test connection" in Settings checks the key, model and whether it accepts images.
|
||||
- **Search by meaning** — switch the search box to *Meaning* to find models by what they are ("wall hook" finds a "coat hanger bracket"). Uses a small embedding model (bge-small-en-v1.5, ~34 MB) that runs on the server, so it's free and private; the index updates in the background as models change.
|
||||
- **Automatic tags** — strict whole-word rules over names, file names and archive contents (part types, mechanisms, materials, techniques) plus your own "word → tag" rules. Switch it off (optionally removing its tags) or re-apply it any time. Tags you add yourself are never changed; AI and rule tags are marked on the model page.
|
||||
|
|
|
|||
|
|
@ -225,3 +225,38 @@ CREATE TABLE IF NOT EXISTS collection_suggestions (
|
|||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_collection_suggestions_model_name ON collection_suggestions(model_id, lower(name));
|
||||
ALTER TABLE ai_run_items ADD COLUMN IF NOT EXISTS collections TEXT[];
|
||||
|
||||
-- MakerWorld collections watched for new models (lib/collectionWatch.js).
|
||||
CREATE TABLE IF NOT EXISTS watched_collections (
|
||||
id SERIAL PRIMARY KEY,
|
||||
site TEXT NOT NULL DEFAULT 'makerworld',
|
||||
source_id TEXT NOT NULL, -- the collection's id on the site
|
||||
url TEXT NOT NULL,
|
||||
title TEXT NOT NULL DEFAULT '', -- its name there, refreshed on each check
|
||||
project_id INTEGER REFERENCES projects(id) ON DELETE SET NULL, -- where new models land
|
||||
ai_tag BOOLEAN NOT NULL DEFAULT false,
|
||||
interval_hours INTEGER NOT NULL DEFAULT 6,
|
||||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||||
last_checked_at TIMESTAMPTZ,
|
||||
next_check_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
last_status TEXT, -- ok | error
|
||||
last_error TEXT,
|
||||
last_error_reason TEXT, -- makerworld_token: offer the token prompt
|
||||
last_added INTEGER NOT NULL DEFAULT 0,
|
||||
listed INTEGER, -- models the page listed at the last check
|
||||
total INTEGER, -- models the site says the collection holds
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (site, source_id)
|
||||
);
|
||||
|
||||
-- What a watch has already seen, so a model isn't imported (or retried) twice.
|
||||
CREATE TABLE IF NOT EXISTS watched_collection_items (
|
||||
watch_id INTEGER NOT NULL REFERENCES watched_collections(id) ON DELETE CASCADE,
|
||||
design_id TEXT NOT NULL,
|
||||
model_id INTEGER REFERENCES models(id) ON DELETE SET NULL,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL, -- imported | existing | error
|
||||
error TEXT,
|
||||
seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (watch_id, design_id)
|
||||
);
|
||||
|
|
|
|||
235
lib/collectionWatch.js
Normal file
235
lib/collectionWatch.js
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
// Watches a MakerWorld collection and imports models added to it. A check reads
|
||||
// the collection page (with your token, so private collections work), imports the
|
||||
// models it hasn't seen before, and remembers each one so it isn't imported twice.
|
||||
// The site is read gently: a few models per check, spaced out.
|
||||
const db = require('./db');
|
||||
const store = require('./store');
|
||||
const makerworld = require('./importers/makerworld');
|
||||
|
||||
const IMPORT_LIMIT = 15; // models per check, so a big collection trickles in
|
||||
const BETWEEN_IMPORTS_MS = 3000; // FlareSolverr runs one browser request at a time
|
||||
const MORE_SOON_MINUTES = 10; // when a check hit the limit, come back sooner
|
||||
const TICK_MS = 5 * 60 * 1000;
|
||||
const INTERVALS = [1, 3, 6, 12, 24, 48];
|
||||
|
||||
// { fetchCollection(url), importFromUrl(url, opts), startAiRun(modelIds) }
|
||||
let deps = null;
|
||||
let timer = null;
|
||||
let busy = false;
|
||||
const checking = new Set(); // watch ids being checked right now, so checks can't overlap
|
||||
|
||||
const httpError = (message, status, reason) => Object.assign(new Error(message), { status, reason });
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function init(dependencies) {
|
||||
deps = dependencies;
|
||||
}
|
||||
|
||||
// Checks anything due, every few minutes.
|
||||
function start() {
|
||||
// A check that was running when the server stopped isn't running now.
|
||||
db.query(`UPDATE watched_collections SET last_status = NULL WHERE last_status = 'checking'`)
|
||||
.catch((err) => console.warn('Could not clear stale watch checks:', err.message));
|
||||
clearInterval(timer);
|
||||
timer = setInterval(() => { runDue().catch((err) => console.error('Collection watch failed:', err.message)); }, TICK_MS);
|
||||
if (timer.unref) timer.unref();
|
||||
runDue().catch((err) => console.error('Collection watch failed:', err.message));
|
||||
}
|
||||
|
||||
async function list() {
|
||||
const { rows } = await db.query(`
|
||||
SELECT w.*, p.name AS project_name,
|
||||
(SELECT COUNT(*)::int FROM watched_collection_items i WHERE i.watch_id = w.id AND i.status = 'imported') AS imported_count,
|
||||
(SELECT COUNT(*)::int FROM watched_collection_items i WHERE i.watch_id = w.id AND i.status = 'error') AS error_count,
|
||||
COALESCE(r.recent, '[]'::json) AS recent
|
||||
FROM watched_collections w
|
||||
LEFT JOIN projects p ON p.id = w.project_id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT json_agg(json_build_object('design_id', i.design_id, 'model_id', i.model_id, 'title', i.title,
|
||||
'status', i.status, 'error', i.error, 'seen_at', i.seen_at)
|
||||
ORDER BY i.seen_at DESC) AS recent
|
||||
FROM (SELECT * FROM watched_collection_items WHERE watch_id = w.id ORDER BY seen_at DESC LIMIT 5) i
|
||||
) r ON true
|
||||
ORDER BY w.id`);
|
||||
return rows;
|
||||
}
|
||||
|
||||
async function get(id) {
|
||||
const { rows: [watch] } = await db.query(`
|
||||
SELECT w.*, p.name AS project_name,
|
||||
(SELECT COUNT(*)::int FROM watched_collection_items i WHERE i.watch_id = w.id AND i.status = 'imported') AS imported_count,
|
||||
(SELECT COUNT(*)::int FROM watched_collection_items i WHERE i.watch_id = w.id AND i.status = 'error') AS error_count
|
||||
FROM watched_collections w LEFT JOIN projects p ON p.id = w.project_id WHERE w.id = $1`, [id]);
|
||||
if (!watch) return null;
|
||||
const { rows: recent } = await db.query(`
|
||||
SELECT i.design_id, i.model_id, i.title, i.status, i.error, i.seen_at, m.name AS model_name
|
||||
FROM watched_collection_items i LEFT JOIN models m ON m.id = i.model_id
|
||||
WHERE i.watch_id = $1 ORDER BY i.seen_at DESC LIMIT 12`, [id]);
|
||||
return { ...watch, recent };
|
||||
}
|
||||
|
||||
// Adds a watch. Reads the collection first, so a bad link or a refused token is
|
||||
// reported now rather than silently later.
|
||||
async function add({ url, aiTag = false, intervalHours = 6 }) {
|
||||
const sourceId = makerworld.collectionIdFromUrl(url);
|
||||
if (!sourceId) throw httpError('That isn’t a MakerWorld collection link (makerworld.com/…/collections/…).', 400);
|
||||
const { rowCount } = await db.query('SELECT 1 FROM watched_collections WHERE site = $1 AND source_id = $2', ['makerworld', sourceId]);
|
||||
if (rowCount) throw httpError('You’re already watching that collection.', 409);
|
||||
|
||||
const collection = await deps.fetchCollection(url);
|
||||
const project = await store.findOrCreateProject(collection.title || `MakerWorld collection ${sourceId}`);
|
||||
const hours = INTERVALS.includes(Number(intervalHours)) ? Number(intervalHours) : 6;
|
||||
const { rows: [watch] } = await db.query(
|
||||
`INSERT INTO watched_collections (site, source_id, url, title, project_id, ai_tag, interval_hours, total, listed, last_status)
|
||||
VALUES ('makerworld', $1, $2, $3, $4, $5, $6, $7, $8, 'checking') RETURNING id`,
|
||||
[sourceId, url, collection.title, project.id, Boolean(aiTag), hours, collection.total, collection.designs.length]);
|
||||
checkNow(watch.id).catch((err) => console.warn(`Watch ${watch.id} first check failed:`, err.message));
|
||||
return get(watch.id);
|
||||
}
|
||||
|
||||
async function update(id, { enabled, aiTag, intervalHours }) {
|
||||
const hours = INTERVALS.includes(Number(intervalHours)) ? Number(intervalHours) : null;
|
||||
const { rowCount } = await db.query(
|
||||
`UPDATE watched_collections SET
|
||||
enabled = COALESCE($2, enabled),
|
||||
ai_tag = COALESCE($3, ai_tag),
|
||||
interval_hours = COALESCE($4, interval_hours),
|
||||
next_check_at = CASE WHEN $2 IS TRUE THEN now() ELSE next_check_at END
|
||||
WHERE id = $1`,
|
||||
[id, enabled === undefined ? null : Boolean(enabled), aiTag === undefined ? null : Boolean(aiTag), hours]);
|
||||
if (!rowCount) return null;
|
||||
if (enabled) runDue().catch(() => {});
|
||||
return get(id);
|
||||
}
|
||||
|
||||
// Stops watching. Models already imported stay in the library.
|
||||
async function remove(id) {
|
||||
const { rowCount } = await db.query('DELETE FROM watched_collections WHERE id = $1', [id]);
|
||||
return rowCount > 0;
|
||||
}
|
||||
|
||||
// Checks one collection now: reads it (awaited, so problems come back to the
|
||||
// caller), then imports in the background.
|
||||
async function checkNow(id) {
|
||||
const watch = await get(id);
|
||||
if (!watch) throw httpError('Not watching that collection.', 404);
|
||||
if (checking.has(id)) return watch; // already going; don't import anything twice
|
||||
let collection;
|
||||
try {
|
||||
collection = await deps.fetchCollection(watch.url);
|
||||
} catch (err) {
|
||||
await recordFailure(watch, err);
|
||||
throw err;
|
||||
}
|
||||
await db.query(
|
||||
`UPDATE watched_collections SET title = $2, total = $3, listed = $4, last_status = 'checking',
|
||||
last_error = NULL, last_error_reason = NULL WHERE id = $1`,
|
||||
[id, collection.title || watch.title, collection.total, collection.designs.length]);
|
||||
checking.add(id);
|
||||
importFound(watch, collection)
|
||||
.catch((err) => console.error(`Watch ${id} import failed:`, err.message))
|
||||
.finally(() => checking.delete(id));
|
||||
return get(id);
|
||||
}
|
||||
|
||||
async function recordFailure(watch, err) {
|
||||
await db.query(
|
||||
`UPDATE watched_collections SET last_checked_at = now(), last_status = 'error', last_error = $2,
|
||||
last_error_reason = $3, next_check_at = now() + make_interval(hours => GREATEST(interval_hours, 1))
|
||||
WHERE id = $1`,
|
||||
[watch.id, String(err.message).slice(0, 300), err.reason || null]);
|
||||
}
|
||||
|
||||
// Imports the models this watch hasn't seen yet, a few at a time.
|
||||
async function importFound(watch, collection) {
|
||||
const { rows: seenRows } = await db.query(
|
||||
`SELECT design_id FROM watched_collection_items WHERE watch_id = $1 AND status <> 'error'`, [watch.id]);
|
||||
const seen = new Set(seenRows.map((r) => r.design_id));
|
||||
const pending = collection.designs.filter((d) => !seen.has(d.id));
|
||||
const todo = pending.slice(0, IMPORT_LIMIT);
|
||||
|
||||
let added = 0;
|
||||
let problem = null;
|
||||
const fresh = [];
|
||||
for (const design of todo) {
|
||||
const url = makerworld.designUrl(design);
|
||||
try {
|
||||
const result = await deps.importFromUrl(url, { projects: watch.project_id ? [watch.project_id] : [] });
|
||||
if (result.existing) {
|
||||
if (watch.project_id) await store.addModelToProject(result.model.id, watch.project_id);
|
||||
await recordItem(watch.id, design, result.model.id, 'existing', null);
|
||||
} else {
|
||||
added += 1;
|
||||
fresh.push(result.model.id);
|
||||
await recordItem(watch.id, design, result.model.id, 'imported', result.filesError || null);
|
||||
}
|
||||
// A refused token means every later download fails too: stop and say so.
|
||||
if (result.filesErrorReason === 'makerworld_token') {
|
||||
problem = httpError(result.filesError, 502, 'makerworld_token');
|
||||
break;
|
||||
}
|
||||
} catch (err) {
|
||||
await recordItem(watch.id, design, null, 'error', err.message);
|
||||
if (err.reason === 'makerworld_token') { problem = err; break; }
|
||||
}
|
||||
await sleep(BETWEEN_IMPORTS_MS);
|
||||
}
|
||||
|
||||
const more = pending.length > todo.length;
|
||||
if (problem) {
|
||||
await recordFailure({ ...watch, id: watch.id }, problem);
|
||||
} else {
|
||||
await db.query(
|
||||
`UPDATE watched_collections SET last_checked_at = now(), last_status = 'ok', last_error = NULL,
|
||||
last_error_reason = NULL, last_added = $2,
|
||||
next_check_at = now() + CASE WHEN $3::boolean THEN make_interval(mins => $4) ELSE make_interval(hours => interval_hours) END
|
||||
WHERE id = $1`,
|
||||
[watch.id, added, more, MORE_SOON_MINUTES]);
|
||||
}
|
||||
|
||||
if (watch.ai_tag && fresh.length && deps.startAiRun) {
|
||||
await deps.startAiRun(fresh).catch((err) => console.warn(`Watch ${watch.id}: AI tagging didn't start:`, err.message));
|
||||
}
|
||||
}
|
||||
|
||||
async function recordItem(watchId, design, modelId, status, error) {
|
||||
await db.query(
|
||||
`INSERT INTO watched_collection_items (watch_id, design_id, model_id, title, status, error, seen_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, now())
|
||||
ON CONFLICT (watch_id, design_id) DO UPDATE SET model_id = EXCLUDED.model_id, title = EXCLUDED.title,
|
||||
status = EXCLUDED.status, error = EXCLUDED.error, seen_at = now()`,
|
||||
[watchId, design.id, modelId, String(design.title || '').slice(0, 200), status, error ? String(error).slice(0, 300) : null]);
|
||||
}
|
||||
|
||||
// Checks every watch that's due, one at a time.
|
||||
async function runDue() {
|
||||
if (busy || !deps) return;
|
||||
busy = true;
|
||||
try {
|
||||
for (;;) {
|
||||
const { rows: [due] } = await db.query(
|
||||
`SELECT id FROM watched_collections WHERE enabled AND next_check_at <= now()
|
||||
AND id <> ALL($1::int[]) ORDER BY next_check_at LIMIT 1`, [[...checking]]);
|
||||
if (!due) return;
|
||||
// checkNow leaves the import running in the background; wait for it here so
|
||||
// watches are checked one after another.
|
||||
const watch = await get(due.id);
|
||||
checking.add(watch.id);
|
||||
try {
|
||||
const collection = await deps.fetchCollection(watch.url);
|
||||
await db.query(
|
||||
`UPDATE watched_collections SET title = $2, total = $3, listed = $4, last_status = 'checking' WHERE id = $1`,
|
||||
[watch.id, collection.title || watch.title, collection.total, collection.designs.length]);
|
||||
await importFound(watch, collection);
|
||||
} catch (err) {
|
||||
await recordFailure(watch, err);
|
||||
} finally {
|
||||
checking.delete(watch.id);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
busy = false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { INTERVALS, init, start, list, get, add, update, remove, checkNow, runDue };
|
||||
|
|
@ -46,10 +46,64 @@ function downloadProblem(answer) {
|
|||
return { message: `MakerWorld didn’t give a download link${detail ? `: ${detail}` : ''}`, tokenRejected: false };
|
||||
}
|
||||
|
||||
// The collection id in a MakerWorld collection link, or null.
|
||||
function collectionIdFromUrl(url) {
|
||||
const m = String(url || '').match(/makerworld\.com\/(?:[a-z-]{2,5}\/)?collections\/(\d+)/i);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
// Where a collection page loads the rest of its models from (its own "load more"
|
||||
// call). A stable `seed` keeps the pages from overlapping.
|
||||
const collectionDesignsUrl = (collectionId, { offset = 0, limit = 64, seed } = {}) =>
|
||||
`https://makerworld.com/api/v1/design-service/favorites/${collectionId}/designs?offset=${offset}&limit=${limit}`
|
||||
+ (seed === undefined || seed === null ? '' : `&seed=${seed}`);
|
||||
|
||||
// That call's answer -> { designs, total, seed }.
|
||||
function parseCollectionDesigns(json) {
|
||||
const hits = (json && json.hits) || [];
|
||||
return {
|
||||
designs: hits
|
||||
.filter((d) => d && (typeof d.id === 'number' || typeof d.id === 'string'))
|
||||
.map((d) => ({ id: String(d.id), slug: d.slug || '', title: d.title || '' })),
|
||||
total: Number(json && json.total) || 0,
|
||||
seed: json ? json.seed : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
const designUrl = (design) => `https://makerworld.com/en/models/${design.id}${design.slug ? `-${design.slug}` : ''}`;
|
||||
|
||||
// A collection page -> { id, title, total, designs: [{ id, slug, title }] }, or null
|
||||
// when the page holds no collection (a private one read without a signed-in token).
|
||||
// `total` is how many models the collection holds; the page may list fewer.
|
||||
function parseCollection(html) {
|
||||
const m = String(html || '').match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/);
|
||||
if (!m) return null;
|
||||
let props;
|
||||
try { props = JSON.parse(m[1]).props.pageProps; } catch { return null; }
|
||||
// A collection you can't see (private, read without a signed-in token) comes back
|
||||
// as a placeholder: id -1, no title, no models.
|
||||
const id = Number(props && props.favorite && props.favorite.id);
|
||||
if (!Number.isFinite(id) || id <= 0) return null;
|
||||
const hits = (props.favoriteDesigns && props.favoriteDesigns.hits) || [];
|
||||
const designs = hits
|
||||
.filter((d) => d && (typeof d.id === 'number' || typeof d.id === 'string'))
|
||||
.map((d) => ({ id: String(d.id), slug: d.slug || '', title: d.title || '' }));
|
||||
const total = Number((props.favoriteDesigns && props.favoriteDesigns.total) ?? props.favorite.designCnt);
|
||||
return {
|
||||
id: String(id),
|
||||
title: props.favorite.title || '',
|
||||
total: Number.isFinite(total) ? total : designs.length,
|
||||
designs,
|
||||
};
|
||||
}
|
||||
|
||||
// Pull the license string out of the page data (e.g. "MakerWorld Exclusive License").
|
||||
function parseLicense(html) {
|
||||
const m = html.match(/"license"\s*:\s*"([^"]+)"/);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
module.exports = { parseInstances, resolveDownload, downloadProblem, parseLicense };
|
||||
module.exports = {
|
||||
parseInstances, resolveDownload, downloadProblem, parseLicense,
|
||||
parseCollection, parseCollectionDesigns, collectionIdFromUrl, collectionDesignsUrl, designUrl,
|
||||
};
|
||||
|
|
|
|||
16
lib/store.js
16
lib/store.js
|
|
@ -503,6 +503,20 @@ async function createProject({ name, description }) {
|
|||
return rows[0];
|
||||
}
|
||||
|
||||
// The collection with this name, created if there isn't one yet.
|
||||
async function findOrCreateProject(name) {
|
||||
const { rows: [existing] } = await db.query(
|
||||
'SELECT * FROM projects WHERE lower(name) = lower($1) ORDER BY id LIMIT 1', [name]);
|
||||
if (existing) return existing;
|
||||
const { rows: [created] } = await db.query('INSERT INTO projects (name) VALUES ($1) RETURNING *', [name]);
|
||||
return created;
|
||||
}
|
||||
|
||||
async function addModelToProject(modelId, projectId) {
|
||||
await db.query('INSERT INTO model_projects (model_id, project_id) VALUES ($1, $2) ON CONFLICT DO NOTHING',
|
||||
[modelId, projectId]);
|
||||
}
|
||||
|
||||
async function updateProject(id, { name, description }) {
|
||||
const { rows } = await db.query(
|
||||
`UPDATE projects SET name = COALESCE($2, name),
|
||||
|
|
@ -636,7 +650,7 @@ module.exports = {
|
|||
getFile, getArchiveEntries, getArchiveEntry, replaceArchiveEntries,
|
||||
getAllTags, replaceRuleTags, removeTagsFrom, applyAiResult, topTags,
|
||||
suggestCollections, collectionNameHints, listCollectionSuggestions, acceptCollectionSuggestion, dismissCollectionSuggestion,
|
||||
getAllProjects, getProject, createProject, updateProject, deleteProject,
|
||||
getAllProjects, getProject, createProject, updateProject, deleteProject, findOrCreateProject, addModelToProject,
|
||||
getSetting, setSetting,
|
||||
countUsers, getUserByEmail, getUserById, createUser, updateUserPassword,
|
||||
createPasswordReset, getValidReset, markResetUsed,
|
||||
|
|
|
|||
217
public/app.js
217
public/app.js
|
|
@ -12,7 +12,7 @@ let duplicatePoll = null;
|
|||
// View preferences, remembered per browser.
|
||||
const prefs = { view: 'grid', tagMatch: 'any', format: '', source: '', license: '', sort: 'newest', group: '', slicer: '', slicerTemplate: '', searchMode: 'words' };
|
||||
const PREFS_KEY = 'printhive.prefs';
|
||||
const SPECIAL_SCOPES = ['all', 'duplicates', 'trash', 'ai'];
|
||||
const SPECIAL_SCOPES = ['all', 'duplicates', 'trash', 'ai', 'watch'];
|
||||
|
||||
// DOM
|
||||
const fileGrid = document.getElementById('fileGrid');
|
||||
|
|
@ -332,13 +332,14 @@ function scopeTitle() {
|
|||
if (selectedScope === 'duplicates') return 'Duplicates';
|
||||
if (selectedScope === 'trash') return 'Trash';
|
||||
if (selectedScope === 'ai') return 'AI & automatic tags';
|
||||
if (selectedScope === 'watch') return 'Watched collections';
|
||||
const p = projects.find((x) => String(x.id) === String(selectedScope));
|
||||
return p ? p.name : 'Collection';
|
||||
}
|
||||
|
||||
function updateScopeChrome() {
|
||||
contentTitle.textContent = scopeTitle();
|
||||
const panel = selectedScope === 'duplicates' || selectedScope === 'ai'; // pages without the model grid
|
||||
const panel = ['duplicates', 'ai', 'watch'].includes(selectedScope); // pages without the model grid
|
||||
document.getElementById('filterBar').hidden = panel;
|
||||
document.querySelector('.view-options').hidden = panel;
|
||||
emptyTrashBtn.hidden = !(selectedScope === 'trash' && trashModels.length);
|
||||
|
|
@ -352,8 +353,10 @@ function renderModels() {
|
|||
updateScopeChrome();
|
||||
if (selectedScope === 'duplicates') { clearTimeout(aiPoll); renderDuplicates(); return; }
|
||||
clearTimeout(duplicatePoll);
|
||||
if (selectedScope === 'ai') { renderAiPanel(); return; }
|
||||
if (selectedScope === 'ai') { clearTimeout(watchPoll); renderAiPanel(); return; }
|
||||
clearTimeout(aiPoll);
|
||||
if (selectedScope === 'watch') { renderWatchPanel(); return; }
|
||||
clearTimeout(watchPoll);
|
||||
|
||||
const list = getFilteredModels();
|
||||
resultCount.textContent = `${list.length} model${list.length === 1 ? '' : 's'}${searchNote()}`;
|
||||
|
|
@ -473,6 +476,7 @@ function getFilteredModels() {
|
|||
function updateCounts() {
|
||||
document.getElementById('allFilesCount').textContent = models.length;
|
||||
document.getElementById('aiBusy').hidden = !stats.aiRunning;
|
||||
document.getElementById('watchCount').textContent = stats.watches ?? 0;
|
||||
document.getElementById('trashCount').textContent = stats.trash ?? 0;
|
||||
document.getElementById('duplicatesCount').textContent = stats.duplicateGroups ?? 0;
|
||||
}
|
||||
|
|
@ -783,6 +787,7 @@ function setupEventListeners() {
|
|||
fileGrid.addEventListener('submit', (e) => {
|
||||
if (e.target.id === 'aiStartForm') handleAiStart(e);
|
||||
if (e.target.id === 'ruleForm') handleAddRule(e);
|
||||
if (e.target.id === 'watchForm') handleAddWatch(e);
|
||||
});
|
||||
fileGrid.addEventListener('change', (e) => {
|
||||
if (e.target.name === 'aiScope') updateAiEstimate();
|
||||
|
|
@ -1653,6 +1658,8 @@ async function handleAiStart(e) {
|
|||
async function handleAiPanelClick(e) {
|
||||
const open = e.target.closest('[data-open-model]');
|
||||
if (open) { openModelDetail(Number(open.dataset.openModel)); return; }
|
||||
const watchBtn = e.target.closest('[data-watch]');
|
||||
if (watchBtn) { handleWatchAction(watchBtn); return; }
|
||||
const btn = e.target.closest('[data-ai]');
|
||||
if (!btn) return;
|
||||
const id = btn.dataset.id;
|
||||
|
|
@ -2060,7 +2067,7 @@ async function handleDownloadFiles() {
|
|||
const data = await res.json();
|
||||
if (!res.ok && data.reason === 'makerworld_token') {
|
||||
resetDownloadFilesBtn(id);
|
||||
promptMakerworldToken(id, data.error);
|
||||
promptMakerworldToken(data.error, () => retryModelDownload(id));
|
||||
return;
|
||||
}
|
||||
if (!res.ok) throw new Error(data.error);
|
||||
|
|
@ -2082,11 +2089,13 @@ function resetDownloadFilesBtn(id) {
|
|||
|
||||
// ---------------------------------------------------------------- MakerWorld token prompt
|
||||
// MakerWorld tokens expire. When one is refused, ask for a fresh one right away and
|
||||
// retry the download with it.
|
||||
let tokenPromptModelId = null;
|
||||
// retry what failed — a model's files, or a watched collection.
|
||||
// retry: async () => ({ ok, message } | { ok: false, reason, error })
|
||||
let tokenPromptRetry = null;
|
||||
|
||||
function promptMakerworldToken(modelId, reason) {
|
||||
tokenPromptModelId = modelId;
|
||||
function promptMakerworldToken(reason, retry, action = 'Save & download files') {
|
||||
tokenPromptRetry = retry;
|
||||
document.getElementById('tokenSubmit').textContent = action;
|
||||
document.getElementById('tokenReason').textContent = reason;
|
||||
document.getElementById('tokenInput').value = '';
|
||||
const status = document.getElementById('tokenStatus');
|
||||
|
|
@ -2101,7 +2110,7 @@ async function handleTokenSubmit(e) {
|
|||
e.preventDefault();
|
||||
const token = document.getElementById('tokenInput').value.trim();
|
||||
if (!token) return;
|
||||
const id = tokenPromptModelId;
|
||||
const retry = tokenPromptRetry;
|
||||
const status = document.getElementById('tokenStatus');
|
||||
const submit = document.getElementById('tokenSubmit');
|
||||
const fail = (text) => {
|
||||
|
|
@ -2111,7 +2120,7 @@ async function handleTokenSubmit(e) {
|
|||
};
|
||||
submit.disabled = true;
|
||||
status.classList.remove('is-error');
|
||||
status.textContent = 'Saving your token and downloading the files… this can take a minute.';
|
||||
status.textContent = 'Saving your token and trying again… this can take a minute.';
|
||||
try {
|
||||
await postJson('/api/settings', { makerworld_token: token }, 'PUT');
|
||||
} catch (err) {
|
||||
|
|
@ -2119,25 +2128,186 @@ async function handleTokenSubmit(e) {
|
|||
return;
|
||||
}
|
||||
try {
|
||||
const res = await fetch(`/api/models/${id}/download-files`, { method: 'POST' });
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
fail(data.reason === 'makerworld_token'
|
||||
const result = await retry();
|
||||
if (!result.ok) {
|
||||
fail(result.reason === 'makerworld_token'
|
||||
? 'MakerWorld still didn’t accept that token. Check you copied the whole value of the token cookie while signed in, then try again.'
|
||||
: `Your token is saved, but the download failed: ${data.error}`);
|
||||
: `Your token is saved, but it didn’t work: ${result.error}`);
|
||||
return;
|
||||
}
|
||||
status.textContent = data.added ? `Downloaded ${plural(data.added, 'file')}.` : 'Token saved. No new files were found.';
|
||||
await loadData();
|
||||
setTimeout(() => {
|
||||
closeModal(tokenModal);
|
||||
if (fileDetailModal.classList.contains('active') && fileDetailModal.dataset.currentModelId === String(id)) openModelDetail(id);
|
||||
}, 900);
|
||||
status.textContent = result.message || 'Token saved.';
|
||||
setTimeout(() => closeModal(tokenModal), 900);
|
||||
} catch (err) {
|
||||
fail(`Your token is saved, but the download failed: ${err.message}`);
|
||||
fail(`Your token is saved, but it didn’t work: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Retries a model's file download after a new token; used by the prompt.
|
||||
async function retryModelDownload(id) {
|
||||
const res = await fetch(`/api/models/${id}/download-files`, { method: 'POST' });
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) return { ok: false, reason: data.reason, error: data.error };
|
||||
await loadData();
|
||||
if (fileDetailModal.classList.contains('active') && fileDetailModal.dataset.currentModelId === String(id)) openModelDetail(id);
|
||||
return { ok: true, message: data.added ? `Downloaded ${plural(data.added, 'file')}.` : 'Token saved. No new files were found.' };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- watched collections
|
||||
let watchPoll = null;
|
||||
|
||||
function watchPanelHtml() {
|
||||
const intervals = [1, 3, 6, 12, 24, 48]
|
||||
.map((h) => `<option value="${h}"${h === 6 ? ' selected' : ''}>${h === 1 ? 'every hour' : `every ${h} hours`}</option>`).join('');
|
||||
return `
|
||||
<section class="ai-card">
|
||||
<div class="ai-card-head"><h3>Watch a MakerWorld collection</h3></div>
|
||||
<p class="form-hint">Printhive checks the collection and imports models added to it, into a collection of the same
|
||||
name here. Private collections work too, using your MakerWorld token. Models you remove there stay here.</p>
|
||||
<form id="watchForm" class="watch-form">
|
||||
<input type="url" id="watchUrl" placeholder="https://makerworld.com/en/collections/…" aria-label="Collection link" required>
|
||||
<label>Check <select id="watchInterval" aria-label="How often to check">${intervals}</select></label>
|
||||
<label><input type="checkbox" id="watchAi"> Tag new models with AI</label>
|
||||
<button type="submit" class="btn btn-primary" id="watchSubmit">Watch collection</button>
|
||||
</form>
|
||||
<div id="watchStatus" class="form-hint"></div>
|
||||
<div id="watchList" class="watch-list"></div>
|
||||
</section>`;
|
||||
}
|
||||
|
||||
const WATCH_STATUS = { ok: 'Up to date', checking: 'Checking…', error: 'Problem' };
|
||||
|
||||
function watchHtml(w) {
|
||||
const when = w.last_checked_at ? `checked ${formatDate(w.last_checked_at)}` : 'not checked yet';
|
||||
const listed = w.total != null && w.listed != null && w.listed < w.total
|
||||
? ` · MakerWorld listed ${w.listed} of ${w.total}` : w.total != null ? ` · ${plural(w.total, 'model')} there` : '';
|
||||
const recent = (w.recent || []).filter((i) => i.status !== 'existing');
|
||||
return `
|
||||
<div class="watch${w.enabled ? '' : ' is-paused'}" data-watch-id="${w.id}">
|
||||
<div class="watch-head">
|
||||
<a href="${escapeHtml(w.url)}" target="_blank" rel="noopener">${escapeHtml(w.title || 'MakerWorld collection')}</a>
|
||||
<span class="pill pill-${w.last_status === 'error' ? 'failed' : w.last_status === 'checking' ? 'running' : 'done'}">
|
||||
${w.enabled ? (WATCH_STATUS[w.last_status] || 'Waiting') : 'Paused'}</span>
|
||||
<span class="form-hint">→ ${escapeHtml(w.project_name || 'no collection')}${w.ai_tag ? ' · AI tagging on' : ''}</span>
|
||||
<span class="watch-actions">
|
||||
<button type="button" class="btn btn-secondary btn-sm" data-watch="check">Check now</button>
|
||||
<button type="button" class="btn btn-secondary btn-sm" data-watch="toggle">${w.enabled ? 'Pause' : 'Resume'}</button>
|
||||
<button type="button" class="btn btn-secondary btn-sm" data-watch="remove">Stop watching</button>
|
||||
</span>
|
||||
</div>
|
||||
<div class="watch-meta">${escapeHtml(when)} · every ${w.interval_hours}h · ${plural(w.imported_count, 'model')} imported${w.error_count ? ` · ${w.error_count} failed` : ''}${listed}</div>
|
||||
${w.last_error ? `<div class="ai-run-error">${escapeHtml(w.last_error)}
|
||||
${w.last_error_reason === 'makerworld_token' ? '<button type="button" class="btn btn-primary btn-sm" data-watch="token">Update token</button>' : ''}</div>` : ''}
|
||||
${recent.length ? `<div class="watch-recent">${recent.map((i) => `
|
||||
<span>${i.model_id
|
||||
? `<button type="button" class="link-btn" data-open-model="${i.model_id}">${escapeHtml(i.model_name || i.title || `Model ${i.model_id}`)}</button>`
|
||||
: escapeHtml(i.title || i.design_id)}${i.status === 'error' ? ` <span class="ai-run-error">${escapeHtml(i.error || 'failed')}</span>` : ''}</span>`).join('')}</div>` : ''}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
async function renderWatchPanel() {
|
||||
fileGrid.className = 'ai-panel';
|
||||
resultCount.textContent = '';
|
||||
if (!document.getElementById('watchForm')) fileGrid.innerHTML = watchPanelHtml();
|
||||
await refreshWatches();
|
||||
}
|
||||
|
||||
async function refreshWatches() {
|
||||
clearTimeout(watchPoll);
|
||||
if (selectedScope !== 'watch') return;
|
||||
let watches;
|
||||
try {
|
||||
watches = await getJson('/api/watches');
|
||||
} catch (err) {
|
||||
const list = document.getElementById('watchList');
|
||||
if (list) list.innerHTML = `<p class="form-hint">Couldn’t load watched collections: ${escapeHtml(err.message)}</p>`;
|
||||
return;
|
||||
}
|
||||
const list = document.getElementById('watchList');
|
||||
if (!list || selectedScope !== 'watch') return;
|
||||
list.innerHTML = watches.length
|
||||
? watches.map(watchHtml).join('')
|
||||
: '<p class="form-hint">Nothing watched yet. Paste a MakerWorld collection link above.</p>';
|
||||
if (watches.some((w) => w.last_status === 'checking')) watchPoll = setTimeout(refreshWatches, 5000);
|
||||
}
|
||||
|
||||
async function handleAddWatch(e) {
|
||||
e.preventDefault();
|
||||
const status = document.getElementById('watchStatus');
|
||||
const submit = document.getElementById('watchSubmit');
|
||||
const url = document.getElementById('watchUrl').value.trim();
|
||||
submit.disabled = true;
|
||||
status.classList.remove('is-error');
|
||||
status.textContent = 'Reading that collection…';
|
||||
try {
|
||||
const watch = await postJson('/api/watches', {
|
||||
url,
|
||||
aiTag: document.getElementById('watchAi').checked,
|
||||
intervalHours: Number(document.getElementById('watchInterval').value),
|
||||
});
|
||||
document.getElementById('watchUrl').value = '';
|
||||
status.textContent = `Watching “${watch.title}” — importing what's in it now.`;
|
||||
await loadData();
|
||||
refreshWatches();
|
||||
} catch (err) {
|
||||
status.classList.add('is-error');
|
||||
status.textContent = err.message;
|
||||
if (/token/i.test(err.message)) {
|
||||
promptMakerworldToken(err.message, async () => {
|
||||
try {
|
||||
const watch = await postJson('/api/watches', {
|
||||
url,
|
||||
aiTag: document.getElementById('watchAi').checked,
|
||||
intervalHours: Number(document.getElementById('watchInterval').value),
|
||||
});
|
||||
await loadData();
|
||||
refreshWatches();
|
||||
return { ok: true, message: `Watching “${watch.title}”.` };
|
||||
} catch (retryErr) {
|
||||
return { ok: false, reason: /token/i.test(retryErr.message) ? 'makerworld_token' : null, error: retryErr.message };
|
||||
}
|
||||
}, 'Save & watch collection');
|
||||
}
|
||||
} finally {
|
||||
submit.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function handleWatchAction(btn) {
|
||||
const id = Number(btn.closest('[data-watch-id]').dataset.watchId);
|
||||
const action = btn.dataset.watch;
|
||||
btn.disabled = true;
|
||||
try {
|
||||
if (action === 'check') {
|
||||
await postJson(`/api/watches/${id}/check`);
|
||||
} else if (action === 'toggle') {
|
||||
const paused = btn.textContent.trim() === 'Pause';
|
||||
await postJson(`/api/watches/${id}`, { enabled: !paused }, 'PUT');
|
||||
} else if (action === 'remove') {
|
||||
if (!confirm('Stop watching this collection?\n\nThe models it already imported are kept.')) { btn.disabled = false; return; }
|
||||
await getJson(`/api/watches/${id}`, { method: 'DELETE' });
|
||||
await loadData();
|
||||
} else if (action === 'token') {
|
||||
promptMakerworldToken('MakerWorld didn’t accept your token when checking this collection.',
|
||||
() => retryWatchCheck(id), 'Save & check collection');
|
||||
}
|
||||
} catch (err) {
|
||||
if (err.message && /token/i.test(err.message)) {
|
||||
promptMakerworldToken(err.message, () => retryWatchCheck(id), 'Save & check collection');
|
||||
} else {
|
||||
alert(err.message);
|
||||
}
|
||||
}
|
||||
refreshWatches();
|
||||
}
|
||||
|
||||
async function retryWatchCheck(id) {
|
||||
const res = await fetch(`/api/watches/${id}/check`, { method: 'POST' });
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) return { ok: false, reason: data.reason, error: data.error };
|
||||
refreshWatches();
|
||||
return { ok: true, message: 'Token saved. Checking the collection now…' };
|
||||
}
|
||||
|
||||
async function handleRefreshMetadata() {
|
||||
const modelId = fileDetailModal.dataset.currentModelId;
|
||||
const btn = document.getElementById('refreshLicenseBtn');
|
||||
|
|
@ -2328,7 +2498,8 @@ async function handleImport(e) {
|
|||
// Details imported, files refused: offer to fix the token and fetch them now.
|
||||
closeModal(importModal);
|
||||
openModelDetail(data.id);
|
||||
promptMakerworldToken(data.id, `Imported “${data.name}”, but its files couldn’t be downloaded: ${data.filesError}`);
|
||||
promptMakerworldToken(`Imported “${data.name}”, but its files couldn’t be downloaded: ${data.filesError}`,
|
||||
() => retryModelDownload(data.id));
|
||||
} else if (data.metadataOnly) {
|
||||
status.textContent = data.filesError
|
||||
? `Imported "${data.name}", but its files couldn’t be downloaded: ${data.filesError}`
|
||||
|
|
|
|||
|
|
@ -98,6 +98,10 @@
|
|||
<span>AI & auto-tags</span>
|
||||
<span class="count ai-busy" id="aiBusy" hidden title="AI tagging is running">●</span>
|
||||
</div>
|
||||
<div class="project-item" data-scope="watch">
|
||||
<span>Watched</span>
|
||||
<span class="count" id="watchCount">0</span>
|
||||
</div>
|
||||
<div class="project-item" data-scope="trash">
|
||||
<span>Trash</span>
|
||||
<span class="count" id="trashCount">0</span>
|
||||
|
|
|
|||
|
|
@ -452,3 +452,18 @@ label { display: block; font-size: 13px; font-weight: 500; color: var(--text-mut
|
|||
.token-steps code, .token-steps kbd { font-family: var(--mono); font-size: 12px; }
|
||||
.token-status { min-height: 1em; }
|
||||
.token-status.is-error { color: var(--danger); }
|
||||
.watch-form { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 12px; }
|
||||
.watch-form input[type="url"] { flex: 1 1 320px; width: auto; }
|
||||
.watch-form label { display: inline-flex; align-items: center; gap: 6px; font-size: 13px; color: var(--text-muted); }
|
||||
.watch-form select { width: auto; padding: 7px 10px; font-size: 13px; }
|
||||
.watch-form input[type="checkbox"] { width: auto; accent-color: var(--accent); }
|
||||
.watch-form .btn { align-self: auto; }
|
||||
.watch-list { display: flex; flex-direction: column; gap: 10px; }
|
||||
.watch { border: 1px solid var(--border); border-radius: var(--radius-sm); background: var(--surface-2); padding: 12px 14px; display: flex; flex-direction: column; gap: 8px; }
|
||||
.watch.is-paused { opacity: 0.65; }
|
||||
.watch-head { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 12px; font-size: 14px; }
|
||||
.watch-actions { margin-left: auto; display: flex; flex-wrap: wrap; gap: 8px; }
|
||||
.watch-actions .btn-sm { align-self: auto; }
|
||||
.watch-meta { font-family: var(--mono); font-size: 12px; color: var(--text-dim); }
|
||||
.watch-recent { display: flex; flex-direction: column; gap: 4px; font-size: 13px; color: var(--text-muted); max-height: 180px; overflow-y: auto; }
|
||||
#watchStatus.is-error { color: var(--danger); }
|
||||
|
|
|
|||
266
server.js
266
server.js
|
|
@ -26,6 +26,7 @@ const thumbnails = require('./lib/thumbnails');
|
|||
const handoff = require('./lib/handoff');
|
||||
const ai = require('./lib/ai');
|
||||
const aiRuns = require('./lib/aiRuns');
|
||||
const collectionWatch = require('./lib/collectionWatch');
|
||||
const autotag = require('./lib/autotag');
|
||||
const embeddings = require('./lib/embeddings');
|
||||
|
||||
|
|
@ -789,93 +790,107 @@ function removeDerivedFiles(fileIds, entryIds) {
|
|||
}
|
||||
|
||||
// ----------------------------------------------------------------- import
|
||||
// Unified import. Thingiverse downloads files automatically (needs a token);
|
||||
// Printables / MakerWorld / any other URL import page metadata + preview image,
|
||||
// and the user attaches the files afterwards.
|
||||
// Imports one model URL: its details, its files where they can be downloaded, and
|
||||
// its license. Returns { model, existing, metadataFetched, filesError, filesErrorReason }.
|
||||
// Used by the import endpoint and by watched collections.
|
||||
async function importFromUrl(url, { projects = [] } = {}) {
|
||||
const detected = generic.detectSite(url);
|
||||
if (!detected) throw Object.assign(new Error('That does not look like a valid URL.'), { status: 400 });
|
||||
|
||||
if (detected.site === 'thingiverse') {
|
||||
return { model: await importThingiverse(url, projects), metadataFetched: true };
|
||||
}
|
||||
|
||||
// Metadata-only import for everything else.
|
||||
if (detected.id) {
|
||||
const existing = await store.findModelBySource(detected.site, detected.id);
|
||||
if (existing) return { model: publicModel(existing), existing: true };
|
||||
}
|
||||
// Best-effort: read the page (via FlareSolverr if it's Cloudflare-blocked).
|
||||
// If it still can't be read, fall back to a stub so it's not a dead end.
|
||||
let html = null;
|
||||
let meta = { title: null, description: '', image: null, author: null };
|
||||
let metadataFetched = false;
|
||||
try {
|
||||
html = await fetchPageHtml(url);
|
||||
meta = generic.parseMetadata(html, url);
|
||||
// Printables og:title carries a " | Download free STL model | Printables.com" suffix.
|
||||
if (detected.site === 'printables' && meta.title) meta.title = meta.title.split(' | ')[0].trim();
|
||||
// cubee3d (Wix) serves a generic site-wide og:title, so use the URL slug instead.
|
||||
if (detected.site === 'cubee3d') meta.title = generic.nameFromUrl(url);
|
||||
metadataFetched = Boolean(meta.title);
|
||||
} catch (err) {
|
||||
console.warn('Metadata fetch failed, creating stub model:', err.message);
|
||||
}
|
||||
|
||||
// Download the actual files where we can (MakerWorld via FlareSolverr+token,
|
||||
// Printables via its public GraphQL API), and capture the license.
|
||||
let files = [];
|
||||
let filesError = null; // why no files could be downloaded, shown after the import
|
||||
let filesErrorReason = null; // makerworld_token: the app offers to update the token
|
||||
let license = meta.license || null;
|
||||
let commercial = null;
|
||||
if (detected.site === 'makerworld' && html) {
|
||||
files = await downloadMakerworldFiles(html).catch((err) => {
|
||||
console.warn('MakerWorld file download failed:', err.message);
|
||||
filesError = err.message;
|
||||
filesErrorReason = err.reason || null;
|
||||
return [];
|
||||
});
|
||||
license = makerworld.parseLicense(html) || license;
|
||||
commercial = await resolveCommercial(license);
|
||||
} else if (detected.site === 'printables' && detected.id) {
|
||||
const result = await downloadPrintablesFiles(detected.id).catch((err) => {
|
||||
console.warn('Printables file download failed:', err.message);
|
||||
filesError = err.message;
|
||||
return { files: [] };
|
||||
});
|
||||
files = result.files;
|
||||
if (result.license) license = result.license;
|
||||
commercial = result.commercial ?? await resolveCommercial(license);
|
||||
} else if (detected.site === 'cubee3d') {
|
||||
// Auth-gated Wix site — files added manually. You hold a commercial license.
|
||||
license = 'Cubee3D Commercial License';
|
||||
commercial = await resolveCommercial(license, true);
|
||||
} else {
|
||||
commercial = await resolveCommercial(license);
|
||||
}
|
||||
|
||||
const thumbnail = (meta.image ? await downloadImageThumbnail(meta.image) : null) || pickThumbnail(files);
|
||||
const model = await store.createModel({
|
||||
name: meta.title || generic.nameFromUrl(url),
|
||||
description: meta.description || '',
|
||||
source_site: detected.site,
|
||||
source_url: url,
|
||||
source_id: detected.id,
|
||||
license,
|
||||
commercial_use: commercial,
|
||||
designer: meta.author ? { name: meta.author, source_site: detected.site } : null,
|
||||
thumbnail_path: thumbnail,
|
||||
tags: [],
|
||||
projects,
|
||||
files,
|
||||
});
|
||||
fingerprint.kick();
|
||||
return { model: publicModel(model), metadataFetched, filesError, filesErrorReason };}
|
||||
|
||||
// Unified import. Thingiverse downloads files automatically (needs a token);
|
||||
// Printables / MakerWorld / any other URL import page metadata + preview image,
|
||||
// and the user attaches the files afterwards.
|
||||
app.post('/api/import', async (req, res, next) => {
|
||||
try {
|
||||
const url = (req.body.url || '').trim();
|
||||
const projects = req.body.projects || [];
|
||||
const detected = generic.detectSite(url);
|
||||
if (!detected) return res.status(400).json({ error: 'That does not look like a valid URL.' });
|
||||
|
||||
if (detected.site === 'thingiverse') {
|
||||
const model = await importThingiverse(url, projects);
|
||||
return res.status(201).json(model);
|
||||
}
|
||||
|
||||
// Metadata-only import for everything else.
|
||||
if (detected.id) {
|
||||
const existing = await store.findModelBySource(detected.site, detected.id);
|
||||
if (existing) return res.status(409).json({ error: 'Already imported', model: publicModel(existing) });
|
||||
}
|
||||
// Best-effort: read the page (via FlareSolverr if it's Cloudflare-blocked).
|
||||
// If it still can't be read, fall back to a stub so it's not a dead end.
|
||||
let html = null;
|
||||
let meta = { title: null, description: '', image: null, author: null };
|
||||
let metadataFetched = false;
|
||||
try {
|
||||
html = await fetchPageHtml(url);
|
||||
meta = generic.parseMetadata(html, url);
|
||||
// Printables og:title carries a " | Download free STL model | Printables.com" suffix.
|
||||
if (detected.site === 'printables' && meta.title) meta.title = meta.title.split(' | ')[0].trim();
|
||||
// cubee3d (Wix) serves a generic site-wide og:title, so use the URL slug instead.
|
||||
if (detected.site === 'cubee3d') meta.title = generic.nameFromUrl(url);
|
||||
metadataFetched = Boolean(meta.title);
|
||||
} catch (err) {
|
||||
console.warn('Metadata fetch failed, creating stub model:', err.message);
|
||||
}
|
||||
|
||||
// Download the actual files where we can (MakerWorld via FlareSolverr+token,
|
||||
// Printables via its public GraphQL API), and capture the license.
|
||||
let files = [];
|
||||
let filesError = null; // why no files could be downloaded, shown after the import
|
||||
let filesErrorReason = null; // makerworld_token: the app offers to update the token
|
||||
let license = meta.license || null;
|
||||
let commercial = null;
|
||||
if (detected.site === 'makerworld' && html) {
|
||||
files = await downloadMakerworldFiles(html).catch((err) => {
|
||||
console.warn('MakerWorld file download failed:', err.message);
|
||||
filesError = err.message;
|
||||
filesErrorReason = err.reason || null;
|
||||
return [];
|
||||
});
|
||||
license = makerworld.parseLicense(html) || license;
|
||||
commercial = await resolveCommercial(license);
|
||||
} else if (detected.site === 'printables' && detected.id) {
|
||||
const result = await downloadPrintablesFiles(detected.id).catch((err) => {
|
||||
console.warn('Printables file download failed:', err.message);
|
||||
filesError = err.message;
|
||||
return { files: [] };
|
||||
});
|
||||
files = result.files;
|
||||
if (result.license) license = result.license;
|
||||
commercial = result.commercial ?? await resolveCommercial(license);
|
||||
} else if (detected.site === 'cubee3d') {
|
||||
// Auth-gated Wix site — files added manually. You hold a commercial license.
|
||||
license = 'Cubee3D Commercial License';
|
||||
commercial = await resolveCommercial(license, true);
|
||||
} else {
|
||||
commercial = await resolveCommercial(license);
|
||||
}
|
||||
|
||||
const thumbnail = (meta.image ? await downloadImageThumbnail(meta.image) : null) || pickThumbnail(files);
|
||||
const model = await store.createModel({
|
||||
name: meta.title || generic.nameFromUrl(url),
|
||||
description: meta.description || '',
|
||||
source_site: detected.site,
|
||||
source_url: url,
|
||||
source_id: detected.id,
|
||||
license,
|
||||
commercial_use: commercial,
|
||||
designer: meta.author ? { name: meta.author, source_site: detected.site } : null,
|
||||
thumbnail_path: thumbnail,
|
||||
tags: [],
|
||||
projects,
|
||||
files,
|
||||
const result = await importFromUrl((req.body.url || '').trim(), { projects: req.body.projects || [] });
|
||||
if (result.existing) return res.status(409).json({ error: 'Already imported', model: result.model });
|
||||
res.status(201).json({
|
||||
...result.model,
|
||||
metadataOnly: (result.model.files || []).length === 0,
|
||||
metadataFetched: Boolean(result.metadataFetched),
|
||||
filesError: result.filesError || null,
|
||||
filesErrorReason: result.filesErrorReason || null,
|
||||
});
|
||||
fingerprint.kick();
|
||||
res.status(201).json({ ...publicModel(model), metadataOnly: files.length === 0, metadataFetched, filesError, filesErrorReason });
|
||||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
|
|
@ -1120,7 +1135,9 @@ app.get('/api/library/stats', async (req, res, next) => {
|
|||
const [trash, groups, pending, thumbnailsPending, tagsPending, aiRunning] = await Promise.all([
|
||||
store.countTrashed(), findDuplicateGroups(), fingerprint.pendingCount(), thumbnails.pendingCount(),
|
||||
autotag.pendingCount(), aiRuns.hasActiveRun()]);
|
||||
res.json({ trash, duplicateGroups: groups.length, fingerprintsPending: pending, thumbnailsPending, tagsPending, aiRunning });
|
||||
const { rows: [watches] } = await db.query('SELECT COUNT(*)::int AS n FROM watched_collections WHERE enabled');
|
||||
res.json({ trash, duplicateGroups: groups.length, fingerprintsPending: pending, thumbnailsPending, tagsPending,
|
||||
aiRunning, watches: watches.n });
|
||||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
|
|
@ -1277,6 +1294,85 @@ app.post('/api/ai/runs/:id/resume', async (req, res, next) => {
|
|||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------------- watched collections
|
||||
// Reads a MakerWorld collection page as you: a private collection is only listed
|
||||
// when the request carries your token.
|
||||
const COLLECTION_PAGE_SIZE = 64; // the most that call returns at once
|
||||
const COLLECTION_MAX_PAGES = 10; // so one check reads at most ~640 models
|
||||
|
||||
async function fetchMakerworldCollection(url) {
|
||||
const { token, flareUrl } = await requireMakerworldSettings();
|
||||
const cookies = [{ name: 'token', value: token, domain: '.makerworld.com' }];
|
||||
const html = await flaresolverr.solve(flareUrl, url, { cookies });
|
||||
const collection = makerworld.parseCollection(html);
|
||||
if (!collection) {
|
||||
throw Object.assign(
|
||||
new Error('MakerWorld didn’t show that collection. If it’s private, your MakerWorld token may have expired; otherwise check the link.'),
|
||||
{ status: 502, reason: 'makerworld_token' });
|
||||
}
|
||||
|
||||
// The page itself lists only the first models; the rest come from the same call
|
||||
// its "load more" uses. A shared seed keeps the pages from overlapping.
|
||||
const byId = new Map(collection.designs.map((d) => [d.id, d]));
|
||||
let seed;
|
||||
for (let page = 0; page < COLLECTION_MAX_PAGES && byId.size < collection.total; page++) {
|
||||
const before = byId.size;
|
||||
let more;
|
||||
try {
|
||||
const json = await flaresolverr.solveJson(flareUrl,
|
||||
makerworld.collectionDesignsUrl(collection.id, { offset: page * COLLECTION_PAGE_SIZE, limit: COLLECTION_PAGE_SIZE, seed }),
|
||||
{ cookies, retries: 1 });
|
||||
more = makerworld.parseCollectionDesigns(json);
|
||||
} catch (err) {
|
||||
console.warn(`MakerWorld collection ${collection.id}: couldn't read more models:`, err.message);
|
||||
break;
|
||||
}
|
||||
if (seed === undefined) seed = more.seed;
|
||||
for (const design of more.designs) if (!byId.has(design.id)) byId.set(design.id, design);
|
||||
if (!more.designs.length || byId.size === before) break; // nothing new: stop asking
|
||||
}
|
||||
return { ...collection, designs: [...byId.values()] };
|
||||
}
|
||||
|
||||
app.get('/api/watches', async (req, res, next) => {
|
||||
try { res.json(await collectionWatch.list()); } catch (e) { next(e); }
|
||||
});
|
||||
|
||||
app.get('/api/watches/:id', async (req, res, next) => {
|
||||
try {
|
||||
const watch = await collectionWatch.get(Number(req.params.id));
|
||||
if (!watch) return res.status(404).json({ error: 'Not watching that collection' });
|
||||
res.json(watch);
|
||||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
app.post('/api/watches', async (req, res, next) => {
|
||||
try {
|
||||
const { url, aiTag, intervalHours } = req.body || {};
|
||||
res.status(201).json(await collectionWatch.add({ url: String(url || '').trim(), aiTag, intervalHours }));
|
||||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
app.put('/api/watches/:id', async (req, res, next) => {
|
||||
try {
|
||||
const watch = await collectionWatch.update(Number(req.params.id), req.body || {});
|
||||
if (!watch) return res.status(404).json({ error: 'Not watching that collection' });
|
||||
res.json(watch);
|
||||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
app.delete('/api/watches/:id', async (req, res, next) => {
|
||||
try {
|
||||
if (!(await collectionWatch.remove(Number(req.params.id)))) return res.status(404).json({ error: 'Not watching that collection' });
|
||||
res.json({ message: 'Stopped watching. The models it imported are kept.' });
|
||||
} catch (e) { next(e); }
|
||||
});
|
||||
|
||||
// Reads the collection now (so problems come back here) and imports in the background.
|
||||
app.post('/api/watches/:id/check', async (req, res, next) => {
|
||||
try { res.json(await collectionWatch.checkNow(Number(req.params.id))); } catch (e) { next(e); }
|
||||
});
|
||||
|
||||
// ----------------------------------------------------------------- collection suggestions
|
||||
function suggestionTarget(body) {
|
||||
const name = typeof (body && body.name) === 'string' ? body.name.trim() : '';
|
||||
|
|
@ -1376,6 +1472,12 @@ db.init()
|
|||
.then(() => {
|
||||
app.listen(PORT, () => console.log(`Printhive running on http://localhost:${PORT}`));
|
||||
fingerprint.kick(); // backfill fingerprints, tags and the search index
|
||||
collectionWatch.init({
|
||||
fetchCollection: fetchMakerworldCollection,
|
||||
importFromUrl,
|
||||
startAiRun: (modelIds) => aiRuns.createRun(modelIds, { describe: true }),
|
||||
});
|
||||
collectionWatch.start();
|
||||
return aiRuns.init(resolveAiConfig);
|
||||
})
|
||||
.catch((err) => { console.error('Failed to start (DB init):', err); process.exit(1); });
|
||||
|
|
|
|||
|
|
@ -28,3 +28,54 @@ test('a download answer without a link says why, and flags a refused token', ()
|
|||
{ message: 'MakerWorld didn’t give a download link: Model not found', tokenRejected: false });
|
||||
assert.deepEqual(makerworld.downloadProblem(null), { message: 'MakerWorld didn’t give a download link', tokenRejected: false });
|
||||
});
|
||||
|
||||
const collectionPage = (favorite, favoriteDesigns) =>
|
||||
`<html><script id="__NEXT_DATA__" type="application/json">${JSON.stringify({ props: { pageProps: { favorite, favoriteDesigns } } })}</script></html>`;
|
||||
|
||||
test('a collection page gives its models, name and total', () => {
|
||||
const html = collectionPage(
|
||||
{ id: 31103374, title: 'markets', designCnt: 3 },
|
||||
{ total: 3, hits: [
|
||||
{ id: 719274, slug: 'skeleton-bookmarks', title: 'Skeleton Bookmarks' },
|
||||
{ id: 926432, slug: 'doomhammer', title: 'Doomhammer' },
|
||||
{ id: 12345, title: 'No slug' },
|
||||
] });
|
||||
const collection = makerworld.parseCollection(html);
|
||||
assert.equal(collection.id, '31103374');
|
||||
assert.equal(collection.title, 'markets');
|
||||
assert.equal(collection.total, 3);
|
||||
assert.deepEqual(collection.designs.map((d) => d.id), ['719274', '926432', '12345']);
|
||||
assert.equal(makerworld.designUrl(collection.designs[0]), 'https://makerworld.com/en/models/719274-skeleton-bookmarks');
|
||||
assert.equal(makerworld.designUrl(collection.designs[2]), 'https://makerworld.com/en/models/12345');
|
||||
|
||||
// The page can hold fewer models than the collection does.
|
||||
const partial = makerworld.parseCollection(collectionPage({ id: 1, title: 'big', designCnt: 120 }, { total: 120, hits: [{ id: 5, slug: 'a', title: 'A' }] }));
|
||||
assert.equal(partial.total, 120);
|
||||
assert.equal(partial.designs.length, 1);
|
||||
});
|
||||
|
||||
test('a page without a collection (private, or Cloudflare) parses as nothing', () => {
|
||||
assert.equal(makerworld.parseCollection('<html><title>Just a moment...</title></html>'), null);
|
||||
assert.equal(makerworld.parseCollection(collectionPage(null, null)), null);
|
||||
// What MakerWorld serves for a private collection when the request isn't signed in.
|
||||
assert.equal(makerworld.parseCollection(collectionPage({ id: -1, title: '', status: 0, designCnt: 0 }, { total: 0, hits: [] })), null);
|
||||
});
|
||||
|
||||
test('more models are read from the collection’s own load-more call', () => {
|
||||
assert.equal(makerworld.collectionDesignsUrl('1465524', { offset: 64, limit: 64, seed: 0 }),
|
||||
'https://makerworld.com/api/v1/design-service/favorites/1465524/designs?offset=64&limit=64&seed=0');
|
||||
assert.equal(makerworld.collectionDesignsUrl('7'),
|
||||
'https://makerworld.com/api/v1/design-service/favorites/7/designs?offset=0&limit=64');
|
||||
|
||||
const page = makerworld.parseCollectionDesigns({ total: 535, seed: 0, hits: [{ id: 516328, slug: 'a-thing', title: 'A thing' }, { id: 887552 }] });
|
||||
assert.equal(page.total, 535);
|
||||
assert.equal(page.seed, 0);
|
||||
assert.deepEqual(page.designs, [{ id: '516328', slug: 'a-thing', title: 'A thing' }, { id: '887552', slug: '', title: '' }]);
|
||||
assert.deepEqual(makerworld.parseCollectionDesigns(null), { designs: [], total: 0, seed: undefined });
|
||||
});
|
||||
|
||||
test('collection links are recognised', () => {
|
||||
assert.equal(makerworld.collectionIdFromUrl('https://makerworld.com/en/collections/31103374-markets'), '31103374');
|
||||
assert.equal(makerworld.collectionIdFromUrl('https://makerworld.com/collections/31103374'), '31103374');
|
||||
assert.equal(makerworld.collectionIdFromUrl('https://makerworld.com/en/models/719274-skeleton-bookmarks'), null);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue