From 4c54f76ae37c79362f4fa542ebbeecf9667a9deb Mon Sep 17 00:00:00 2001 From: dlawler489 <104159223@student.swin.edu.au> Date: Wed, 16 Sep 2026 09:58:09 +1000 Subject: [PATCH] Read whole collections, and never check one twice at once - A collection page lists only its first ~20 models. Checks now page through the same load-more call the page uses (offset + a shared seed, 64 at a time, up to ~640 models), so a watch sees the whole collection. - A private collection read without a valid token comes back as a placeholder (id -1, no models), which was being treated as an empty collection. It now reports the token problem instead. - Adding a watch starts a check, and "Check now" could start a second one alongside it, importing models twice. One check per collection at a time now, and a check interrupted by a restart no longer looks like it's still running. Co-Authored-By: Claude Opus 5 --- README.md | 2 +- lib/collectionWatch.js | 20 ++++++++++++++++---- lib/importers/makerworld.js | 30 +++++++++++++++++++++++++++--- server.js | 31 +++++++++++++++++++++++++++---- test/makerworld.test.js | 15 +++++++++++++++ 5 files changed, 86 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4fd44cc..ab55fd6 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ A self-hosted web app for organizing your 3D-printing files (STL, 3MF, OBJ, FBX, - **Notes** — free-form notes per model, saved as you type. - **Search, filters & sorting** — search across names, descriptions, notes, tags, file names and designers; filter by format, source and license; include *or exclude* tags (match any/all); sort and group by source or format. Press / to search. - **Duplicate finder** — every file gets a SHA-256 content hash plus (for STL/OBJ/3MF) a geometry fingerprint, computed in the background. Models sharing files are grouped as *exact copies* or *same shape* (renamed, re-exported, converted — or mirrored, so check those), with a recommended copy to keep. Keeping one merges the others' tags, collections and notes into it and moves them to the trash; groups can be dismissed as "not duplicates". -- **Watch a MakerWorld collection** — paste a collection link (private ones work, using your MakerWorld token) and Printhive checks it on a schedule you choose, importing models added to it into a collection of the same name here, optionally AI-tagging the new ones. It imports a few per check and spaces them out; models you remove on MakerWorld stay in your library. Each watch shows when it last ran, what it imported, and why a check failed — with a one-click token update when MakerWorld refuses your token. +- **Watch a MakerWorld collection** — paste a collection link (private ones work, using your MakerWorld token) and Printhive checks it on a schedule you choose, importing models added to it into a collection of the same name here, optionally AI-tagging the new ones. Each check reads the whole collection (paging through MakerWorld's own load-more call, up to ~640 models) and imports up to 15 of the new ones, spaced out, returning shortly after if more are waiting; models you remove on MakerWorld stay in your library. Each watch shows when it last ran, what it imported, and why a check failed — with a one-click token update when MakerWorld refuses your token. - **AI tags & descriptions** — bring your own key for DeepSeek (the default), OpenRouter, OpenAI or any OpenAI-compatible server (Ollama, LM Studio). Tag the whole library, only models not yet tagged, or a single model from its page. Each model's name, file names and (optionally) thumbnail are sent; tags are only ever added and descriptions only filled where empty unless you ask. Optionally it also gives models with unclear names (file-name leftovers, typos, site boilerplate) a clearer one; the old name is kept and can be restored with Undo on the model page. It can also suggest collections (reusing yours where they fit): suggestions wait on the AI page, grouped by name, until you create/add or dismiss them. Each run can do any mix of tags & descriptions, names and collections, and a reply that isn't the JSON asked for is retried once. Runs show live progress, tokens and cost, can be stopped and resumed, stop at an optional spending limit, and pause (rather than keep spending) if the server restarts. "Save & test connection" in Settings checks the key, model and whether it accepts images. - **Search by meaning** — switch the search box to *Meaning* to find models by what they are ("wall hook" finds a "coat hanger bracket"). Uses a small embedding model (bge-small-en-v1.5, ~34 MB) that runs on the server, so it's free and private; the index updates in the background as models change. - **Automatic tags** — strict whole-word rules over names, file names and archive contents (part types, mechanisms, materials, techniques) plus your own "word → tag" rules. Switch it off (optionally removing its tags) or re-apply it any time. Tags you add yourself are never changed; AI and rule tags are marked on the model page. diff --git a/lib/collectionWatch.js b/lib/collectionWatch.js index adb034f..9166600 100644 --- a/lib/collectionWatch.js +++ b/lib/collectionWatch.js @@ -16,6 +16,7 @@ const INTERVALS = [1, 3, 6, 12, 24, 48]; let deps = null; let timer = null; let busy = false; +const checking = new Set(); // watch ids being checked right now, so checks can't overlap const httpError = (message, status, reason) => Object.assign(new Error(message), { status, reason }); const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -26,6 +27,9 @@ function init(dependencies) { // Checks anything due, every few minutes. function start() { + // A check that was running when the server stopped isn't running now. + db.query(`UPDATE watched_collections SET last_status = NULL WHERE last_status = 'checking'`) + .catch((err) => console.warn('Could not clear stale watch checks:', err.message)); clearInterval(timer); timer = setInterval(() => { runDue().catch((err) => console.error('Collection watch failed:', err.message)); }, TICK_MS); if (timer.unref) timer.unref(); @@ -76,8 +80,8 @@ async function add({ url, aiTag = false, intervalHours = 6 }) { const project = await store.findOrCreateProject(collection.title || `MakerWorld collection ${sourceId}`); const hours = INTERVALS.includes(Number(intervalHours)) ? Number(intervalHours) : 6; const { rows: [watch] } = await db.query( - `INSERT INTO watched_collections (site, source_id, url, title, project_id, ai_tag, interval_hours, total, listed) - VALUES ('makerworld', $1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, + `INSERT INTO watched_collections (site, source_id, url, title, project_id, ai_tag, interval_hours, total, listed, last_status) + VALUES ('makerworld', $1, $2, $3, $4, $5, $6, $7, $8, 'checking') RETURNING id`, [sourceId, url, collection.title, project.id, Boolean(aiTag), hours, collection.total, collection.designs.length]); checkNow(watch.id).catch((err) => console.warn(`Watch ${watch.id} first check failed:`, err.message)); return get(watch.id); @@ -109,6 +113,7 @@ async function remove(id) { async function checkNow(id) { const watch = await get(id); if (!watch) throw httpError('Not watching that collection.', 404); + if (checking.has(id)) return watch; // already going; don't import anything twice let collection; try { collection = await deps.fetchCollection(watch.url); @@ -120,7 +125,10 @@ async function checkNow(id) { `UPDATE watched_collections SET title = $2, total = $3, listed = $4, last_status = 'checking', last_error = NULL, last_error_reason = NULL WHERE id = $1`, [id, collection.title || watch.title, collection.total, collection.designs.length]); - importFound(watch, collection).catch((err) => console.error(`Watch ${id} import failed:`, err.message)); + checking.add(id); + importFound(watch, collection) + .catch((err) => console.error(`Watch ${id} import failed:`, err.message)) + .finally(() => checking.delete(id)); return get(id); } @@ -200,11 +208,13 @@ async function runDue() { try { for (;;) { const { rows: [due] } = await db.query( - `SELECT id FROM watched_collections WHERE enabled AND next_check_at <= now() ORDER BY next_check_at LIMIT 1`); + `SELECT id FROM watched_collections WHERE enabled AND next_check_at <= now() + AND id <> ALL($1::int[]) ORDER BY next_check_at LIMIT 1`, [[...checking]]); if (!due) return; // checkNow leaves the import running in the background; wait for it here so // watches are checked one after another. const watch = await get(due.id); + checking.add(watch.id); try { const collection = await deps.fetchCollection(watch.url); await db.query( @@ -213,6 +223,8 @@ async function runDue() { await importFound(watch, collection); } catch (err) { await recordFailure(watch, err); + } finally { + checking.delete(watch.id); } } } finally { diff --git a/lib/importers/makerworld.js b/lib/importers/makerworld.js index 9cb1a65..8c02a54 100644 --- a/lib/importers/makerworld.js +++ b/lib/importers/makerworld.js @@ -52,6 +52,24 @@ function collectionIdFromUrl(url) { return m ? m[1] : null; } +// Where a collection page loads the rest of its models from (its own "load more" +// call). A stable `seed` keeps the pages from overlapping. +const collectionDesignsUrl = (collectionId, { offset = 0, limit = 64, seed } = {}) => + `https://makerworld.com/api/v1/design-service/favorites/${collectionId}/designs?offset=${offset}&limit=${limit}` + + (seed === undefined || seed === null ? '' : `&seed=${seed}`); + +// That call's answer -> { designs, total, seed }. +function parseCollectionDesigns(json) { + const hits = (json && json.hits) || []; + return { + designs: hits + .filter((d) => d && (typeof d.id === 'number' || typeof d.id === 'string')) + .map((d) => ({ id: String(d.id), slug: d.slug || '', title: d.title || '' })), + total: Number(json && json.total) || 0, + seed: json ? json.seed : undefined, + }; +} + const designUrl = (design) => `https://makerworld.com/en/models/${design.id}${design.slug ? `-${design.slug}` : ''}`; // A collection page -> { id, title, total, designs: [{ id, slug, title }] }, or null @@ -62,14 +80,17 @@ function parseCollection(html) { if (!m) return null; let props; try { props = JSON.parse(m[1]).props.pageProps; } catch { return null; } - if (!props || !props.favorite || !props.favorite.id) return null; + // A collection you can't see (private, read without a signed-in token) comes back + // as a placeholder: id -1, no title, no models. + const id = Number(props && props.favorite && props.favorite.id); + if (!Number.isFinite(id) || id <= 0) return null; const hits = (props.favoriteDesigns && props.favoriteDesigns.hits) || []; const designs = hits .filter((d) => d && (typeof d.id === 'number' || typeof d.id === 'string')) .map((d) => ({ id: String(d.id), slug: d.slug || '', title: d.title || '' })); const total = Number((props.favoriteDesigns && props.favoriteDesigns.total) ?? props.favorite.designCnt); return { - id: String(props.favorite.id), + id: String(id), title: props.favorite.title || '', total: Number.isFinite(total) ? total : designs.length, designs, @@ -82,4 +103,7 @@ function parseLicense(html) { return m ? m[1] : null; } -module.exports = { parseInstances, resolveDownload, downloadProblem, parseLicense, parseCollection, collectionIdFromUrl, designUrl }; +module.exports = { + parseInstances, resolveDownload, downloadProblem, parseLicense, + parseCollection, parseCollectionDesigns, collectionIdFromUrl, collectionDesignsUrl, designUrl, +}; diff --git a/server.js b/server.js index bed2d18..03f0e19 100644 --- a/server.js +++ b/server.js @@ -1297,18 +1297,41 @@ app.post('/api/ai/runs/:id/resume', async (req, res, next) => { // ----------------------------------------------------------------- watched collections // Reads a MakerWorld collection page as you: a private collection is only listed // when the request carries your token. +const COLLECTION_PAGE_SIZE = 64; // the most that call returns at once +const COLLECTION_MAX_PAGES = 10; // so one check reads at most ~640 models + async function fetchMakerworldCollection(url) { const { token, flareUrl } = await requireMakerworldSettings(); - const html = await flaresolverr.solve(flareUrl, url, { - cookies: [{ name: 'token', value: token, domain: '.makerworld.com' }], - }); + const cookies = [{ name: 'token', value: token, domain: '.makerworld.com' }]; + const html = await flaresolverr.solve(flareUrl, url, { cookies }); const collection = makerworld.parseCollection(html); if (!collection) { throw Object.assign( new Error('MakerWorld didn’t show that collection. If it’s private, your MakerWorld token may have expired; otherwise check the link.'), { status: 502, reason: 'makerworld_token' }); } - return collection; + + // The page itself lists only the first models; the rest come from the same call + // its "load more" uses. A shared seed keeps the pages from overlapping. + const byId = new Map(collection.designs.map((d) => [d.id, d])); + let seed; + for (let page = 0; page < COLLECTION_MAX_PAGES && byId.size < collection.total; page++) { + const before = byId.size; + let more; + try { + const json = await flaresolverr.solveJson(flareUrl, + makerworld.collectionDesignsUrl(collection.id, { offset: page * COLLECTION_PAGE_SIZE, limit: COLLECTION_PAGE_SIZE, seed }), + { cookies, retries: 1 }); + more = makerworld.parseCollectionDesigns(json); + } catch (err) { + console.warn(`MakerWorld collection ${collection.id}: couldn't read more models:`, err.message); + break; + } + if (seed === undefined) seed = more.seed; + for (const design of more.designs) if (!byId.has(design.id)) byId.set(design.id, design); + if (!more.designs.length || byId.size === before) break; // nothing new: stop asking + } + return { ...collection, designs: [...byId.values()] }; } app.get('/api/watches', async (req, res, next) => { diff --git a/test/makerworld.test.js b/test/makerworld.test.js index c4144c6..e54130f 100644 --- a/test/makerworld.test.js +++ b/test/makerworld.test.js @@ -57,6 +57,21 @@ test('a collection page gives its models, name and total', () => { test('a page without a collection (private, or Cloudflare) parses as nothing', () => { assert.equal(makerworld.parseCollection('Just a moment...'), null); assert.equal(makerworld.parseCollection(collectionPage(null, null)), null); + // What MakerWorld serves for a private collection when the request isn't signed in. + assert.equal(makerworld.parseCollection(collectionPage({ id: -1, title: '', status: 0, designCnt: 0 }, { total: 0, hits: [] })), null); +}); + +test('more models are read from the collection’s own load-more call', () => { + assert.equal(makerworld.collectionDesignsUrl('1465524', { offset: 64, limit: 64, seed: 0 }), + 'https://makerworld.com/api/v1/design-service/favorites/1465524/designs?offset=64&limit=64&seed=0'); + assert.equal(makerworld.collectionDesignsUrl('7'), + 'https://makerworld.com/api/v1/design-service/favorites/7/designs?offset=0&limit=64'); + + const page = makerworld.parseCollectionDesigns({ total: 535, seed: 0, hits: [{ id: 516328, slug: 'a-thing', title: 'A thing' }, { id: 887552 }] }); + assert.equal(page.total, 535); + assert.equal(page.seed, 0); + assert.deepEqual(page.designs, [{ id: '516328', slug: 'a-thing', title: 'A thing' }, { id: '887552', slug: '', title: '' }]); + assert.deepEqual(makerworld.parseCollectionDesigns(null), { designs: [], total: 0, seed: undefined }); }); test('collection links are recognised', () => {