From 32299da05e3b79de0aef4c5d2f3e4b55de52e5f8 Mon Sep 17 00:00:00 2001 From: dlawler489 <104159223@student.swin.edu.au> Date: Wed, 16 Sep 2026 19:18:55 +1000 Subject: [PATCH] Let a watch download files as models arrive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checks have imported metadata only since MakerWorld refused bulk downloads. Today's measurement changed that picture: it refuses bursts a few seconds apart, but 67 models downloaded in a row about 20 seconds apart without one refusal. So downloading during a check is viable — at the right pace. Per watch, off by default: "Files: on/off" on the row, a checkbox when adding one. With it on, a check spaces downloads 20 seconds apart instead of 3, and stops after two refusals in a row, coming back shortly rather than collecting refusals for the whole batch. A model whose files were refused still lands as 'partial', so "Get files" can finish it later. Auto-imports now get the same AI treatment a manual run does — tags, a clearer name where the site's is boilerplate, and collections to review — rather than tags alone. Co-Authored-By: Claude Opus 5 --- db/schema.sql | 3 +++ lib/collectionWatch.js | 42 ++++++++++++++++++++++++++++++------------ public/app.js | 12 ++++++++++-- server.js | 9 ++++++--- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/db/schema.sql b/db/schema.sql index d86de6c..94e317d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -235,6 +235,7 @@ CREATE TABLE IF NOT EXISTS watched_collections ( title TEXT NOT NULL DEFAULT '', -- its name there, refreshed on each check project_id INTEGER REFERENCES projects(id) ON DELETE SET NULL, -- where new models land ai_tag BOOLEAN NOT NULL DEFAULT false, + fetch_files BOOLEAN NOT NULL DEFAULT false, -- download files during a check, not just metadata interval_hours INTEGER NOT NULL DEFAULT 6, enabled BOOLEAN NOT NULL DEFAULT true, last_checked_at TIMESTAMPTZ, @@ -260,3 +261,5 @@ CREATE TABLE IF NOT EXISTS watched_collection_items ( seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), PRIMARY KEY (watch_id, design_id) ); + +ALTER TABLE watched_collections ADD COLUMN IF NOT EXISTS fetch_files BOOLEAN NOT NULL DEFAULT false; diff --git a/lib/collectionWatch.js b/lib/collectionWatch.js index 5cce293..39033ac 100644 --- a/lib/collectionWatch.js +++ b/lib/collectionWatch.js @@ -8,6 +8,9 @@ const makerworld = require('./importers/makerworld'); const IMPORT_LIMIT = 15; // models per check, so a big collection trickles in const BETWEEN_IMPORTS_MS = 3000; // FlareSolverr runs one browser request at a time +// Downloads need much more room: MakerWorld refuses bursts a few seconds apart, +// but 67 in a row about 20 seconds apart went through without one refusal. +const BETWEEN_DOWNLOADS_MS = 20000; const MORE_SOON_MINUTES = 10; // when a check hit the limit, come back sooner const TICK_MS = 5 * 60 * 1000; const INTERVALS = [1, 3, 6, 12, 24, 48]; @@ -83,7 +86,7 @@ async function get(id) { // Adds a watch. Reads the collection first, so a bad link or a refused token is // reported now rather than silently later. -async function add({ url, aiTag = false, intervalHours = 6 }) { +async function add({ url, aiTag = false, intervalHours = 6, fetchFiles = false }) { const sourceId = makerworld.collectionIdFromUrl(url); if (!sourceId) throw httpError('That isn’t a MakerWorld collection link (makerworld.com/…/collections/…).', 400); const { rowCount } = await db.query('SELECT 1 FROM watched_collections WHERE site = $1 AND source_id = $2', ['makerworld', sourceId]); @@ -93,9 +96,10 @@ async function add({ url, aiTag = false, intervalHours = 6 }) { const project = await store.findOrCreateProject(collection.title || `MakerWorld collection ${sourceId}`); const hours = INTERVALS.includes(Number(intervalHours)) ? Number(intervalHours) : 6; const { rows: [watch] } = await db.query( - `INSERT INTO watched_collections (site, source_id, url, title, project_id, ai_tag, interval_hours, total, listed, last_status) - VALUES ('makerworld', $1, $2, $3, $4, $5, $6, $7, $8, 'checking') RETURNING id`, - [sourceId, url, collection.title, project.id, Boolean(aiTag), hours, collection.total, collection.designs.length]); + `INSERT INTO watched_collections (site, source_id, url, title, project_id, ai_tag, interval_hours, total, listed, last_status, fetch_files) + VALUES ('makerworld', $1, $2, $3, $4, $5, $6, $7, $8, 'checking', $9) RETURNING id`, + [sourceId, url, collection.title, project.id, Boolean(aiTag), hours, collection.total, collection.designs.length, + Boolean(fetchFiles)]); // The collection was just read, so import straight from it rather than // fetching the whole thing again. const added = await get(watch.id); @@ -106,16 +110,18 @@ async function add({ url, aiTag = false, intervalHours = 6 }) { return added; } -async function update(id, { enabled, aiTag, intervalHours }) { +async function update(id, { enabled, aiTag, intervalHours, fetchFiles }) { const hours = INTERVALS.includes(Number(intervalHours)) ? Number(intervalHours) : null; const { rowCount } = await db.query( `UPDATE watched_collections SET enabled = COALESCE($2, enabled), ai_tag = COALESCE($3, ai_tag), interval_hours = COALESCE($4, interval_hours), + fetch_files = COALESCE($5, fetch_files), next_check_at = CASE WHEN $2 IS TRUE THEN now() ELSE next_check_at END WHERE id = $1`, - [id, enabled === undefined ? null : Boolean(enabled), aiTag === undefined ? null : Boolean(aiTag), hours]); + [id, enabled === undefined ? null : Boolean(enabled), aiTag === undefined ? null : Boolean(aiTag), hours, + fetchFiles === undefined ? null : Boolean(fetchFiles)]); if (!rowCount) return null; if (enabled) runDue().catch(() => {}); return get(id); @@ -169,6 +175,8 @@ async function importFound(watch, collection) { let added = 0; let problem = null; + let fileMisses = 0; // downloads refused in a row + let stoppedEarly = false; // left models unimported, so come back soon const fresh = []; for (const design of todo) { // Paused while this batch was running: stop, rather than import the rest. @@ -178,7 +186,7 @@ async function importFound(watch, collection) { try { const result = await deps.importFromUrl(url, { projects: watch.project_id ? [watch.project_id] : [], - withFiles: false, // MakerWorld allows only a couple of downloads in a row + withFiles: Boolean(watch.fetch_files), // off by default: MakerWorld refuses quick bursts }); if (result.existing) { if (watch.project_id) await store.addModelToProject(result.model.id, watch.project_id); @@ -186,9 +194,19 @@ async function importFound(watch, collection) { } else { added += 1; fresh.push(result.model.id); - // A check brings in the page, not the files, so every new model starts - // 'partial' — "Get files" on the ones you want fills them in. - await recordItem(watch.id, design, result.model.id, 'partial', result.filesError || null); + // Without files (or when they couldn't be fetched) the model is only + // 'partial' — "Get files" fills it in later. + const gotFiles = watch.fetch_files && !result.filesError; + await recordItem(watch.id, design, result.model.id, gotFiles ? 'imported' : 'partial', + result.filesError || null); + } + // Refused downloads mean we're asking too fast. Stop the batch and come + // back shortly, rather than collecting refusals for the whole of it. + if (watch.fetch_files && result.filesError) { + fileMisses += 1; + if (fileMisses >= 2 && result.filesErrorReason !== 'makerworld_token') { stoppedEarly = true; break; } + } else if (!result.filesError) { + fileMisses = 0; } // A refused token means every later download fails too: stop and say so. if (result.filesErrorReason === 'makerworld_token') { @@ -199,10 +217,10 @@ async function importFound(watch, collection) { await recordItem(watch.id, design, null, 'error', err.message); if (err.reason === 'makerworld_token') { problem = err; break; } } - await sleep(BETWEEN_IMPORTS_MS); + await sleep(watch.fetch_files ? BETWEEN_DOWNLOADS_MS : BETWEEN_IMPORTS_MS); } - const more = pending.length > todo.length; + const more = pending.length > todo.length || stoppedEarly; if (problem) { await recordFailure({ ...watch, id: watch.id }, problem); } else { diff --git a/public/app.js b/public/app.js index 398bc12..23e0add 100644 --- a/public/app.js +++ b/public/app.js @@ -2183,12 +2183,14 @@ function watchPanelHtml() {

Watch a MakerWorld collection

Printhive checks the collection and imports models added to it, into a collection of the same name here. Private collections work too, using your MakerWorld token. Models you remove there stay here. - A check brings in each model's page — name, description, picture, licence — but not its files, because MakerWorld - only allows a couple of downloads in a row; use “Get files” on the ones you want.

+ A check brings in each model's page — name, description, picture, licence. Files are left for you to fetch with + “Get files”, unless you switch downloads on: MakerWorld refuses quick bursts, so a check that downloads spaces + them about 20 seconds apart and takes correspondingly longer.

+
@@ -2217,6 +2219,7 @@ function watchHtml(w) { + @@ -2279,6 +2282,7 @@ async function handleAddWatch(e) { const watch = await postJson('/api/watches', { url, aiTag: document.getElementById('watchAi').checked, + fetchFiles: document.getElementById('watchFiles').checked, intervalHours: Number(document.getElementById('watchInterval').value), }); document.getElementById('watchUrl').value = ''; @@ -2294,6 +2298,7 @@ async function handleAddWatch(e) { const watch = await postJson('/api/watches', { url, aiTag: document.getElementById('watchAi').checked, + fetchFiles: document.getElementById('watchFiles').checked, intervalHours: Number(document.getElementById('watchInterval').value), }); await loadData(); @@ -2324,6 +2329,9 @@ async function handleWatchAction(btn) { if (!confirm('Stop watching this collection?\n\nThe models it already imported are kept.')) { btn.disabled = false; return; } await getJson(`/api/watches/${id}`, { method: 'DELETE' }); await loadData(); + } else if (action === 'filemode') { + const on = btn.textContent.trim() === 'Files: on'; + await postJson(`/api/watches/${id}`, { fetchFiles: !on }, 'PUT'); } else if (action === 'files') { btn.textContent = 'Getting files…'; await postJson(`/api/models/${Number(btn.dataset.modelId)}/download-files`); diff --git a/server.js b/server.js index 2cfa59a..a6f0573 100644 --- a/server.js +++ b/server.js @@ -1358,8 +1358,8 @@ app.get('/api/watches/:id', async (req, res, next) => { app.post('/api/watches', async (req, res, next) => { try { - const { url, aiTag, intervalHours } = req.body || {}; - res.status(201).json(await collectionWatch.add({ url: String(url || '').trim(), aiTag, intervalHours })); + const { url, aiTag, intervalHours, fetchFiles } = req.body || {}; + res.status(201).json(await collectionWatch.add({ url: String(url || '').trim(), aiTag, intervalHours, fetchFiles })); } catch (e) { next(e); } }); @@ -1485,7 +1485,10 @@ db.init() collectionWatch.init({ fetchCollection: fetchMakerworldCollection, importFromUrl, - startAiRun: (modelIds) => aiRuns.createRun(modelIds, { describe: true }), + // Same treatment a manual run gives: tags and a description, a clearer name + // where the site's one is boilerplate, and collections to review. + startAiRun: (modelIds) => aiRuns.createRun(modelIds, + { describe: true, renameUnclear: true, suggestCollections: true }), }); collectionWatch.start(); return aiRuns.init(resolveAiConfig);