Merge per-watch file downloading

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

db/schema.sql

@@ -235,6 +235,7 @@ CREATE TABLE IF NOT EXISTS watched_collections (
   title             TEXT NOT NULL DEFAULT '',   -- its name there, refreshed on each check
   project_id        INTEGER REFERENCES projects(id) ON DELETE SET NULL, -- where new models land
   ai_tag            BOOLEAN NOT NULL DEFAULT false,
+  fetch_files       BOOLEAN NOT NULL DEFAULT false, -- download files during a check, not just metadata
   interval_hours    INTEGER NOT NULL DEFAULT 6,
   enabled           BOOLEAN NOT NULL DEFAULT true,
   last_checked_at   TIMESTAMPTZ,
@@ -260,3 +261,5 @@ CREATE TABLE IF NOT EXISTS watched_collection_items (
   seen_at   TIMESTAMPTZ NOT NULL DEFAULT now(),
   PRIMARY KEY (watch_id, design_id)
 );
+
+ALTER TABLE watched_collections ADD COLUMN IF NOT EXISTS fetch_files BOOLEAN NOT NULL DEFAULT false;

lib/collectionWatch.js

@@ -8,6 +8,9 @@ const makerworld = require('./importers/makerworld');
 
 const IMPORT_LIMIT = 15;          // models per check, so a big collection trickles in
 const BETWEEN_IMPORTS_MS = 3000;  // FlareSolverr runs one browser request at a time
+// Downloads need much more room: MakerWorld refuses bursts a few seconds apart,
+// but 67 in a row about 20 seconds apart went through without one refusal.
+const BETWEEN_DOWNLOADS_MS = 20000;
 const MORE_SOON_MINUTES = 10;     // when a check hit the limit, come back sooner
 const TICK_MS = 5 * 60 * 1000;
 const INTERVALS = [1, 3, 6, 12, 24, 48];
@@ -83,7 +86,7 @@ async function get(id) {
 
 // Adds a watch. Reads the collection first, so a bad link or a refused token is
 // reported now rather than silently later.
-async function add({ url, aiTag = false, intervalHours = 6 }) {
+async function add({ url, aiTag = false, intervalHours = 6, fetchFiles = false }) {
   const sourceId = makerworld.collectionIdFromUrl(url);
   if (!sourceId) throw httpError('That isn’t a MakerWorld collection link (makerworld.com/…/collections/…).', 400);
   const { rowCount } = await db.query('SELECT 1 FROM watched_collections WHERE site = $1 AND source_id = $2', ['makerworld', sourceId]);
@@ -93,9 +96,10 @@ async function add({ url, aiTag = false, intervalHours = 6 }) {
   const project = await store.findOrCreateProject(collection.title || `MakerWorld collection ${sourceId}`);
   const hours = INTERVALS.includes(Number(intervalHours)) ? Number(intervalHours) : 6;
   const { rows: [watch] } = await db.query(
-    `INSERT INTO watched_collections (site, source_id, url, title, project_id, ai_tag, interval_hours, total, listed, last_status)
-     VALUES ('makerworld', $1, $2, $3, $4, $5, $6, $7, $8, 'checking') RETURNING id`,
-    [sourceId, url, collection.title, project.id, Boolean(aiTag), hours, collection.total, collection.designs.length]);
+    `INSERT INTO watched_collections (site, source_id, url, title, project_id, ai_tag, interval_hours, total, listed, last_status, fetch_files)
+     VALUES ('makerworld', $1, $2, $3, $4, $5, $6, $7, $8, 'checking', $9) RETURNING id`,
+    [sourceId, url, collection.title, project.id, Boolean(aiTag), hours, collection.total, collection.designs.length,
+     Boolean(fetchFiles)]);
   // The collection was just read, so import straight from it rather than
   // fetching the whole thing again.
   const added = await get(watch.id);
@@ -106,16 +110,18 @@ async function add({ url, aiTag = false, intervalHours = 6 }) {
   return added;
 }
 
-async function update(id, { enabled, aiTag, intervalHours }) {
+async function update(id, { enabled, aiTag, intervalHours, fetchFiles }) {
   const hours = INTERVALS.includes(Number(intervalHours)) ? Number(intervalHours) : null;
   const { rowCount } = await db.query(
     `UPDATE watched_collections SET
        enabled = COALESCE($2, enabled),
        ai_tag = COALESCE($3, ai_tag),
        interval_hours = COALESCE($4, interval_hours),
+       fetch_files = COALESCE($5, fetch_files),
        next_check_at = CASE WHEN $2 IS TRUE THEN now() ELSE next_check_at END
      WHERE id = $1`,
-    [id, enabled === undefined ? null : Boolean(enabled), aiTag === undefined ? null : Boolean(aiTag), hours]);
+    [id, enabled === undefined ? null : Boolean(enabled), aiTag === undefined ? null : Boolean(aiTag), hours,
+     fetchFiles === undefined ? null : Boolean(fetchFiles)]);
   if (!rowCount) return null;
   if (enabled) runDue().catch(() => {});
   return get(id);
@@ -169,6 +175,8 @@ async function importFound(watch, collection) {
 
   let added = 0;
   let problem = null;
+  let fileMisses = 0;      // downloads refused in a row
+  let stoppedEarly = false; // left models unimported, so come back soon
   const fresh = [];
   for (const design of todo) {
     // Paused while this batch was running: stop, rather than import the rest.
@@ -178,7 +186,7 @@ async function importFound(watch, collection) {
     try {
       const result = await deps.importFromUrl(url, {
         projects: watch.project_id ? [watch.project_id] : [],
-        withFiles: false, // MakerWorld allows only a couple of downloads in a row
+        withFiles: Boolean(watch.fetch_files), // off by default: MakerWorld refuses quick bursts
       });
       if (result.existing) {
         if (watch.project_id) await store.addModelToProject(result.model.id, watch.project_id);
@@ -186,9 +194,19 @@ async function importFound(watch, collection) {
       } else {
         added += 1;
         fresh.push(result.model.id);
-        // A check brings in the page, not the files, so every new model starts
-        // 'partial' — "Get files" on the ones you want fills them in.
-        await recordItem(watch.id, design, result.model.id, 'partial', result.filesError || null);
+        // Without files (or when they couldn't be fetched) the model is only
+        // 'partial' — "Get files" fills it in later.
+        const gotFiles = watch.fetch_files && !result.filesError;
+        await recordItem(watch.id, design, result.model.id, gotFiles ? 'imported' : 'partial',
+          result.filesError || null);
+      }
+      // Refused downloads mean we're asking too fast. Stop the batch and come
+      // back shortly, rather than collecting refusals for the whole of it.
+      if (watch.fetch_files && result.filesError) {
+        fileMisses += 1;
+        if (fileMisses >= 2 && result.filesErrorReason !== 'makerworld_token') { stoppedEarly = true; break; }
+      } else if (!result.filesError) {
+        fileMisses = 0;
       }
       // A refused token means every later download fails too: stop and say so.
       if (result.filesErrorReason === 'makerworld_token') {
@@ -199,10 +217,10 @@ async function importFound(watch, collection) {
       await recordItem(watch.id, design, null, 'error', err.message);
       if (err.reason === 'makerworld_token') { problem = err; break; }
     }
-    await sleep(BETWEEN_IMPORTS_MS);
+    await sleep(watch.fetch_files ? BETWEEN_DOWNLOADS_MS : BETWEEN_IMPORTS_MS);
   }
 
-  const more = pending.length > todo.length;
+  const more = pending.length > todo.length || stoppedEarly;
   if (problem) {
     await recordFailure({ ...watch, id: watch.id }, problem);
   } else {

public/app.js

@@ -2183,12 +2183,14 @@ function watchPanelHtml() {
       <div class="ai-card-head"><h3>Watch a MakerWorld collection</h3></div>
       <p class="form-hint">Printhive checks the collection and imports models added to it, into a collection of the same
         name here. Private collections work too, using your MakerWorld token. Models you remove there stay here.
-        A check brings in each model's page — name, description, picture, licence — but not its files, because MakerWorld
-        only allows a couple of downloads in a row; use “Get files” on the ones you want.</p>
+        A check brings in each model's page — name, description, picture, licence. Files are left for you to fetch with
+        “Get files”, unless you switch downloads on: MakerWorld refuses quick bursts, so a check that downloads spaces
+        them about 20 seconds apart and takes correspondingly longer.</p>
       <form id="watchForm" class="watch-form">
         <input type="url" id="watchUrl" placeholder="https://makerworld.com/en/collections/…" aria-label="Collection link" required>
         <label>Check <select id="watchInterval" aria-label="How often to check">${intervals}</select></label>
         <label><input type="checkbox" id="watchAi"> Tag new models with AI</label>
+        <label><input type="checkbox" id="watchFiles"> Download files too (about 20s per model)</label>
         <button type="submit" class="btn btn-primary" id="watchSubmit">Watch collection</button>
       </form>
       <div id="watchStatus" class="form-hint"></div>
@@ -2217,6 +2219,7 @@ function watchHtml(w) {
         <span class="watch-actions">
           <button type="button" class="btn btn-secondary btn-sm" data-watch="check">Check now</button>
           <button type="button" class="btn btn-secondary btn-sm" data-watch="toggle">${w.enabled ? 'Pause' : 'Resume'}</button>
+          <button type="button" class="btn btn-secondary btn-sm" data-watch="filemode">${w.fetch_files ? 'Files: on' : 'Files: off'}</button>
           <button type="button" class="btn btn-secondary btn-sm" data-watch="remove">Stop watching</button>
         </span>
       </div>
@@ -2279,6 +2282,7 @@ async function handleAddWatch(e) {
     const watch = await postJson('/api/watches', {
       url,
       aiTag: document.getElementById('watchAi').checked,
+      fetchFiles: document.getElementById('watchFiles').checked,
       intervalHours: Number(document.getElementById('watchInterval').value),
     });
     document.getElementById('watchUrl').value = '';
@@ -2294,6 +2298,7 @@ async function handleAddWatch(e) {
           const watch = await postJson('/api/watches', {
             url,
             aiTag: document.getElementById('watchAi').checked,
+      fetchFiles: document.getElementById('watchFiles').checked,
             intervalHours: Number(document.getElementById('watchInterval').value),
           });
           await loadData();
@@ -2324,6 +2329,9 @@ async function handleWatchAction(btn) {
       if (!confirm('Stop watching this collection?\n\nThe models it already imported are kept.')) { btn.disabled = false; return; }
       await getJson(`/api/watches/${id}`, { method: 'DELETE' });
       await loadData();
+    } else if (action === 'filemode') {
+      const on = btn.textContent.trim() === 'Files: on';
+      await postJson(`/api/watches/${id}`, { fetchFiles: !on }, 'PUT');
     } else if (action === 'files') {
       btn.textContent = 'Getting files…';
       await postJson(`/api/models/${Number(btn.dataset.modelId)}/download-files`);

server.js

@@ -1358,8 +1358,8 @@ app.get('/api/watches/:id', async (req, res, next) => {
 
 app.post('/api/watches', async (req, res, next) => {
   try {
-    const { url, aiTag, intervalHours } = req.body || {};
-    res.status(201).json(await collectionWatch.add({ url: String(url || '').trim(), aiTag, intervalHours }));
+    const { url, aiTag, intervalHours, fetchFiles } = req.body || {};
+    res.status(201).json(await collectionWatch.add({ url: String(url || '').trim(), aiTag, intervalHours, fetchFiles }));
   } catch (e) { next(e); }
 });
 
@@ -1485,7 +1485,10 @@ db.init()
     collectionWatch.init({
       fetchCollection: fetchMakerworldCollection,
       importFromUrl,
-      startAiRun: (modelIds) => aiRuns.createRun(modelIds, { describe: true }),
+      // Same treatment a manual run gives: tags and a description, a clearer name
+      // where the site's one is boilerplate, and collections to review.
+      startAiRun: (modelIds) => aiRuns.createRun(modelIds,
+        { describe: true, renameUnclear: true, suggestCollections: true }),
     });
     collectionWatch.start();
     return aiRuns.init(resolveAiConfig);