Say when MakerWorld is waiting for you to prove you're not a robot

The challenge was detected but treated as an ordinary download failure, so
the message didn't say that only you can clear it. Now it stops the run,
is remembered centrally, and watches skip downloads while it stands —
metadata still imports, and models stay partial for "Get files". The
Watched page shows a banner with a link to MakerWorld and an "I've solved
it" button; a successful download clears it too. Detection also covers a
challenge page returned where JSON was expected.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

lib/collectionWatch.js

@@ -4,6 +4,7 @@
 // The site is read gently: a few models per check, spaced out.
 const db = require('./db');
 const store = require('./store');
+const downloadBlock = require('./downloadBlock');
 const makerworld = require('./importers/makerworld');
 
 const IMPORT_LIMIT = 15;          // models per check, so a big collection trickles in
@@ -184,9 +185,12 @@ async function importFound(watch, collection) {
     if (!live || !live.enabled) break;
     const url = makerworld.designUrl(design);
     try {
+      // While MakerWorld is waiting for you to prove you're not a robot, the
+      // check still brings in the models — their files wait for you.
+      const withFiles = Boolean(watch.fetch_files) && !downloadBlock.isBlocked('makerworld');
       const result = await deps.importFromUrl(url, {
         projects: watch.project_id ? [watch.project_id] : [],
-        withFiles: Boolean(watch.fetch_files), // off by default: MakerWorld refuses quick bursts
+        withFiles, // off by default: MakerWorld refuses quick bursts
       });
       if (result.existing) {
         if (watch.project_id) await store.addModelToProject(result.model.id, watch.project_id);
@@ -196,28 +200,29 @@ async function importFound(watch, collection) {
         fresh.push(result.model.id);
         // Without files (or when they couldn't be fetched) the model is only
         // 'partial' — "Get files" fills it in later.
-        const gotFiles = watch.fetch_files && !result.filesError;
+        const gotFiles = withFiles && !result.filesError;
         await recordItem(watch.id, design, result.model.id, gotFiles ? 'imported' : 'partial',
           result.filesError || null);
       }
       // Refused downloads mean we're asking too fast. Stop the batch and come
       // back shortly, rather than collecting refusals for the whole of it.
-      if (watch.fetch_files && result.filesError) {
+      if (withFiles && result.filesError) {
         fileMisses += 1;
         if (fileMisses >= 2 && result.filesErrorReason !== 'makerworld_token') { stoppedEarly = true; break; }
       } else if (!result.filesError) {
         fileMisses = 0;
       }
-      // A refused token means every later download fails too: stop and say so.
-      if (result.filesErrorReason === 'makerworld_token') {
-        problem = httpError(result.filesError, 502, 'makerworld_token');
+      // A refused token, or a check only you can solve, means every later
+      // download fails too: stop and say so.
+      if (['makerworld_token', 'makerworld_challenge'].includes(result.filesErrorReason)) {
+        problem = httpError(result.filesError, 502, result.filesErrorReason);
         break;
       }
     } catch (err) {
       await recordItem(watch.id, design, null, 'error', err.message);
-      if (err.reason === 'makerworld_token') { problem = err; break; }
+      if (['makerworld_token', 'makerworld_challenge'].includes(err.reason)) { problem = err; break; }
     }
-    await sleep(watch.fetch_files ? BETWEEN_DOWNLOADS_MS : BETWEEN_IMPORTS_MS);
+    await sleep(withFiles ? BETWEEN_DOWNLOADS_MS : BETWEEN_IMPORTS_MS);
   }
 
   const more = pending.length > todo.length || stoppedEarly;

lib/downloadBlock.js

@@ -0,0 +1,29 @@
+// When a site blocks downloads until you prove you're not a robot.
+//
+// MakerWorld does this after a few downloads in a row. Only you can clear it —
+// by solving the check in your own browser — so Printhive stops asking until
+// you say you have. Attempts while blocked are wasted and may prolong it.
+//
+// Held in memory: a restart clears it, and the next attempt finds out again.
+const blocks = new Map(); // site → { since, message }
+
+const DEFAULT = 'MakerWorld is asking you to confirm you’re not a robot before it will allow more downloads.';
+
+function block(site, message = DEFAULT) {
+  if (!blocks.has(site)) blocks.set(site, { since: new Date(), message });
+  return blocks.get(site);
+}
+
+// Called when you say you've solved it, and whenever a download succeeds.
+function clear(site) {
+  return blocks.delete(site);
+}
+
+const isBlocked = (site) => blocks.has(site);
+
+function state(site) {
+  const b = blocks.get(site);
+  return b ? { blocked: true, since: b.since, message: b.message } : { blocked: false };
+}
+
+module.exports = { block, clear, isBlocked, state, DEFAULT };

lib/flaresolverr.js

@@ -51,7 +51,9 @@ async function solveJson(flaresolverrUrl, targetUrl, opts = {}) {
   try {
     return JSON.parse(text);
   } catch {
-    throw new Error('FlareSolverr did not return parseable JSON.');
+    // Keep a little of what came back: a site answering with a page instead of
+    // JSON is usually asking the visitor to prove they're human.
+    throw Object.assign(new Error('FlareSolverr did not return parseable JSON.'), { raw: text.slice(0, 300) });
   }
 }
 

lib/importers/makerworld.js

@@ -31,10 +31,13 @@ function parseInstances(html) {
 // couple of files in a row. It isn't Cloudflare — FlareSolverr reports no
 // challenge, and an unauthenticated call still gets a normal reply — so retrying
 // doesn't clear it; only asking for fewer files does.
-const CHALLENGE_RE = /not a robot|just a moment|captcha|attention required|cf[- ]challenge/i;
+const CHALLENGE_RE = /not a robot|just a moment|captcha|attention required|cf[- ]challenge|verify (?:you|your|that)|verification (?:required|needed)|security check|select in this order/i;
+
+const isChallengeText = (text) => CHALLENGE_RE.test(String(text || ''));
+
 function isChallenge(answer) {
   if (!answer || answer.url) return false;
-  return CHALLENGE_RE.test(String(answer.error || answer.message || answer.msg || ''));
+  return isChallengeText(answer.error || answer.message || answer.msg);
 }
 
 // Resolve an instance to { name, url } (a signed, short-lived CDN download URL).
@@ -121,6 +124,6 @@ function parseLicense(html) {
 }
 
 module.exports = {
-  parseInstances, resolveDownload, downloadProblem, isChallenge, parseLicense,
+  parseInstances, resolveDownload, downloadProblem, isChallenge, isChallengeText, parseLicense,
   parseCollection, parseCollectionDesigns, collectionIdFromUrl, collectionDesignsUrl, designUrl,
 };

public/app.js

@@ -3145,6 +3145,7 @@ function watchPanelHtml() {
   const intervals = [1, 3, 6, 12, 24, 48]
     .map((h) => `<option value="${h}"${h === 6 ? ' selected' : ''}>${h === 1 ? 'every hour' : `every ${h} hours`}</option>`).join('');
   return `
+    <div id="downloadBlock" class="ai-callout is-warning" hidden></div>
     <section class="ai-card">
       <div class="ai-card-head"><h3>Watch a MakerWorld collection</h3></div>
       <p class="form-hint">Printhive checks the collection and imports models added to it, into a collection of the same
@@ -3191,7 +3192,8 @@ function watchHtml(w) {
       </div>
       <div class="watch-meta">${escapeHtml(when)} · every ${w.interval_hours}h · ${plural(w.imported_count, 'model')} imported${w.partial_count ? ` · ${w.partial_count} still without files` : ''}${w.error_count ? ` · ${w.error_count} failed` : ''}${listed}</div>
       ${w.last_error ? `<div class="ai-run-error">${escapeHtml(w.last_error)}
-        ${w.last_error_reason === 'makerworld_token' ? '<button type="button" class="btn btn-primary btn-sm" data-watch="token">Update token</button>' : ''}</div>` : ''}
+        ${w.last_error_reason === 'makerworld_token' ? '<button type="button" class="btn btn-primary btn-sm" data-watch="token">Update token</button>' : ''}
+        ${w.last_error_reason === 'makerworld_challenge' ? '<a href="https://makerworld.com" target="_blank" rel="noopener">Open MakerWorld to solve it</a>' : ''}</div>` : ''}
       ${watchNotices.has(w.id) ? `<div class="ai-run-error">${escapeHtml(watchNotices.get(w.id))}</div>` : ''}
       ${recent.length ? `<div class="watch-recent">${recent.map((i) => `
         <span>${i.model_id
@@ -3211,7 +3213,34 @@ async function renderWatchPanel() {
   fileGrid.className = 'ai-panel';
   resultCount.textContent = '';
   if (!document.getElementById('watchForm')) fileGrid.innerHTML = watchPanelHtml();
-  await refreshWatches();
+  await Promise.all([refreshWatches(), refreshDownloadBlock()]);
+}
+
+// MakerWorld asks you to prove you're not a robot after a few downloads. Only
+// you can answer it, so Printhive stops downloading until you say you have.
+async function refreshDownloadBlock() {
+  const box = document.getElementById('downloadBlock');
+  if (!box) return;
+  let state;
+  try {
+    state = await getJson('/api/downloads/block');
+  } catch {
+    return; // leave whatever's shown; the next refresh tries again
+  }
+  if (!state.blocked) { box.hidden = true; box.innerHTML = ''; return; }
+  box.hidden = false;
+  box.innerHTML = `<strong>MakerWorld has paused downloads.</strong>
+    ${escapeHtml(state.message)} Checks still bring in new models — their files wait for you.
+    <a href="https://makerworld.com" target="_blank" rel="noopener">Open MakerWorld</a>, solve the check, then:
+    <button type="button" class="btn btn-primary btn-sm" id="clearBlockBtn">I’ve solved it</button>`;
+  document.getElementById('clearBlockBtn').addEventListener('click', async () => {
+    try {
+      await postJson('/api/downloads/block/clear');
+      await refreshDownloadBlock();
+    } catch (err) {
+      box.innerHTML += `<span class="ai-run-error">${escapeHtml(err.message)}</span>`;
+    }
+  });
 }
 
 async function refreshWatches() {
@@ -3232,6 +3261,7 @@ async function refreshWatches() {
     : '<p class="form-hint">Nothing watched yet. Paste a MakerWorld collection link above.</p>';
   // Keep the page current even between checks: one may start from the schedule,
   // another tab, or another browser. refreshWatches stops itself when you leave.
+  refreshDownloadBlock();
   const checking = watches.some((w) => w.last_status === 'checking');
   watchPoll = setTimeout(refreshWatches, checking ? 5000 : 20000);
 }

server.js

@@ -36,6 +36,7 @@ const spoolSync = require('./lib/spoolSync');
 
 const printerLive = require('./lib/printerLive');
 const printerCamera = require('./lib/printerCamera');
+const downloadBlock = require('./lib/downloadBlock');
 
 // What a printer has loaded → its card and the Filament page.
 async function syncTrays(printerId, trays) {
@@ -1171,6 +1172,12 @@ async function downloadMakerworldFiles(html, skipNames = new Set()) {
       dl = await makerworld.resolveDownload(flareUrl, inst.id, token);
     } catch (err) {
       console.warn(`MakerWorld instance ${inst.id} resolve failed:`, err.message);
+      // A page where JSON was expected is usually the "prove you're human" check.
+      if (makerworld.isChallengeText(err.raw)) {
+        problems.push({ message: downloadBlock.DEFAULT, challenged: true });
+        downloadBlock.block('makerworld');
+        break;
+      }
       problems.push({ message: `Couldn’t ask MakerWorld for a download link: ${err.message}` });
       continue;
     }
@@ -1178,6 +1185,8 @@ async function downloadMakerworldFiles(html, skipNames = new Set()) {
     if (problem) {
       console.warn(`MakerWorld instance ${inst.id}: ${problem.message}`);
       problems.push(problem);
+      // Only you can clear this one, so stop rather than collect refusals.
+      if (problem.challenged) { downloadBlock.block('makerworld', problem.message); break; }
       continue;
     }
     const original = dl.name || `instance-${inst.id}.3mf`;
@@ -1193,11 +1202,14 @@ async function downloadMakerworldFiles(html, skipNames = new Set()) {
     const filePath = path.join(UPLOAD_DIR, fileName);
     fs.writeFileSync(filePath, Buffer.from(await resp.arrayBuffer()));
     files.push(describeFile(filePath, fileName, original));
+    downloadBlock.clear('makerworld'); // downloads are going through again
   }
   if (!files.length && problems.length) {
     const refused = problems.find((p) => p.tokenRejected);
-    throw Object.assign(new Error((refused || problems[0]).message),
-      { status: 502, reason: refused ? 'makerworld_token' : undefined });
+    const challenged = problems.find((p) => p.challenged);
+    const worst = refused || challenged || problems[0];
+    throw Object.assign(new Error(worst.message),
+      { status: 502, reason: refused ? 'makerworld_token' : challenged ? 'makerworld_challenge' : undefined });
   }
   return files;
 }
@@ -1533,6 +1545,18 @@ async function fetchMakerworldCollection(url) {
   return { ...collection, designs: [...byId.values()] };
 }
 
+// Whether a site is waiting for you to prove you're not a robot, and the
+// button that says you have. Clearing it only lets Printhive try again.
+app.get('/api/downloads/block', (req, res) => {
+  res.set('Cache-Control', 'no-store');
+  res.json(downloadBlock.state('makerworld'));
+});
+
+app.post('/api/downloads/block/clear', (req, res) => {
+  downloadBlock.clear('makerworld');
+  res.json({ message: 'Printhive will try MakerWorld downloads again.' });
+});
+
 app.get('/api/watches', async (req, res, next) => {
   try { res.json(await collectionWatch.list()); } catch (e) { next(e); }
 });

test/downloadBlock.test.js

@@ -0,0 +1,34 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const block = require('../lib/downloadBlock');
+const makerworld = require('../lib/importers/makerworld');
+
+test('a blocked site stays blocked until it’s cleared', () => {
+  assert.equal(block.isBlocked('makerworld'), false);
+  block.block('makerworld', 'Prove you’re not a robot.');
+  assert.equal(block.isBlocked('makerworld'), true);
+  const state = block.state('makerworld');
+  assert.equal(state.blocked, true);
+  assert.equal(state.message, 'Prove you’re not a robot.');
+  assert.ok(state.since instanceof Date);
+  // Blocking again keeps the original message and time.
+  const again = block.block('makerworld', 'something else');
+  assert.equal(again.message, 'Prove you’re not a robot.');
+  block.clear('makerworld');
+  assert.deepEqual(block.state('makerworld'), { blocked: false });
+});
+
+test('MakerWorld’s anti-bot answers are recognised, ordinary refusals aren’t', () => {
+  for (const text of ['Please verify you are not a robot', 'Just a moment…', 'Select in this order',
+    'Security check required', 'Verification required']) {
+    assert.equal(makerworld.isChallengeText(text), true, text);
+  }
+  for (const text of ['Please log in to download models.', 'Model not found', '']) {
+    assert.equal(makerworld.isChallengeText(text), false, text);
+  }
+  // An answer carrying a link is never a challenge.
+  assert.equal(makerworld.isChallenge({ url: 'https://cdn/x.3mf', message: 'captcha' }), false);
+  const problem = makerworld.downloadProblem({ error: 'Please verify you are not a robot' });
+  assert.equal(problem.challenged, true);
+  assert.equal(makerworld.downloadProblem({ error: 'Please log in to download models.' }).tokenRejected, true);
+});