diff --git a/db/schema.sql b/db/schema.sql
index d86de6c..94e317d 100644
--- a/db/schema.sql
+++ b/db/schema.sql
@@ -235,6 +235,7 @@ CREATE TABLE IF NOT EXISTS watched_collections (
title TEXT NOT NULL DEFAULT '', -- its name there, refreshed on each check
project_id INTEGER REFERENCES projects(id) ON DELETE SET NULL, -- where new models land
ai_tag BOOLEAN NOT NULL DEFAULT false,
+ fetch_files BOOLEAN NOT NULL DEFAULT false, -- download files during a check, not just metadata
interval_hours INTEGER NOT NULL DEFAULT 6,
enabled BOOLEAN NOT NULL DEFAULT true,
last_checked_at TIMESTAMPTZ,
@@ -260,3 +261,5 @@ CREATE TABLE IF NOT EXISTS watched_collection_items (
seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (watch_id, design_id)
);
+
+ALTER TABLE watched_collections ADD COLUMN IF NOT EXISTS fetch_files BOOLEAN NOT NULL DEFAULT false;
diff --git a/lib/collectionWatch.js b/lib/collectionWatch.js
index 5cce293..39033ac 100644
--- a/lib/collectionWatch.js
+++ b/lib/collectionWatch.js
@@ -8,6 +8,9 @@ const makerworld = require('./importers/makerworld');
const IMPORT_LIMIT = 15; // models per check, so a big collection trickles in
const BETWEEN_IMPORTS_MS = 3000; // FlareSolverr runs one browser request at a time
+// Downloads need much more room: MakerWorld refuses bursts a few seconds apart,
+// but 67 in a row about 20 seconds apart went through without one refusal.
+const BETWEEN_DOWNLOADS_MS = 20000;
const MORE_SOON_MINUTES = 10; // when a check hit the limit, come back sooner
const TICK_MS = 5 * 60 * 1000;
const INTERVALS = [1, 3, 6, 12, 24, 48];
@@ -83,7 +86,7 @@ async function get(id) {
// Adds a watch. Reads the collection first, so a bad link or a refused token is
// reported now rather than silently later.
-async function add({ url, aiTag = false, intervalHours = 6 }) {
+async function add({ url, aiTag = false, intervalHours = 6, fetchFiles = false }) {
const sourceId = makerworld.collectionIdFromUrl(url);
if (!sourceId) throw httpError('That isn’t a MakerWorld collection link (makerworld.com/…/collections/…).', 400);
const { rowCount } = await db.query('SELECT 1 FROM watched_collections WHERE site = $1 AND source_id = $2', ['makerworld', sourceId]);
@@ -93,9 +96,10 @@ async function add({ url, aiTag = false, intervalHours = 6 }) {
const project = await store.findOrCreateProject(collection.title || `MakerWorld collection ${sourceId}`);
const hours = INTERVALS.includes(Number(intervalHours)) ? Number(intervalHours) : 6;
const { rows: [watch] } = await db.query(
- `INSERT INTO watched_collections (site, source_id, url, title, project_id, ai_tag, interval_hours, total, listed, last_status)
- VALUES ('makerworld', $1, $2, $3, $4, $5, $6, $7, $8, 'checking') RETURNING id`,
- [sourceId, url, collection.title, project.id, Boolean(aiTag), hours, collection.total, collection.designs.length]);
+ `INSERT INTO watched_collections (site, source_id, url, title, project_id, ai_tag, interval_hours, total, listed, last_status, fetch_files)
+ VALUES ('makerworld', $1, $2, $3, $4, $5, $6, $7, $8, 'checking', $9) RETURNING id`,
+ [sourceId, url, collection.title, project.id, Boolean(aiTag), hours, collection.total, collection.designs.length,
+ Boolean(fetchFiles)]);
// The collection was just read, so import straight from it rather than
// fetching the whole thing again.
const added = await get(watch.id);
@@ -106,16 +110,18 @@ async function add({ url, aiTag = false, intervalHours = 6 }) {
return added;
}
-async function update(id, { enabled, aiTag, intervalHours }) {
+async function update(id, { enabled, aiTag, intervalHours, fetchFiles }) {
const hours = INTERVALS.includes(Number(intervalHours)) ? Number(intervalHours) : null;
const { rowCount } = await db.query(
`UPDATE watched_collections SET
enabled = COALESCE($2, enabled),
ai_tag = COALESCE($3, ai_tag),
interval_hours = COALESCE($4, interval_hours),
+ fetch_files = COALESCE($5, fetch_files),
next_check_at = CASE WHEN $2 IS TRUE THEN now() ELSE next_check_at END
WHERE id = $1`,
- [id, enabled === undefined ? null : Boolean(enabled), aiTag === undefined ? null : Boolean(aiTag), hours]);
+ [id, enabled === undefined ? null : Boolean(enabled), aiTag === undefined ? null : Boolean(aiTag), hours,
+ fetchFiles === undefined ? null : Boolean(fetchFiles)]);
if (!rowCount) return null;
if (enabled) runDue().catch(() => {});
return get(id);
@@ -169,6 +175,8 @@ async function importFound(watch, collection) {
let added = 0;
let problem = null;
+ let fileMisses = 0; // downloads refused in a row
+ let stoppedEarly = false; // left models unimported, so come back soon
const fresh = [];
for (const design of todo) {
// Paused while this batch was running: stop, rather than import the rest.
@@ -178,7 +186,7 @@ async function importFound(watch, collection) {
try {
const result = await deps.importFromUrl(url, {
projects: watch.project_id ? [watch.project_id] : [],
- withFiles: false, // MakerWorld allows only a couple of downloads in a row
+ withFiles: Boolean(watch.fetch_files), // off by default: MakerWorld refuses quick bursts
});
if (result.existing) {
if (watch.project_id) await store.addModelToProject(result.model.id, watch.project_id);
@@ -186,9 +194,19 @@ async function importFound(watch, collection) {
} else {
added += 1;
fresh.push(result.model.id);
- // A check brings in the page, not the files, so every new model starts
- // 'partial' — "Get files" on the ones you want fills them in.
- await recordItem(watch.id, design, result.model.id, 'partial', result.filesError || null);
+ // Without files (or when they couldn't be fetched) the model is only
+ // 'partial' — "Get files" fills it in later.
+ const gotFiles = watch.fetch_files && !result.filesError;
+ await recordItem(watch.id, design, result.model.id, gotFiles ? 'imported' : 'partial',
+ result.filesError || null);
+ }
+ // Refused downloads mean we're asking too fast. Stop the batch and come
+ // back shortly, rather than collecting refusals for the whole of it.
+ if (watch.fetch_files && result.filesError) {
+ fileMisses += 1;
+ if (fileMisses >= 2 && result.filesErrorReason !== 'makerworld_token') { stoppedEarly = true; break; }
+ } else if (!result.filesError) {
+ fileMisses = 0;
}
// A refused token means every later download fails too: stop and say so.
if (result.filesErrorReason === 'makerworld_token') {
@@ -199,10 +217,10 @@ async function importFound(watch, collection) {
await recordItem(watch.id, design, null, 'error', err.message);
if (err.reason === 'makerworld_token') { problem = err; break; }
}
- await sleep(BETWEEN_IMPORTS_MS);
+ await sleep(watch.fetch_files ? BETWEEN_DOWNLOADS_MS : BETWEEN_IMPORTS_MS);
}
- const more = pending.length > todo.length;
+ const more = pending.length > todo.length || stoppedEarly;
if (problem) {
await recordFailure({ ...watch, id: watch.id }, problem);
} else {
diff --git a/public/app.js b/public/app.js
index 398bc12..23e0add 100644
--- a/public/app.js
+++ b/public/app.js
@@ -2183,12 +2183,14 @@ function watchPanelHtml() {
Watch a MakerWorld collection
Printhive checks the collection and imports models added to it, into a collection of the same
name here. Private collections work too, using your MakerWorld token. Models you remove there stay here.
- A check brings in each model's page — name, description, picture, licence — but not its files, because MakerWorld
- only allows a couple of downloads in a row; use “Get files” on the ones you want.
+ A check brings in each model's page — name, description, picture, licence. Files are left for you to fetch with
+ “Get files”, unless you switch downloads on: MakerWorld refuses quick bursts, so a check that downloads spaces
+ them about 20 seconds apart and takes correspondingly longer.
@@ -2217,6 +2219,7 @@ function watchHtml(w) {
+
@@ -2279,6 +2282,7 @@ async function handleAddWatch(e) {
const watch = await postJson('/api/watches', {
url,
aiTag: document.getElementById('watchAi').checked,
+ fetchFiles: document.getElementById('watchFiles').checked,
intervalHours: Number(document.getElementById('watchInterval').value),
});
document.getElementById('watchUrl').value = '';
@@ -2294,6 +2298,7 @@ async function handleAddWatch(e) {
const watch = await postJson('/api/watches', {
url,
aiTag: document.getElementById('watchAi').checked,
+ fetchFiles: document.getElementById('watchFiles').checked,
intervalHours: Number(document.getElementById('watchInterval').value),
});
await loadData();
@@ -2324,6 +2329,9 @@ async function handleWatchAction(btn) {
if (!confirm('Stop watching this collection?\n\nThe models it already imported are kept.')) { btn.disabled = false; return; }
await getJson(`/api/watches/${id}`, { method: 'DELETE' });
await loadData();
+ } else if (action === 'filemode') {
+ const on = btn.textContent.trim() === 'Files: on';
+ await postJson(`/api/watches/${id}`, { fetchFiles: !on }, 'PUT');
} else if (action === 'files') {
btn.textContent = 'Getting files…';
await postJson(`/api/models/${Number(btn.dataset.modelId)}/download-files`);
diff --git a/server.js b/server.js
index 2cfa59a..a6f0573 100644
--- a/server.js
+++ b/server.js
@@ -1358,8 +1358,8 @@ app.get('/api/watches/:id', async (req, res, next) => {
app.post('/api/watches', async (req, res, next) => {
try {
- const { url, aiTag, intervalHours } = req.body || {};
- res.status(201).json(await collectionWatch.add({ url: String(url || '').trim(), aiTag, intervalHours }));
+ const { url, aiTag, intervalHours, fetchFiles } = req.body || {};
+ res.status(201).json(await collectionWatch.add({ url: String(url || '').trim(), aiTag, intervalHours, fetchFiles }));
} catch (e) { next(e); }
});
@@ -1485,7 +1485,10 @@ db.init()
collectionWatch.init({
fetchCollection: fetchMakerworldCollection,
importFromUrl,
- startAiRun: (modelIds) => aiRuns.createRun(modelIds, { describe: true }),
+ // Same treatment a manual run gives: tags and a description, a clearer name
+ // where the site's one is boilerplate, and collections to review.
+ startAiRun: (modelIds) => aiRuns.createRun(modelIds,
+ { describe: true, renameUnclear: true, suggestCollections: true }),
});
collectionWatch.start();
return aiRuns.init(resolveAiConfig);