Compare commits

..

2 commits

Author SHA1 Message Date
dlawler489
1f22dd0189 Merge visible AI run failures and retry for finished runs
All checks were successful
build-and-push / docker (push) Successful in 27s
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-16 14:08:08 +10:00
dlawler489
b0054876ff Name every model an AI run failed on, and let a finished run retry them
A run's detail only carried the last dozen items, so a model that failed early
scrolled out of reach: run 21 reported "1 failed" with no way to learn which
model or why. The detail now carries every failed item, listed in full.

"Retry failed" existed but could never appear: it needs a run that is finished
with failures, and such a run is 'done', which neither canResume nor resume()
accepted. Both now do; resume() already no-ops when nothing is pending, so a
clean run is unaffected.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-16 14:08:08 +10:00
3 changed files with 23 additions and 3 deletions

View file

@ -243,7 +243,13 @@ async function getRun(id) {
`SELECT i.model_id, m.name, i.status, i.tags, i.error, i.renamed_from, i.renamed_to, i.collections `SELECT i.model_id, m.name, i.status, i.tags, i.error, i.renamed_from, i.renamed_to, i.collections
FROM ai_run_items i LEFT JOIN models m ON m.id = i.model_id FROM ai_run_items i LEFT JOIN models m ON m.id = i.model_id
WHERE i.run_id = $1 AND i.status <> 'pending' ORDER BY i.updated_at DESC LIMIT 12`, [id]); WHERE i.run_id = $1 AND i.status <> 'pending' ORDER BY i.updated_at DESC LIMIT 12`, [id]);
return { ...shapeRun(run), recent }; // Every failure, not only the ones still inside that window: a model that
// failed early would otherwise scroll out of sight with no way to find it.
const { rows: failures } = await db.query(
`SELECT i.model_id, m.name, i.error
FROM ai_run_items i LEFT JOIN models m ON m.id = i.model_id
WHERE i.run_id = $1 AND i.status = 'error' ORDER BY i.updated_at DESC LIMIT 200`, [id]);
return { ...shapeRun(run), recent, failures };
} }
async function stop(id) { async function stop(id) {
@ -257,7 +263,7 @@ async function stop(id) {
async function resume(id) { async function resume(id) {
const resumed = await db.transaction(async (client) => { const resumed = await db.transaction(async (client) => {
const { rows: [run] } = await client.query( const { rows: [run] } = await client.query(
`SELECT id FROM ai_runs WHERE id = $1 AND status IN ('paused', 'stopped', 'failed') FOR UPDATE`, [id]); `SELECT id FROM ai_runs WHERE id = $1 AND status IN ('paused', 'stopped', 'failed', 'done') FOR UPDATE`, [id]);
if (!run) return false; if (!run) return false;
const retried = await client.query( const retried = await client.query(
`UPDATE ai_run_items SET status = 'pending', error = NULL WHERE run_id = $1 AND status = 'error'`, [id]); `UPDATE ai_run_items SET status = 'pending', error = NULL WHERE run_id = $1 AND status = 'error'`, [id]);

View file

@ -1420,7 +1420,8 @@ function runHtml(run, detail) {
const options = run.options || {}; const options = run.options || {};
const cost = options.pricesKnown ? money(run.cost_usd) : 'cost unknown (add prices in Settings)'; const cost = options.pricesKnown ? money(run.cost_usd) : 'cost unknown (add prices in Settings)';
const canStop = run.status === 'running' || run.status === 'paused'; const canStop = run.status === 'running' || run.status === 'paused';
const canResume = ['paused', 'stopped', 'failed'].includes(run.status) && (processed < run.total || run.failed > 0); // 'done' too: a run that finished with failures still has something to retry.
const canResume = ['paused', 'stopped', 'failed', 'done'].includes(run.status) && (processed < run.total || run.failed > 0);
const feed = detail && detail.recent.length ? ` const feed = detail && detail.recent.length ? `
<ul class="ai-feed">${detail.recent.map((i) => ` <ul class="ai-feed">${detail.recent.map((i) => `
<li> <li>
@ -1432,6 +1433,17 @@ function runHtml(run, detail) {
: `<span class="ai-run-error">${escapeHtml(i.error || 'failed')}</span>`} : `<span class="ai-run-error">${escapeHtml(i.error || 'failed')}</span>`}
</li>`).join('')} </li>`).join('')}
</ul>` : ''; </ul>` : '';
// Failures are listed in full: the feed above only holds the last dozen items,
// so anything that failed early would otherwise be impossible to track down.
const failed = detail && (detail.failures || []).length ? `
<details class="ai-failures">
<summary>${plural(detail.failures.length, 'model')} failed</summary>
<ul class="ai-feed">${detail.failures.map((i) => `
<li>
<button type="button" class="link-btn" data-open-model="${i.model_id}">${escapeHtml(i.name || `Model ${i.model_id}`)}</button>
<span class="ai-run-error">${escapeHtml(i.error || 'failed')}</span>
</li>`).join('')}</ul>
</details>` : '';
return ` return `
<div class="ai-run"> <div class="ai-run">
<div class="ai-run-head"> <div class="ai-run-head">

View file

@ -464,6 +464,8 @@ label { display: block; font-size: 13px; font-weight: 500; color: var(--text-mut
.watch-head { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 12px; font-size: 14px; } .watch-head { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 12px; font-size: 14px; }
.watch-actions { margin-left: auto; display: flex; flex-wrap: wrap; gap: 8px; } .watch-actions { margin-left: auto; display: flex; flex-wrap: wrap; gap: 8px; }
.watch-actions .btn-sm { align-self: auto; } .watch-actions .btn-sm { align-self: auto; }
.ai-failures summary { cursor: pointer; font-size: 13px; color: var(--text-muted); }
.ai-failures summary:hover { color: var(--text); }
.watch-pending summary { cursor: pointer; font-size: 13px; color: var(--text-muted); } .watch-pending summary { cursor: pointer; font-size: 13px; color: var(--text-muted); }
.watch-pending summary:hover { color: var(--text); } .watch-pending summary:hover { color: var(--text); }
.watch-pending .watch-recent { margin-top: 6px; } .watch-pending .watch-recent { margin-top: 6px; }