jason/nyaa-crawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: torrent downloads — User-Agent, failure tracking, retry logic

Browse Source 
Problems fixed:
- No User-Agent header: Nyaa returns HTML/redirect for bot requests;
  add browser-like UA + Accept + Referer + redirect:follow
- Silent failures: downloads failed with no visible feedback; add
  'failed' episode status + download_error column to store the error
- No retry: failed/pending episodes were skipped forever on subsequent
  polls; scheduler now retries them via ON CONFLICT upsert
- Content validation: check response starts with 'd' (bencoded dict)
  to catch HTML error pages masquerading as torrents

Also adds:
- /api/health write-test for torrent dir (shows ⚠ banner if not writable)
- /api/poll endpoint for manual immediate poll trigger
- '↻ Poll Now' button in ShowDetailPage
- Failed episode badge with hover tooltip showing the error message
- Migration to widen status CHECK constraint to include 'failed'

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jason
2026-03-17 15:16:25 -05:00
parent 96e7c6e8e5
commit 3f2362a4a4
9 changed files with 272 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/client/api.ts
View File

@@ -17,12 +17,21 @@ export interface Episode {
title: string
torrent_id: string
torrent_url: string
status: 'pending' | 'downloaded_auto' | 'downloaded_manual'
status: 'pending' | 'downloaded_auto' | 'downloaded_manual' | 'failed'
download_error: string | null
file_path: string | null
downloaded_at: string | null
created_at: string
updated_at: string
}
export interface HealthStatus {
status: 'ok' | 'degraded'
torrent_dir: string
torrent_dir_writable: boolean
torrent_dir_error: string | null
}
export interface NyaaItem {
torrent_id: string
title: string
@@ -87,4 +96,8 @@ export const api = {
update: (data: Partial<Omit<Settings, 'torrent_output_dir'>>) =>
request<Settings>('/api/settings', { method: 'PATCH', body: JSON.stringify(data) }),
},
health: {
get: () => request<HealthStatus>('/api/health'),
poll: () => request<{ status: string }>('/api/poll', { method: 'POST' }),
},
}

12
src/client/index.css
View File

@@ -73,6 +73,18 @@ tr:hover td { background: var(--surface2); }
.badge.pending { background: #2a2a1a; color: var(--yellow); }
.badge.downloaded_auto { background: #1a2a1a; color: var(--green); }
.badge.downloaded_manual { background: #1a2533; color: #4da6ff; }
.badge.failed { background: #2a1a1a; color: var(--red); }
.error-tooltip { position: relative; display: inline-flex; align-items: center; gap: 4px; cursor: help; }
.error-tooltip .tooltip-text {
visibility: hidden; opacity: 0; transition: opacity 0.15s;
position: absolute; z-index: 10; bottom: 125%; left: 0;
background: #1a0a0a; color: var(--red); border: 1px solid var(--red);
border-radius: var(--radius); padding: 6px 10px; font-size: 11px;
white-space: pre-wrap; max-width: 360px; min-width: 180px; word-break: break-word;
pointer-events: none;
}
.error-tooltip:hover .tooltip-text { visibility: visible; opacity: 1; }
.layout { display: flex; height: 100vh; }
.sidebar { width: 200px; flex-shrink: 0; background: var(--surface); border-right: 1px solid var(--border); display: flex; flex-direction: column; padding: 16px 0; }

86
src/client/pages/ShowDetailPage.tsx
View File

@@ -1,5 +1,5 @@
import { useEffect, useState } from 'react'
import { api, type Show, type Episode } from '../api'
import { api, type Show, type Episode, type HealthStatus } from '../api'
interface Props {
showId: number
@@ -9,19 +9,21 @@ interface Props {
export default function ShowDetailPage({ showId, onBack }: Props) {
const [show, setShow] = useState<Show | null>(null)
const [episodes, setEpisodes] = useState<Episode[]>([])
const [health, setHealth] = useState<HealthStatus | null>(null)
const [loading, setLoading] = useState(true)
const [bulkCode, setBulkCode] = useState('')
const [bulking, setBulking] = useState(false)
const [polling, setPolling] = useState(false)
useEffect(() => {
Promise.all([api.shows.get(showId), api.episodes.list(showId)])
.then(([s, eps]) => { setShow(s); setEpisodes(eps) })
Promise.all([api.shows.get(showId), api.episodes.list(showId), api.health.get()])
.then(([s, eps, h]) => { setShow(s); setEpisodes(eps); setHealth(h) })
.finally(() => setLoading(false))
}, [showId])
const handleStatusChange = async (ep: Episode, status: Episode['status']) => {
const updated = await api.episodes.update(showId, ep.id, status)
setEpisodes((prev) => prev.map((e) => (e.id === updated.id ? updated : e)))
setEpisodes((prev) => prev.map((e) => (e.id === updated.id ? (updated as Episode) : e)))
}
const handleBulkMark = async (e: React.FormEvent) => {
@@ -44,14 +46,27 @@ export default function ShowDetailPage({ showId, onBack }: Props) {
setShow(updated)
}
const handlePollNow = async () => {
setPolling(true)
try {
await api.health.poll()
const eps = await api.episodes.list(showId)
setEpisodes(eps)
} finally {
setPolling(false)
}
}
if (loading) return <div className="empty-state"><div className="spinner" /></div>
if (!show) return <div className="empty-state">Show not found.</div>
const failed = episodes.filter((e) => e.status === 'failed')
const stats = {
total: episodes.length,
pending: episodes.filter((e) => e.status === 'pending').length,
auto: episodes.filter((e) => e.status === 'downloaded_auto').length,
manual: episodes.filter((e) => e.status === 'downloaded_manual').length,
failed: failed.length,
}
return (
@@ -60,6 +75,19 @@ export default function ShowDetailPage({ showId, onBack }: Props) {
← Shows
</a>
{/* Health warning banner */}
{health && !health.torrent_dir_writable && (
<div className="card" style={{ marginBottom: 16, borderColor: 'var(--red)', background: '#1a0a0a' }}>
<div style={{ color: 'var(--red)', fontWeight: 600, marginBottom: 4 }}>⚠ Torrent directory is not writable</div>
<div style={{ fontSize: 12, color: 'var(--text-muted)' }}>
Path: <code>{health.torrent_dir}</code>
</div>
{health.torrent_dir_error && (
<div style={{ fontSize: 11, color: 'var(--red)', marginTop: 4 }}>{health.torrent_dir_error}</div>
)}
</div>
)}
<div className="page-header">
<div>
<h1>{show.name}</h1>
@@ -71,28 +99,51 @@ export default function ShowDetailPage({ showId, onBack }: Props) {
</span>
</div>
</div>
<div className="flex" style={{ gap: 8 }}>
<button className="ghost" onClick={handlePollNow} disabled={polling}>
{polling ? <span className="spinner" /> : '↻ Poll Now'}
</button>
<button className="ghost" onClick={handleToggleActive}>
{show.is_active ? 'Pause Polling' : 'Resume Polling'}
</button>
</div>
</div>
{/* Stats */}
<div className="flex" style={{ gap: 12, marginBottom: 20, flexWrap: 'wrap' }}>
{[
{ label: 'Total', value: stats.total },
{ label: 'Pending', value: stats.pending },
{ label: 'Auto DL', value: stats.auto },
{ label: 'Manual', value: stats.manual },
].map(({ label, value }) => (
{ label: 'Total', value: stats.total, color: undefined },
{ label: 'Pending', value: stats.pending, color: undefined },
{ label: 'Auto DL', value: stats.auto, color: undefined },
{ label: 'Manual', value: stats.manual, color: undefined },
{ label: 'Failed', value: stats.failed, color: stats.failed > 0 ? 'var(--red)' : undefined },
].map(({ label, value, color }) => (
<div key={label} className="card" style={{ minWidth: 100, textAlign: 'center' }}>
<div style={{ fontSize: 22, fontWeight: 700 }}>{value}</div>
<div style={{ fontSize: 22, fontWeight: 700, color }}>{value}</div>
<div className="text-muted text-sm">{label}</div>
</div>
))}
</div>
{/* Failed downloads summary */}
{failed.length > 0 && (
<div className="card" style={{ marginBottom: 16, borderColor: 'var(--red)' }}>
<div style={{ fontWeight: 600, marginBottom: 8, color: 'var(--red)' }}>
{failed.length} failed download{failed.length > 1 ? 's' : ''} — will retry on next poll (or click ↻ Poll Now)
</div>
{failed.slice(0, 3).map((ep) => (
<div key={ep.id} style={{ fontSize: 12, color: 'var(--text-muted)', marginBottom: 4 }}>
<strong>Ep {ep.episode_code}</strong>: {ep.download_error ?? 'Unknown error'}
</div>
))}
{failed.length > 3 && (
<div style={{ fontSize: 12, color: 'var(--text-muted)' }}>…and {failed.length - 3} more</div>
)}
</div>
)}
{/* Bulk mark */}
<form onSubmit={handleBulkMark} className="card mb-4" style={{ marginBottom: 16 }}>
<form onSubmit={handleBulkMark} className="card" style={{ marginBottom: 16 }}>
<div style={{ marginBottom: 8, fontWeight: 600 }}>Bulk Mark as Downloaded</div>
<div className="flex" style={{ gap: 8 }}>
<input
@@ -136,7 +187,16 @@ export default function ShowDetailPage({ showId, onBack }: Props) {
</a>
</td>
<td className="text-muted">{ep.torrent_id}</td>
<td><span className={`badge ${ep.status}`}>{ep.status.replace('_', ' ')}</span></td>
<td>
{ep.status === 'failed' && ep.download_error ? (
<span className="error-tooltip">
<span className="badge failed">failed</span>
<span className="tooltip-text">{ep.download_error}</span>
</span>
) : (
<span className={`badge ${ep.status}`}>{ep.status.replace(/_/g, ' ')}</span>
)}
</td>
<td className="text-muted text-sm">
{ep.downloaded_at ? new Date(ep.downloaded_at).toLocaleDateString() : '—'}
</td>
@@ -169,7 +229,7 @@ function EpisodeActions({
if (busy) return <span className="spinner" />
if (ep.status === 'pending') {
if (ep.status === 'pending' || ep.status === 'failed') {
return (
<button style={{ fontSize: 12 }} onClick={() => handle('downloaded_manual')}>
Mark Downloaded

65
src/server/db/index.ts
View File

@@ -17,6 +17,7 @@ db.exec("PRAGMA journal_mode = WAL")
db.exec("PRAGMA foreign_keys = ON")
export function runMigrations() {
// Base schema
db.exec(`
CREATE TABLE IF NOT EXISTS shows (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -38,7 +39,7 @@ export function runMigrations() {
torrent_id TEXT NOT NULL,
torrent_url TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending'
CHECK(status IN ('pending','downloaded_auto','downloaded_manual')),
CHECK(status IN ('pending','downloaded_auto','downloaded_manual','failed')),
downloaded_at TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
@@ -55,5 +56,67 @@ export function runMigrations() {
('default_quality', '1080p'),
('default_sub_group', '');
`)
// Incremental migrations — safe to re-run (errors are swallowed for "already exists")
runAlterIfMissing("ALTER TABLE episodes ADD COLUMN download_error TEXT")
runAlterIfMissing("ALTER TABLE episodes ADD COLUMN file_path TEXT")
// Widen the status CHECK constraint to include 'failed' for existing DBs.
// SQLite doesn't support ALTER COLUMN, so we recreate the table if the old
// constraint is still in place (detected by absence of 'failed' in the schema).
widenEpisodesStatusConstraint()
console.log('[db] Migrations applied.')
}
function runAlterIfMissing(sql: string) {
try {
db.exec(sql)
} catch {
// Column already exists — ignore
}
}
function widenEpisodesStatusConstraint() {
const schema = db
.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='episodes'")
.get() as { sql: string } | undefined
if (!schema) return
if (schema.sql.includes("'failed'")) return // already widened
// Recreate episodes table with the updated CHECK constraint
db.exec(`
PRAGMA foreign_keys = OFF;
CREATE TABLE episodes_new (
id INTEGER PRIMARY KEY AUTOINCREMENT,
show_id INTEGER NOT NULL REFERENCES shows(id) ON DELETE CASCADE,
episode_code TEXT NOT NULL,
title TEXT NOT NULL,
torrent_id TEXT NOT NULL,
torrent_url TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending'
CHECK(status IN ('pending','downloaded_auto','downloaded_manual','failed')),
download_error TEXT,
file_path TEXT,
downloaded_at TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE(show_id, torrent_id)
);
INSERT INTO episodes_new
(id, show_id, episode_code, title, torrent_id, torrent_url, status,
download_error, file_path, downloaded_at, created_at, updated_at)
SELECT
id, show_id, episode_code, title, torrent_id, torrent_url, status,
NULL, NULL, downloaded_at, created_at, updated_at
FROM episodes;
DROP TABLE episodes;
ALTER TABLE episodes_new RENAME TO episodes;
PRAGMA foreign_keys = ON;
`)
}

23
src/server/index.ts
View File

@@ -1,7 +1,8 @@
import express from 'express'
import path from 'path'
import { runMigrations } from './db/index.js'
import { startScheduler } from './services/scheduler.js'
import { startScheduler, pollAllShows } from './services/scheduler.js'
import { getTorrentDir, isTorrentDirWritable } from './services/downloader.js'
import showsRouter from './routes/shows.js'
import episodesRouter from './routes/episodes.js'
import nyaaRouter from './routes/nyaa.js'
@@ -18,8 +19,24 @@ app.use('/api/shows/:showId/episodes', episodesRouter)
app.use('/api/nyaa', nyaaRouter)
app.use('/api/settings', settingsRouter)
// Health check
app.get('/api/health', (_req, res) => res.json({ status: 'ok' }))
// Health check — includes torrent dir write test
app.get('/api/health', (_req, res) => {
const torrentDir = getTorrentDir()
const writeTest = isTorrentDirWritable()
res.json({
status: writeTest.ok ? 'ok' : 'degraded',
torrent_dir: torrentDir,
torrent_dir_writable: writeTest.ok,
torrent_dir_error: writeTest.error ?? null,
})
})
// Trigger an immediate poll of all active shows
app.post('/api/poll', (_req, res) => {
pollAllShows()
.then(() => res.json({ status: 'ok' }))
.catch((err: unknown) => res.status(500).json({ error: String(err) }))
})
// Serve static client build in production
const clientDist = path.join(__dirname, '../client')

2
src/server/routes/episodes.ts
View File

@@ -23,7 +23,7 @@ router.patch('/:id', (req: Request<EpParams>, res) => {
if (!ep) return res.status(404).json({ error: 'Episode not found' })
const { status } = req.body as { status?: string }
const allowed = ['pending', 'downloaded_auto', 'downloaded_manual']
const allowed = ['pending', 'downloaded_auto', 'downloaded_manual', 'failed']
if (!status || !allowed.includes(status)) {
return res.status(400).json({ error: `status must be one of: ${allowed.join(', ')}` })
}

39
src/server/services/downloader.ts
View File

@@ -7,6 +7,21 @@ export function getTorrentDir(): string {
return TORRENT_DIR
}
/** Returns true if the torrent output directory exists and is writable. */
export function isTorrentDirWritable(): { ok: boolean; error?: string } {
try {
if (!fs.existsSync(TORRENT_DIR)) {
fs.mkdirSync(TORRENT_DIR, { recursive: true })
}
const testFile = path.join(TORRENT_DIR, '.write-test')
fs.writeFileSync(testFile, 'ok')
fs.unlinkSync(testFile)
return { ok: true }
} catch (err) {
return { ok: false, error: String(err) }
}
}
function ensureTorrentDir() {
if (!fs.existsSync(TORRENT_DIR)) {
fs.mkdirSync(TORRENT_DIR, { recursive: true })
@@ -32,9 +47,29 @@ export async function downloadTorrent(
const timeout = setTimeout(() => controller.abort(), 30_000)
try {
const res = await fetch(torrentUrl, { signal: controller.signal })
if (!res.ok) throw new Error(`Failed to download torrent: ${res.status} ${res.statusText}`)
const res = await fetch(torrentUrl, {
signal: controller.signal,
headers: {
// Nyaa blocks requests without a browser-like User-Agent
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
'Accept': 'application/x-bittorrent, application/octet-stream, */*',
'Referer': 'https://nyaa.si/',
},
redirect: 'follow',
})
if (!res.ok) {
throw new Error(`HTTP ${res.status} ${res.statusText} — URL: ${torrentUrl}`)
}
// Sanity check: torrent files start with 'd' (bencoded dict)
const buf = await res.arrayBuffer()
const bytes = new Uint8Array(buf)
if (bytes.length < 10 || bytes[0] !== 0x64 /* 'd' */) {
const preview = Buffer.from(buf).toString('utf8', 0, 120).replace(/\n/g, ' ')
throw new Error(`Response does not look like a torrent file (got: ${preview}…)`)
}
fs.writeFileSync(filePath, Buffer.from(buf))
} finally {
clearTimeout(timeout)

60
src/server/services/scheduler.ts
View File

@@ -69,20 +69,30 @@ async function pollShow(show: Show) {
return
}
// Filter out batches and already-tracked items, sort oldest→newest
// Separate brand-new items from previously-failed/pending ones that need a retry
const retryableEpisodes = db
.prepare(`SELECT * FROM episodes WHERE show_id = ? AND status IN ('pending', 'failed')`)
.all(show.id) as unknown as Episode[]
const retryIds = new Set(retryableEpisodes.map((e) => e.torrent_id))
const newItems = items
.filter((item) => {
if (isBatchRelease(item.title)) return false
const existing = db
.prepare('SELECT id FROM episodes WHERE show_id = ? AND torrent_id = ?')
.get(show.id, item.torrent_id) as Episode | undefined
return !existing
.prepare('SELECT id, status FROM episodes WHERE show_id = ? AND torrent_id = ?')
.get(show.id, item.torrent_id) as { id: number; status: string } | undefined
// Include if: never seen, or previously failed/pending
return !existing || existing.status === 'pending' || existing.status === 'failed'
})
.sort((a, b) => episodeNum(parseEpisodeCode(a.title)) - episodeNum(parseEpisodeCode(b.title)))
if (newItems.length === 0) return
if (newItems.length === 0 && retryIds.size === 0) return
if (newItems.length > 0) {
console.log(`[scheduler] Found ${newItems.length} new/retryable episode(s) for "${show.name}"`)
}
console.log(`[scheduler] Found ${newItems.length} new episode(s) for "${show.name}"`)
await downloadItems(show, newItems)
}
@@ -96,31 +106,51 @@ export async function downloadItems(show: Show, items: NyaaItem[]) {
for (const item of items) {
const episodeCode = parseEpisodeCode(item.title)
// Insert as pending (idempotent)
const insertResult = db
// Upsert: insert new as pending, or reset a failed/pending row so we can retry it
db
.prepare(`
INSERT OR IGNORE INTO episodes
INSERT INTO episodes
(show_id, episode_code, title, torrent_id, torrent_url, status, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, 'pending', datetime('now'), datetime('now'))
ON CONFLICT(show_id, torrent_id) DO UPDATE
SET status = 'pending',
download_error = NULL,
updated_at = datetime('now')
WHERE status IN ('failed', 'pending')
`)
.run(show.id, episodeCode, item.title, item.torrent_id, item.torrent_url)
if (insertResult.changes === 0) continue
// Fetch the episode id (whether just inserted or updated)
const ep = db
.prepare(`SELECT id FROM episodes WHERE show_id = ? AND torrent_id = ? AND status = 'pending'`)
.get(show.id, item.torrent_id) as { id: number } | undefined
const epId = insertResult.lastInsertRowid
if (!ep) continue // already downloaded_auto or downloaded_manual — skip
// Download the .torrent file
try {
console.log(`[scheduler] Downloading torrent for "${item.title}"`)
await downloadTorrent(item.torrent_url, slug, episodeCode, item.torrent_id)
const filePath = await downloadTorrent(item.torrent_url, slug, episodeCode, item.torrent_id)
db.prepare(`
UPDATE episodes
SET status = 'downloaded_auto', downloaded_at = datetime('now'), updated_at = datetime('now')
SET status = 'downloaded_auto',
file_path = ?,
downloaded_at = datetime('now'),
download_error = NULL,
updated_at = datetime('now')
WHERE id = ?
`).run(epId)
`).run(filePath, ep.id)
console.log(`[scheduler] Downloaded: ${item.title}`)
} catch (err) {
console.error(`[scheduler] Download failed for "${item.title}":`, err)
const msg = err instanceof Error ? err.message : String(err)
console.error(`[scheduler] Download failed for "${item.title}": ${msg}`)
db.prepare(`
UPDATE episodes
SET status = 'failed',
download_error = ?,
updated_at = datetime('now')
WHERE id = ?
`).run(msg, ep.id)
}
}
}

4
src/server/types.ts
View File

@@ -17,7 +17,9 @@ export interface Episode {
title: string
torrent_id: string
torrent_url: string
status: 'pending' | 'downloaded_auto' | 'downloaded_manual'
status: 'pending' | 'downloaded_auto' | 'downloaded_manual' | 'failed'
download_error: string | null
file_path: string | null
downloaded_at: string | null
created_at: string
updated_at: string