const path = require("path"); const { fetchChapterLinks, isLikelyComicLandSite } = require("./chapters.js"); const { stemForComicLandChapterCbz, chapterUrlToCbz, sanitizeComicDirName } = require("./downloadChapterCbz.js"); /** * @typedef {object} ParsedCli * @property {boolean} [help] - Print usage and exit when true. * @property {string|undefined} listingUrl - Series listing URL (required unless help). * @property {boolean} listOnly - Only print the chapter menu. * @property {boolean} allChapters - Queue every chapter (API already oldest-first). * @property {boolean} missingOnly - Queue chapters that need download (SQLite + CBZ on disk). * @property {string|undefined} selectSpec - `--select` string (1-based menu indices). * @property {string|undefined} numbersSpec - `--numbers` string (chapter numbers from titles). * @property {string} outDir - Output directory for `.cbz` files. * @property {boolean} noDb - Skip SQLite updates. * @property {number|undefined} titleId - Load listing URL from SQLite title id. * @property {boolean} outDirExplicit - True when `--out` was passed. * @property {boolean} verbose - Print informational progress logs. * @property {number} delaySeconds - Pause between chapter downloads (0 = no delay). */ /** * @param {string[]} argv - `process.argv`. * @returns {ParsedCli} Parsed flags and positional URL. */ function parseCli(argv) { /** @type {ParsedCli} */ const result = { help: false, listingUrl: undefined, listOnly: false, allChapters: false, missingOnly: false, selectSpec: undefined, numbersSpec: undefined, outDir: path.join(process.cwd(), "data", "comicland", "comics"), noDb: false, titleId: undefined, outDirExplicit: false, verbose: false, delaySeconds: 0, }; for (let i = 2; i < argv.length; i += 1) { const a = argv[i]; if (a === "--help" || a === "-h") { result.help = true; } else if (a === "--list-only" || a === "-l") { result.listOnly = true; } else if (a === "--all") { result.allChapters = true; } else if (a === "--missing") { result.missingOnly = true; } else if (a === "--verbose" || a === "-v") { result.verbose = true; } else if (a === "--no-db") { result.noDb = true; } else if (a === "--delay") { const next = argv[i + 1]; if (next === undefined || (next.startsWith("-") && !/^-?\d+$/u.test(next))) { throw new Error("Missing value for --delay"); } const seconds = Number.parseInt(next, 10); if (!Number.isFinite(seconds) || seconds < 0) { throw new Error(`Invalid --delay: ${next} (expected non-negative integer seconds)`); } result.delaySeconds = seconds; i += 1; } else if (a === "--title-id") { const next = argv[i + 1]; if (next === undefined || next.startsWith("-")) { throw new Error("Missing value for --title-id"); } const id = Number.parseInt(next, 10); if (!Number.isFinite(id) || id < 1) { throw new Error(`Invalid --title-id: ${next}`); } result.titleId = id; i += 1; } else if (a === "--select" || a === "-s") { const next = argv[i + 1]; if (next === undefined || next.startsWith("-")) { throw new Error("Missing value for --select"); } result.selectSpec = next; i += 1; } else if (a === "--numbers" || a === "-n") { const next = argv[i + 1]; if (next === undefined || next.startsWith("-")) { throw new Error("Missing value for --numbers"); } result.numbersSpec = next; i += 1; } else if (a === "--out" || a === "-o") { const next = argv[i + 1]; if (next === undefined || next.startsWith("-")) { throw new Error("Missing value for --out"); } result.outDir = path.resolve(next); result.outDirExplicit = true; i += 1; } else if (!a.startsWith("-")) { if (result.listingUrl) { throw new Error(`Unexpected extra argument: ${a}`); } result.listingUrl = a; } else { throw new Error(`Unknown option: ${a}`); } } return result; } /** * @param {boolean} verbose - When false, no-op. * @returns {(msg: string) => void} Logger. */ function makeVlog(verbose) { return (msg) => { if (verbose) { console.log(`info: ${msg}`); } }; } /** * @param {number} seconds - Seconds to wait. * @returns {Promise} */ function sleepSeconds(seconds) { return new Promise((resolve) => { setTimeout(resolve, seconds * 1000); }); } /** * Build a download queue from SQLite chapters that still need a CBZ. * @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first listing. * @param {object[]} needingRows - Chapter rows from SQLite. * @returns {Array<{ url: string, title: string, pickLabel: string }>} Queue items. */ function queueItemsMissing(chapters, needingRows) { /** @type {Map} */ const byUrl = new Map(); for (const row of needingRows) { if (typeof row.url === "string" && row.url) { byUrl.set(row.url, row); } } /** @type {Array<{ url: string, title: string, pickLabel: string }>} */ const items = []; /** @type {Set} */ const used = new Set(); for (const ch of chapters) { const row = byUrl.get(ch.url); if (!row) { continue; } used.add(ch.url); const reason = row.status === "downloaded" ? "file-missing" : String(row.status); items.push({ url: ch.url, title: ch.title, pickLabel: `missing:${reason}`, }); } for (const row of needingRows) { if (typeof row.url !== "string" || !row.url || used.has(row.url)) { continue; } const reason = row.status === "downloaded" ? "file-missing" : String(row.status); items.push({ url: row.url, title: row.label || row.chapter_key || row.url, pickLabel: `missing:${reason}`, }); } return items; } /** * Chapters that need a download (status pending/failed, or downloaded CBZ gone). * @param {import("better-sqlite3").Database} db - Database handle. * @param {number} titleId - Title id. * @param {(path: string|null|undefined) => boolean} cbzExistsFn - File check. * @returns {object[]} Chapter rows. */ function listChaptersMissing(db, titleId, cbzExistsFn) { const dbApi = require("../server/db.js"); const rows = dbApi.listChapters(db, titleId); return rows.filter((ch) => { if (ch.status === "pending" || ch.status === "failed") { return true; } if (ch.status === "downloaded" && !cbzExistsFn(ch.cbz_path)) { return true; } return false; }); } /** * ComicLand API already returns oldest-first; copy so callers can mutate safely. * @param {import("./chapters.js").ChapterLink[]} chapters - Site/API order. * @returns {import("./chapters.js").ChapterLink[]} New array, oldest chapter first. */ function chaptersInReadingOrder(chapters) { return [...chapters]; } /** * @param {string} spec - `--select` value. * @param {number} count - Chapter count. * @returns {number[]} Sorted unique 1-based row indices. */ function parseChapterSelection(spec, count) { if (count <= 0) { throw new Error("Chapter list is empty; cannot select indices."); } const trimmed = spec.trim(); if (trimmed.length === 0) { throw new Error("--select value is empty."); } /** @type {Set} */ const seen = new Set(); for (const rawPart of trimmed.split(",")) { const part = rawPart.trim(); if (part.length === 0) { continue; } if (/^\d+$/u.test(part)) { const n = Number.parseInt(part, 10); if (n < 1 || n > count) { throw new Error( `Index ${String(n)} is out of range (valid: 1–${String(count)}).`, ); } seen.add(n); continue; } const rangeMatch = /^(\d+)\s*-\s*(\d+)$/u.exec(part); if (rangeMatch) { const lo = Number.parseInt(rangeMatch[1], 10); const hi = Number.parseInt(rangeMatch[2], 10); const a = Math.min(lo, hi); const b = Math.max(lo, hi); for (let j = a; j <= b; j += 1) { if (j < 1 || j > count) { throw new Error( `Range ${part} includes invalid index ${String(j)} (valid: 1–${String(count)}).`, ); } seen.add(j); } continue; } throw new Error(`Invalid --select segment: "${part}"`); } if (seen.size === 0) { throw new Error("No indices parsed from --select."); } return [...seen].sort((x, y) => x - y); } /** * @param {string} title - e.g. `Chapter 22`. * @returns {string|null} Chapter number string from title, or null. */ function chapterNumberFromListingTitle(title) { const m = /^Chapter\s+([\d.]+)\s*$/iu.exec(title.trim().replace(/\s+/gu, " ")); return m ? m[1] : null; } /** * @param {string} spec - `--numbers` argument. * @returns {string[]} Ordered unique chapter number strings as entered. */ function parseChapterNumbersSpec(spec) { const trimmed = spec.trim(); if (trimmed.length === 0) { throw new Error("--numbers value is empty."); } /** @type {string[]} */ const ordered = []; /** @type {Set} */ const seen = new Set(); for (const rawPart of trimmed.split(",")) { const part = rawPart.trim(); if (part.length === 0) { continue; } const rangeMatch = /^(\d+)\s*-\s*(\d+)$/u.exec(part); if (rangeMatch) { const lo = Number.parseInt(rangeMatch[1], 10); const hi = Number.parseInt(rangeMatch[2], 10); const a = Math.min(lo, hi); const b = Math.max(lo, hi); for (let n = a; n <= b; n += 1) { const k = String(n); if (!seen.has(k)) { seen.add(k); ordered.push(k); } } continue; } if (/^\d+(\.\d+)?$/u.test(part)) { if (!seen.has(part)) { seen.add(part); ordered.push(part); } continue; } throw new Error(`Invalid --numbers segment: "${part}"`); } if (ordered.length === 0) { throw new Error("No chapter numbers parsed from --numbers."); } return ordered; } /** * @param {string[]} orderedNums - Chapter numbers requested. * @param {import("./chapters.js").ChapterLink[]} chapters - Oldest→newest. * @returns {Array<{ url: string, title: string, pickLabel: string }>} Rows to download. */ function queueItemsByChapterNumbers(orderedNums, chapters) { /** @type {Map} */ const first = new Map(); for (const ch of chapters) { const num = chapterNumberFromListingTitle(ch.title); if (num !== null && !first.has(num)) { first.set(num, ch); } } return orderedNums.map((num) => { const ch = first.get(num); if (!ch) { throw new Error( `No chapter titled "Chapter ${num}" in the ComicLand listing. ` + "Use --list-only and check titles.", ); } return { url: ch.url, title: ch.title, pickLabel: `Ch.${num}`, }; }); } /** * @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first list. * @returns {void} */ function printChapterMenu(chapters) { const digits = String(chapters.length).length; for (let i = 0; i < chapters.length; i += 1) { const n = i + 1; const pad = String(n).padStart(digits, " "); const title = chapters[i].title.replace(/\s+/gu, " ").trim(); console.log(`${pad}. ${title}`); console.log(` ${chapters[i].url}`); } } /** * @param {number[]} indices1 - Sorted 1-based indices. * @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first chapter list. * @returns {Array<{ url: string, title: string, pickLabel: string }>} Queue rows. */ function chaptersByIndices(indices1, chapters) { return indices1.map((idx) => ({ url: chapters[idx - 1].url, title: chapters[idx - 1].title, pickLabel: `#${String(idx)}`, })); } /** * CLI entry: fetch listing, optionally build `.cbz` archives. * Upserts titles/chapters into SQLite unless `--no-db`. * @returns {Promise} */ async function main() { let parsed; try { parsed = parseCli(process.argv); } catch (err) { console.error(err instanceof Error ? err.message : err); process.exitCode = 1; return; } if (parsed.help) { console.log(`ComicLand chapter queue (JSON API; URLs like /comic/{slug}): Usage: node src/comicland/downloadChaptersQueue.js [--list-only|-l] node src/comicland/downloadChaptersQueue.js --numbers|-n [--out|-o ] node src/comicland/downloadChaptersQueue.js --select|-s [--out|-o ] node src/comicland/downloadChaptersQueue.js --all [--out|-o ] node src/comicland/downloadChaptersQueue.js --missing [--out|-o ] node src/comicland/downloadChaptersQueue.js --title-id --missing … node src/comicland/downloadChaptersQueue.js --title-id --numbers|-n … Listing URL: series page https://comicland.org/comic/{slug} (chapter URL is also accepted). --title-id: load listing URL from SQLite (DATA_DIR/app.db) instead of a URL argument --missing: download chapters that are pending, failed, or downloaded but CBZ missing (requires SQLite) --delay N: wait N seconds between chapter downloads (e.g. --delay 300) --no-db: write CBZ files only; do not update SQLite (incompatible with --missing / --title-id) --verbose|-v: print informational progress logs DATA_DIR: data root (default: ./data) Default --out: data/comicland/comics// (with DB and no --out: $DATA_DIR/comicland/comics//) --numbers: titles like "Chapter 22", e.g. 22,23,24 or 22-24 --select: 1-based rows on printed list (line 1 = Chapter 1) `); return; } const vlog = makeVlog(parsed.verbose); const cliStore = require("../server/cli-store.js"); const { resolveCbzPath, cbzExists } = require("../server/sources.js"); const dbApi = require("../server/db.js"); /** @type {import("better-sqlite3").Database|null} */ let db = null; /** @type {string|undefined} */ let dataDir; /** @type {object|undefined} */ let dbTitle; try { if (parsed.missingOnly && parsed.noDb) { console.error("Cannot combine --missing with --no-db (SQLite is required)."); process.exitCode = 1; return; } if (!parsed.noDb || parsed.titleId !== undefined || parsed.missingOnly) { if (parsed.noDb && parsed.titleId !== undefined) { console.error("Cannot combine --no-db with --title-id."); process.exitCode = 1; return; } vlog(`opening SQLite under DATA_DIR=${cliStore.resolveDataDir()}`); ({ db, dataDir } = cliStore.openCliDatabase()); vlog(`database ready: ${path.join(dataDir, "app.db")}`); } if (parsed.titleId !== undefined) { vlog(`looking up --title-id ${String(parsed.titleId)}`); dbTitle = dbApi.getTitle(/** @type {import("better-sqlite3").Database} */ (db), parsed.titleId); if (!dbTitle) { console.error(`Title id ${String(parsed.titleId)} not found in SQLite.`); process.exitCode = 1; return; } if (dbTitle.source !== "comicland") { console.error( `Title id ${String(parsed.titleId)} is source "${dbTitle.source}"; use the matching CLI.`, ); process.exitCode = 1; return; } parsed.listingUrl = dbTitle.url; vlog(`resolved title "${dbTitle.title}" → ${dbTitle.url}`); } if (!parsed.listingUrl) { console.error( "Usage: node src/comicland/downloadChaptersQueue.js [options]\n" + "Or: --title-id [options]", ); process.exitCode = 1; return; } if (!isLikelyComicLandSite(parsed.listingUrl)) { console.warn( "Warning: URL hostname does not look like ComicLand; the API client may fail.", ); } vlog(`fetching chapter listing: ${parsed.listingUrl}`); let chapters; let comicTitle; try { ({ chapters, comicTitle } = await fetchChapterLinks(parsed.listingUrl)); chapters = chaptersInReadingOrder(chapters); } catch (err) { console.error(err instanceof Error ? err.message : err); process.exitCode = 1; return; } console.log(`Found ${String(chapters.length)} ComicLand chapter link(s).`); printChapterMenu(chapters); vlog(`comic title from listing: ${comicTitle}`); if (parsed.listOnly) { if (db && dataDir) { vlog("upserting title/chapters into SQLite (--list-only)"); const result = await cliStore.ensureTitleFromUrl(db, parsed.listingUrl, dataDir); console.log( `\nSQLite: title id=${String(result.title.id)} "${result.title.title}" ` + `(${result.created ? "created" : "updated"}), ` + `+${String(result.added)} chapter(s), total ${String(result.total)}`, ); } return; } const wantsAll = parsed.allChapters; const wantsMissing = parsed.missingOnly; const wantsSelect = Boolean(parsed.selectSpec); const wantsNumbers = Boolean(parsed.numbersSpec); const pickModes = [wantsAll, wantsMissing, wantsSelect, wantsNumbers].filter( Boolean, ).length; if (pickModes === 0) { console.error("Choose --numbers, --select, --all, or --missing (or --list-only)."); process.exitCode = 1; return; } if (pickModes > 1) { console.error("Use only one of --all, --missing, --select, or --numbers."); process.exitCode = 1; return; } if (db && dataDir) { vlog("upserting title/chapters into SQLite"); const result = await cliStore.ensureTitleFromUrl(db, parsed.listingUrl, dataDir); dbTitle = result.title; console.log( `SQLite: title id=${String(dbTitle.id)} "${dbTitle.title}" ` + `(${result.created ? "created" : "updated"}), ` + `+${String(result.added)} chapter(s), total ${String(result.total)}`, ); } /** @type {Array<{ url: string, title: string, pickLabel: string }>} */ let queue; try { if (wantsMissing) { if (!db || !dbTitle) { console.error("--missing requires SQLite (omit --no-db)."); process.exitCode = 1; return; } vlog("selecting chapters that need download (pending/failed/missing CBZ)"); const needing = listChaptersMissing(db, dbTitle.id, cbzExists); queue = queueItemsMissing(chapters, needing); console.log( `Missing: ${String(queue.length)} chapter(s) need download ` + `(of ${String(chapters.length)} on listing).`, ); if (queue.length === 0) { console.log("Nothing to download."); return; } } else if (wantsAll) { vlog(`queue mode: --all (${String(chapters.length)} chapter(s))`); const indices = Array.from({ length: chapters.length }, (_, i) => i + 1); queue = chaptersByIndices(indices, chapters); } else if (wantsSelect) { vlog(`queue mode: --select ${parsed.selectSpec}`); const indices = parseChapterSelection( /** @type {string} */ (parsed.selectSpec), chapters.length, ); queue = chaptersByIndices(indices, chapters); } else { vlog(`queue mode: --numbers ${parsed.numbersSpec}`); const orderedNums = parseChapterNumbersSpec( /** @type {string} */ (parsed.numbersSpec), ); queue = queueItemsByChapterNumbers(orderedNums, chapters); } } catch (err) { console.error(err instanceof Error ? err.message : err); process.exitCode = 1; return; } vlog(`download queue size: ${String(queue.length)}`); const titleForPaths = (dbTitle && dbTitle.title) || comicTitle; const useDbPaths = Boolean(db && dataDir && dbTitle && !parsed.outDirExplicit); const fallbackOutDir = path.join(parsed.outDir, sanitizeComicDirName(comicTitle)); const outDir = useDbPaths && queue.length > 0 ? path.dirname( resolveCbzPath( "comicland", titleForPaths, queue[0].url, /** @type {string} */ (dataDir), ), ) : fallbackOutDir; console.log(`Comic: ${comicTitle}`); console.log(`Output directory: ${outDir}`); if (parsed.delaySeconds > 0) { console.log(`Delay between chapters: ${String(parsed.delaySeconds)}s`); } let failedCount = 0; let done = 0; for (const item of queue) { done += 1; const stem = stemForComicLandChapterCbz(item.url); let targetPath = path.join(fallbackOutDir, `${stem}.cbz`); /** @type {object|undefined} */ let chapterRow; if (useDbPaths) { targetPath = resolveCbzPath( "comicland", titleForPaths, item.url, /** @type {string} */ (dataDir), ); chapterRow = cliStore.findChapterByUrl( /** @type {import("better-sqlite3").Database} */ (db), dbTitle.id, item.url, ); } else if (db && dbTitle) { chapterRow = cliStore.findChapterByUrl(db, dbTitle.id, item.url); } console.log(`\n[${String(done)}/${String(queue.length)}] ${item.pickLabel} ${item.title}`); console.log(` → ${item.url}`); vlog(`target CBZ: ${targetPath}`); try { if (db && chapterRow) { cliStore.markDownload(db, chapterRow, { ok: true, phase: "start" }); } const { imageCount, outputPath } = await chapterUrlToCbz(item.url, targetPath); console.log(` ✓ ${String(imageCount)} page(s) → ${outputPath}`); if (db && chapterRow) { cliStore.markDownload(db, chapterRow, { ok: true, cbzPath: outputPath, phase: "finish", }); } } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.error(` ✗ ${msg}`); if (db && chapterRow) { cliStore.markDownload(db, chapterRow, { ok: false, error: msg, phase: "finish", }); } failedCount += 1; } if (parsed.delaySeconds > 0 && done < queue.length) { console.log( ` … waiting ${String(parsed.delaySeconds)}s before next chapter`, ); await sleepSeconds(parsed.delaySeconds); } } if (failedCount > 0) { console.error( `\n${String(failedCount)} chapter(s) failed (${String(queue.length - failedCount)} succeeded).`, ); process.exitCode = 1; } } finally { if (db) { db.close(); } } } module.exports = { chapterNumberFromListingTitle, chaptersByIndices, chaptersInReadingOrder, listChaptersMissing, parseChapterNumbersSpec, parseChapterSelection, parseCli, printChapterMenu, queueItemsByChapterNumbers, queueItemsMissing, }; if (require.main === module) { main().catch((err) => { console.error(err); process.exitCode = 1; }); }