Files
cxyz/src/comicland/downloadChaptersQueue.js
T
nsrb a7c6058540 Add cxyz Android project structure and initial configuration
- Introduced a new Flutter project for the cxyz Android manga/comic reader, including essential files such as pubspec.yaml, AndroidManifest.xml, and Gradle configuration.
- Added .gitignore files to exclude unnecessary files and directories for both the main project and Android-specific files.
- Implemented initial README.md with project description, features, setup instructions, and API details for ComicLand, MangaRead, and MangaTown.
- Configured analysis options and dependencies in pubspec.yaml to support Flutter development.
- Established a pnpm workspace and lockfile for dependency management.
- Included various Android resources such as icons, styles, and launch backgrounds for a complete application setup.
2026-08-29 17:50:31 +07:00

733 lines
23 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
const path = require("path");
const { fetchChapterLinks, isLikelyComicLandSite } = require("./chapters.js");
const { stemForComicLandChapterCbz, chapterUrlToCbz, sanitizeComicDirName } =
require("./downloadChapterCbz.js");
/**
* @typedef {object} ParsedCli
* @property {boolean} [help] - Print usage and exit when true.
* @property {string|undefined} listingUrl - Series listing URL (required unless help).
* @property {boolean} listOnly - Only print the chapter menu.
* @property {boolean} allChapters - Queue every chapter (API already oldest-first).
* @property {boolean} missingOnly - Queue chapters that need download (SQLite + CBZ on disk).
* @property {string|undefined} selectSpec - `--select` string (1-based menu indices).
* @property {string|undefined} numbersSpec - `--numbers` string (chapter numbers from titles).
* @property {string} outDir - Output directory for `.cbz` files.
* @property {boolean} noDb - Skip SQLite updates.
* @property {number|undefined} titleId - Load listing URL from SQLite title id.
* @property {boolean} outDirExplicit - True when `--out` was passed.
* @property {boolean} verbose - Print informational progress logs.
* @property {number} delaySeconds - Pause between chapter downloads (0 = no delay).
*/
/**
* @param {string[]} argv - `process.argv`.
* @returns {ParsedCli} Parsed flags and positional URL.
*/
function parseCli(argv) {
/** @type {ParsedCli} */
const result = {
help: false,
listingUrl: undefined,
listOnly: false,
allChapters: false,
missingOnly: false,
selectSpec: undefined,
numbersSpec: undefined,
outDir: path.join(process.cwd(), "data", "comicland", "comics"),
noDb: false,
titleId: undefined,
outDirExplicit: false,
verbose: false,
delaySeconds: 0,
};
for (let i = 2; i < argv.length; i += 1) {
const a = argv[i];
if (a === "--help" || a === "-h") {
result.help = true;
} else if (a === "--list-only" || a === "-l") {
result.listOnly = true;
} else if (a === "--all") {
result.allChapters = true;
} else if (a === "--missing") {
result.missingOnly = true;
} else if (a === "--verbose" || a === "-v") {
result.verbose = true;
} else if (a === "--no-db") {
result.noDb = true;
} else if (a === "--delay") {
const next = argv[i + 1];
if (next === undefined || (next.startsWith("-") && !/^-?\d+$/u.test(next))) {
throw new Error("Missing value for --delay");
}
const seconds = Number.parseInt(next, 10);
if (!Number.isFinite(seconds) || seconds < 0) {
throw new Error(`Invalid --delay: ${next} (expected non-negative integer seconds)`);
}
result.delaySeconds = seconds;
i += 1;
} else if (a === "--title-id") {
const next = argv[i + 1];
if (next === undefined || next.startsWith("-")) {
throw new Error("Missing value for --title-id");
}
const id = Number.parseInt(next, 10);
if (!Number.isFinite(id) || id < 1) {
throw new Error(`Invalid --title-id: ${next}`);
}
result.titleId = id;
i += 1;
} else if (a === "--select" || a === "-s") {
const next = argv[i + 1];
if (next === undefined || next.startsWith("-")) {
throw new Error("Missing value for --select");
}
result.selectSpec = next;
i += 1;
} else if (a === "--numbers" || a === "-n") {
const next = argv[i + 1];
if (next === undefined || next.startsWith("-")) {
throw new Error("Missing value for --numbers");
}
result.numbersSpec = next;
i += 1;
} else if (a === "--out" || a === "-o") {
const next = argv[i + 1];
if (next === undefined || next.startsWith("-")) {
throw new Error("Missing value for --out");
}
result.outDir = path.resolve(next);
result.outDirExplicit = true;
i += 1;
} else if (!a.startsWith("-")) {
if (result.listingUrl) {
throw new Error(`Unexpected extra argument: ${a}`);
}
result.listingUrl = a;
} else {
throw new Error(`Unknown option: ${a}`);
}
}
return result;
}
/**
* @param {boolean} verbose - When false, no-op.
* @returns {(msg: string) => void} Logger.
*/
function makeVlog(verbose) {
return (msg) => {
if (verbose) {
console.log(`info: ${msg}`);
}
};
}
/**
* @param {number} seconds - Seconds to wait.
* @returns {Promise<void>}
*/
function sleepSeconds(seconds) {
return new Promise((resolve) => {
setTimeout(resolve, seconds * 1000);
});
}
/**
* Build a download queue from SQLite chapters that still need a CBZ.
* @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first listing.
* @param {object[]} needingRows - Chapter rows from SQLite.
* @returns {Array<{ url: string, title: string, pickLabel: string }>} Queue items.
*/
function queueItemsMissing(chapters, needingRows) {
/** @type {Map<string, object>} */
const byUrl = new Map();
for (const row of needingRows) {
if (typeof row.url === "string" && row.url) {
byUrl.set(row.url, row);
}
}
/** @type {Array<{ url: string, title: string, pickLabel: string }>} */
const items = [];
/** @type {Set<string>} */
const used = new Set();
for (const ch of chapters) {
const row = byUrl.get(ch.url);
if (!row) {
continue;
}
used.add(ch.url);
const reason =
row.status === "downloaded" ? "file-missing" : String(row.status);
items.push({
url: ch.url,
title: ch.title,
pickLabel: `missing:${reason}`,
});
}
for (const row of needingRows) {
if (typeof row.url !== "string" || !row.url || used.has(row.url)) {
continue;
}
const reason =
row.status === "downloaded" ? "file-missing" : String(row.status);
items.push({
url: row.url,
title: row.label || row.chapter_key || row.url,
pickLabel: `missing:${reason}`,
});
}
return items;
}
/**
* Chapters that need a download (status pending/failed, or downloaded CBZ gone).
* @param {import("better-sqlite3").Database} db - Database handle.
* @param {number} titleId - Title id.
* @param {(path: string|null|undefined) => boolean} cbzExistsFn - File check.
* @returns {object[]} Chapter rows.
*/
function listChaptersMissing(db, titleId, cbzExistsFn) {
const dbApi = require("../server/db.js");
const rows = dbApi.listChapters(db, titleId);
return rows.filter((ch) => {
if (ch.status === "pending" || ch.status === "failed") {
return true;
}
if (ch.status === "downloaded" && !cbzExistsFn(ch.cbz_path)) {
return true;
}
return false;
});
}
/**
* ComicLand API already returns oldest-first; copy so callers can mutate safely.
* @param {import("./chapters.js").ChapterLink[]} chapters - Site/API order.
* @returns {import("./chapters.js").ChapterLink[]} New array, oldest chapter first.
*/
function chaptersInReadingOrder(chapters) {
return [...chapters];
}
/**
* @param {string} spec - `--select` value.
* @param {number} count - Chapter count.
* @returns {number[]} Sorted unique 1-based row indices.
*/
function parseChapterSelection(spec, count) {
if (count <= 0) {
throw new Error("Chapter list is empty; cannot select indices.");
}
const trimmed = spec.trim();
if (trimmed.length === 0) {
throw new Error("--select value is empty.");
}
/** @type {Set<number>} */
const seen = new Set();
for (const rawPart of trimmed.split(",")) {
const part = rawPart.trim();
if (part.length === 0) {
continue;
}
if (/^\d+$/u.test(part)) {
const n = Number.parseInt(part, 10);
if (n < 1 || n > count) {
throw new Error(
`Index ${String(n)} is out of range (valid: 1–${String(count)}).`,
);
}
seen.add(n);
continue;
}
const rangeMatch = /^(\d+)\s*-\s*(\d+)$/u.exec(part);
if (rangeMatch) {
const lo = Number.parseInt(rangeMatch[1], 10);
const hi = Number.parseInt(rangeMatch[2], 10);
const a = Math.min(lo, hi);
const b = Math.max(lo, hi);
for (let j = a; j <= b; j += 1) {
if (j < 1 || j > count) {
throw new Error(
`Range ${part} includes invalid index ${String(j)} (valid: 1–${String(count)}).`,
);
}
seen.add(j);
}
continue;
}
throw new Error(`Invalid --select segment: "${part}"`);
}
if (seen.size === 0) {
throw new Error("No indices parsed from --select.");
}
return [...seen].sort((x, y) => x - y);
}
/**
* @param {string} title - e.g. `Chapter 22`.
* @returns {string|null} Chapter number string from title, or null.
*/
function chapterNumberFromListingTitle(title) {
const m = /^Chapter\s+([\d.]+)\s*$/iu.exec(title.trim().replace(/\s+/gu, " "));
return m ? m[1] : null;
}
/**
* @param {string} spec - `--numbers` argument.
* @returns {string[]} Ordered unique chapter number strings as entered.
*/
function parseChapterNumbersSpec(spec) {
const trimmed = spec.trim();
if (trimmed.length === 0) {
throw new Error("--numbers value is empty.");
}
/** @type {string[]} */
const ordered = [];
/** @type {Set<string>} */
const seen = new Set();
for (const rawPart of trimmed.split(",")) {
const part = rawPart.trim();
if (part.length === 0) {
continue;
}
const rangeMatch = /^(\d+)\s*-\s*(\d+)$/u.exec(part);
if (rangeMatch) {
const lo = Number.parseInt(rangeMatch[1], 10);
const hi = Number.parseInt(rangeMatch[2], 10);
const a = Math.min(lo, hi);
const b = Math.max(lo, hi);
for (let n = a; n <= b; n += 1) {
const k = String(n);
if (!seen.has(k)) {
seen.add(k);
ordered.push(k);
}
}
continue;
}
if (/^\d+(\.\d+)?$/u.test(part)) {
if (!seen.has(part)) {
seen.add(part);
ordered.push(part);
}
continue;
}
throw new Error(`Invalid --numbers segment: "${part}"`);
}
if (ordered.length === 0) {
throw new Error("No chapter numbers parsed from --numbers.");
}
return ordered;
}
/**
* @param {string[]} orderedNums - Chapter numbers requested.
* @param {import("./chapters.js").ChapterLink[]} chapters - Oldest→newest.
* @returns {Array<{ url: string, title: string, pickLabel: string }>} Rows to download.
*/
function queueItemsByChapterNumbers(orderedNums, chapters) {
/** @type {Map<string, import("./chapters.js").ChapterLink>} */
const first = new Map();
for (const ch of chapters) {
const num = chapterNumberFromListingTitle(ch.title);
if (num !== null && !first.has(num)) {
first.set(num, ch);
}
}
return orderedNums.map((num) => {
const ch = first.get(num);
if (!ch) {
throw new Error(
`No chapter titled "Chapter ${num}" in the ComicLand listing. ` +
"Use --list-only and check titles.",
);
}
return {
url: ch.url,
title: ch.title,
pickLabel: `Ch.${num}`,
};
});
}
/**
* @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first list.
* @returns {void}
*/
function printChapterMenu(chapters) {
const digits = String(chapters.length).length;
for (let i = 0; i < chapters.length; i += 1) {
const n = i + 1;
const pad = String(n).padStart(digits, " ");
const title = chapters[i].title.replace(/\s+/gu, " ").trim();
console.log(`${pad}. ${title}`);
console.log(` ${chapters[i].url}`);
}
}
/**
* @param {number[]} indices1 - Sorted 1-based indices.
* @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first chapter list.
* @returns {Array<{ url: string, title: string, pickLabel: string }>} Queue rows.
*/
function chaptersByIndices(indices1, chapters) {
return indices1.map((idx) => ({
url: chapters[idx - 1].url,
title: chapters[idx - 1].title,
pickLabel: `#${String(idx)}`,
}));
}
/**
* CLI entry: fetch listing, optionally build `.cbz` archives.
* Upserts titles/chapters into SQLite unless `--no-db`.
* @returns {Promise<void>}
*/
async function main() {
let parsed;
try {
parsed = parseCli(process.argv);
} catch (err) {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
return;
}
if (parsed.help) {
console.log(`ComicLand chapter queue (JSON API; URLs like /comic/{slug}):
Usage:
node src/comicland/downloadChaptersQueue.js <listing-url> [--list-only|-l]
node src/comicland/downloadChaptersQueue.js <listing-url> --numbers|-n <spec> [--out|-o <dir>]
node src/comicland/downloadChaptersQueue.js <listing-url> --select|-s <spec> [--out|-o <dir>]
node src/comicland/downloadChaptersQueue.js <listing-url> --all [--out|-o <dir>]
node src/comicland/downloadChaptersQueue.js <listing-url> --missing [--out|-o <dir>]
node src/comicland/downloadChaptersQueue.js --title-id <id> --missing …
node src/comicland/downloadChaptersQueue.js --title-id <id> --numbers|-n <spec> …
Listing URL: series page https://comicland.org/comic/{slug} (chapter URL is also accepted).
--title-id: load listing URL from SQLite (DATA_DIR/app.db) instead of a URL argument
--missing: download chapters that are pending, failed, or downloaded but CBZ missing (requires SQLite)
--delay N: wait N seconds between chapter downloads (e.g. --delay 300)
--no-db: write CBZ files only; do not update SQLite (incompatible with --missing / --title-id)
--verbose|-v: print informational progress logs
DATA_DIR: data root (default: ./data)
Default --out: data/comicland/comics/<comic title>/
(with DB and no --out: $DATA_DIR/comicland/comics/<comic title>/)
--numbers: titles like "Chapter 22", e.g. 22,23,24 or 22-24
--select: 1-based rows on printed list (line 1 = Chapter 1)
`);
return;
}
const vlog = makeVlog(parsed.verbose);
const cliStore = require("../server/cli-store.js");
const { resolveCbzPath, cbzExists } = require("../server/sources.js");
const dbApi = require("../server/db.js");
/** @type {import("better-sqlite3").Database|null} */
let db = null;
/** @type {string|undefined} */
let dataDir;
/** @type {object|undefined} */
let dbTitle;
try {
if (parsed.missingOnly && parsed.noDb) {
console.error("Cannot combine --missing with --no-db (SQLite is required).");
process.exitCode = 1;
return;
}
if (!parsed.noDb || parsed.titleId !== undefined || parsed.missingOnly) {
if (parsed.noDb && parsed.titleId !== undefined) {
console.error("Cannot combine --no-db with --title-id.");
process.exitCode = 1;
return;
}
vlog(`opening SQLite under DATA_DIR=${cliStore.resolveDataDir()}`);
({ db, dataDir } = cliStore.openCliDatabase());
vlog(`database ready: ${path.join(dataDir, "app.db")}`);
}
if (parsed.titleId !== undefined) {
vlog(`looking up --title-id ${String(parsed.titleId)}`);
dbTitle = dbApi.getTitle(/** @type {import("better-sqlite3").Database} */ (db), parsed.titleId);
if (!dbTitle) {
console.error(`Title id ${String(parsed.titleId)} not found in SQLite.`);
process.exitCode = 1;
return;
}
if (dbTitle.source !== "comicland") {
console.error(
`Title id ${String(parsed.titleId)} is source "${dbTitle.source}"; use the matching CLI.`,
);
process.exitCode = 1;
return;
}
parsed.listingUrl = dbTitle.url;
vlog(`resolved title "${dbTitle.title}" → ${dbTitle.url}`);
}
if (!parsed.listingUrl) {
console.error(
"Usage: node src/comicland/downloadChaptersQueue.js <listing-url> [options]\n" +
"Or: --title-id <id> [options]",
);
process.exitCode = 1;
return;
}
if (!isLikelyComicLandSite(parsed.listingUrl)) {
console.warn(
"Warning: URL hostname does not look like ComicLand; the API client may fail.",
);
}
vlog(`fetching chapter listing: ${parsed.listingUrl}`);
let chapters;
let comicTitle;
try {
({ chapters, comicTitle } = await fetchChapterLinks(parsed.listingUrl));
chapters = chaptersInReadingOrder(chapters);
} catch (err) {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
return;
}
console.log(`Found ${String(chapters.length)} ComicLand chapter link(s).`);
printChapterMenu(chapters);
vlog(`comic title from listing: ${comicTitle}`);
if (parsed.listOnly) {
if (db && dataDir) {
vlog("upserting title/chapters into SQLite (--list-only)");
const result = await cliStore.ensureTitleFromUrl(db, parsed.listingUrl, dataDir);
console.log(
`\nSQLite: title id=${String(result.title.id)} "${result.title.title}" ` +
`(${result.created ? "created" : "updated"}), ` +
`+${String(result.added)} chapter(s), total ${String(result.total)}`,
);
}
return;
}
const wantsAll = parsed.allChapters;
const wantsMissing = parsed.missingOnly;
const wantsSelect = Boolean(parsed.selectSpec);
const wantsNumbers = Boolean(parsed.numbersSpec);
const pickModes = [wantsAll, wantsMissing, wantsSelect, wantsNumbers].filter(
Boolean,
).length;
if (pickModes === 0) {
console.error("Choose --numbers, --select, --all, or --missing (or --list-only).");
process.exitCode = 1;
return;
}
if (pickModes > 1) {
console.error("Use only one of --all, --missing, --select, or --numbers.");
process.exitCode = 1;
return;
}
if (db && dataDir) {
vlog("upserting title/chapters into SQLite");
const result = await cliStore.ensureTitleFromUrl(db, parsed.listingUrl, dataDir);
dbTitle = result.title;
console.log(
`SQLite: title id=${String(dbTitle.id)} "${dbTitle.title}" ` +
`(${result.created ? "created" : "updated"}), ` +
`+${String(result.added)} chapter(s), total ${String(result.total)}`,
);
}
/** @type {Array<{ url: string, title: string, pickLabel: string }>} */
let queue;
try {
if (wantsMissing) {
if (!db || !dbTitle) {
console.error("--missing requires SQLite (omit --no-db).");
process.exitCode = 1;
return;
}
vlog("selecting chapters that need download (pending/failed/missing CBZ)");
const needing = listChaptersMissing(db, dbTitle.id, cbzExists);
queue = queueItemsMissing(chapters, needing);
console.log(
`Missing: ${String(queue.length)} chapter(s) need download ` +
`(of ${String(chapters.length)} on listing).`,
);
if (queue.length === 0) {
console.log("Nothing to download.");
return;
}
} else if (wantsAll) {
vlog(`queue mode: --all (${String(chapters.length)} chapter(s))`);
const indices = Array.from({ length: chapters.length }, (_, i) => i + 1);
queue = chaptersByIndices(indices, chapters);
} else if (wantsSelect) {
vlog(`queue mode: --select ${parsed.selectSpec}`);
const indices = parseChapterSelection(
/** @type {string} */ (parsed.selectSpec),
chapters.length,
);
queue = chaptersByIndices(indices, chapters);
} else {
vlog(`queue mode: --numbers ${parsed.numbersSpec}`);
const orderedNums = parseChapterNumbersSpec(
/** @type {string} */ (parsed.numbersSpec),
);
queue = queueItemsByChapterNumbers(orderedNums, chapters);
}
} catch (err) {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
return;
}
vlog(`download queue size: ${String(queue.length)}`);
const titleForPaths = (dbTitle && dbTitle.title) || comicTitle;
const useDbPaths = Boolean(db && dataDir && dbTitle && !parsed.outDirExplicit);
const fallbackOutDir = path.join(parsed.outDir, sanitizeComicDirName(comicTitle));
const outDir =
useDbPaths && queue.length > 0
? path.dirname(
resolveCbzPath(
"comicland",
titleForPaths,
queue[0].url,
/** @type {string} */ (dataDir),
),
)
: fallbackOutDir;
console.log(`Comic: ${comicTitle}`);
console.log(`Output directory: ${outDir}`);
if (parsed.delaySeconds > 0) {
console.log(`Delay between chapters: ${String(parsed.delaySeconds)}s`);
}
let failedCount = 0;
let done = 0;
for (const item of queue) {
done += 1;
const stem = stemForComicLandChapterCbz(item.url);
let targetPath = path.join(fallbackOutDir, `${stem}.cbz`);
/** @type {object|undefined} */
let chapterRow;
if (useDbPaths) {
targetPath = resolveCbzPath(
"comicland",
titleForPaths,
item.url,
/** @type {string} */ (dataDir),
);
chapterRow = cliStore.findChapterByUrl(
/** @type {import("better-sqlite3").Database} */ (db),
dbTitle.id,
item.url,
);
} else if (db && dbTitle) {
chapterRow = cliStore.findChapterByUrl(db, dbTitle.id, item.url);
}
console.log(`\n[${String(done)}/${String(queue.length)}] ${item.pickLabel} ${item.title}`);
console.log(` → ${item.url}`);
vlog(`target CBZ: ${targetPath}`);
try {
if (db && chapterRow) {
cliStore.markDownload(db, chapterRow, { ok: true, phase: "start" });
}
const { imageCount, outputPath } = await chapterUrlToCbz(item.url, targetPath);
console.log(` ✓ ${String(imageCount)} page(s) → ${outputPath}`);
if (db && chapterRow) {
cliStore.markDownload(db, chapterRow, {
ok: true,
cbzPath: outputPath,
phase: "finish",
});
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
console.error(` ✗ ${msg}`);
if (db && chapterRow) {
cliStore.markDownload(db, chapterRow, {
ok: false,
error: msg,
phase: "finish",
});
}
failedCount += 1;
}
if (parsed.delaySeconds > 0 && done < queue.length) {
console.log(
` … waiting ${String(parsed.delaySeconds)}s before next chapter`,
);
await sleepSeconds(parsed.delaySeconds);
}
}
if (failedCount > 0) {
console.error(
`\n${String(failedCount)} chapter(s) failed (${String(queue.length - failedCount)} succeeded).`,
);
process.exitCode = 1;
}
} finally {
if (db) {
db.close();
}
}
}
module.exports = {
chapterNumberFromListingTitle,
chaptersByIndices,
chaptersInReadingOrder,
listChaptersMissing,
parseChapterNumbersSpec,
parseChapterSelection,
parseCli,
printChapterMenu,
queueItemsByChapterNumbers,
queueItemsMissing,
};
if (require.main === module) {
main().catch((err) => {
console.error(err);
process.exitCode = 1;
});
}