Add initial project structure with Docker support and chapter downloading functionality
- Created .dockerignore and .gitignore files to exclude unnecessary files from Docker builds. - Added Dockerfile for multi-stage build setup, including dependencies and production environment configuration. - Introduced docker-compose.yml for service orchestration, including FlareSolverr and the main application. - Implemented core functionality for downloading manga chapters, including utilities for handling file paths and chapter metadata. - Added ESLint configuration for code quality and consistency. - Included JSDoc configuration for documentation generation. - Established initial package.json and package-lock.json with necessary dependencies for the project.
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
node_modules
|
||||
.git
|
||||
docs
|
||||
data
|
||||
public
|
||||
.env
|
||||
.env.*
|
||||
*.md
|
||||
@@ -0,0 +1,5 @@
|
||||
node_modules/
|
||||
docs/
|
||||
data/
|
||||
public/*
|
||||
!public/.gitkeep
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
FROM node:22-bookworm-slim AS web-build
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends python3 make g++ \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY web ./web
|
||||
COPY src ./src
|
||||
RUN npm run build:web
|
||||
|
||||
FROM node:22-bookworm-slim AS runtime
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends python3 make g++ \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV DATA_DIR=/app/data
|
||||
ENV PORT=3000
|
||||
ENV WATCH_CRON="0 * * * *"
|
||||
ENV WATCH_TITLE_INTERVAL_HOURS="24"
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci --omit=dev \
|
||||
&& apt-get purge -y python3 make g++ \
|
||||
&& apt-get autoremove -y \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY src ./src
|
||||
COPY --from=web-build /app/public ./public
|
||||
|
||||
RUN mkdir -p /app/data
|
||||
|
||||
EXPOSE 3000
|
||||
VOLUME ["/app/data"]
|
||||
|
||||
CMD ["node", "src/server/index.js"]
|
||||
@@ -0,0 +1,26 @@
|
||||
services:
|
||||
flaresolverr:
|
||||
image: ghcr.io/flaresolverr/flaresolverr:latest
|
||||
environment:
|
||||
LOG_LEVEL: info
|
||||
ports:
|
||||
- "8191:8191"
|
||||
restart: unless-stopped
|
||||
|
||||
cxyz:
|
||||
build: .
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
PORT: "3000"
|
||||
DATA_DIR: /app/data
|
||||
# Wake hourly; sync at most one due title per wake.
|
||||
WATCH_CRON: "0 * * * *"
|
||||
# Each title is eligible again after this many hours (daily).
|
||||
WATCH_TITLE_INTERVAL_HOURS: "24"
|
||||
FLARESOLVERR_URL: http://flaresolverr:8191/v1
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
depends_on:
|
||||
- flaresolverr
|
||||
restart: unless-stopped
|
||||
@@ -0,0 +1,18 @@
|
||||
import js from "@eslint/js";
|
||||
import globals from "globals";
|
||||
import jsdoc from "eslint-plugin-jsdoc";
|
||||
|
||||
export default [
|
||||
{
|
||||
ignores: ["node_modules/**", "docs/**", "public/**", "web/**", "data/**"],
|
||||
},
|
||||
js.configs.recommended,
|
||||
jsdoc.configs["flat/recommended"],
|
||||
{
|
||||
files: ["**/*.js"],
|
||||
languageOptions: {
|
||||
ecmaVersion: 2024,
|
||||
globals: globals.node,
|
||||
},
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"source": {
|
||||
"include": ["./src"],
|
||||
"includePattern": "\\.js$",
|
||||
"excludePattern": "(node_modules/|docs)"
|
||||
},
|
||||
"opts": {
|
||||
"destination": "./docs",
|
||||
"recurse": true
|
||||
}
|
||||
}
|
||||
Generated
+3961
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"name": "cxyz",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "src/index.js",
|
||||
"scripts": {
|
||||
"start": "node src/index.js",
|
||||
"web": "node src/server/index.js",
|
||||
"dev:web": "vite --config web/vite.config.mjs",
|
||||
"build:web": "vite build --config web/vite.config.mjs",
|
||||
"lint": "eslint .",
|
||||
"lint:fix": "eslint . --fix",
|
||||
"docs": "jsdoc -c jsdoc.conf.json",
|
||||
"mangaread:fetch": "node src/mangareadFetchChapters.js",
|
||||
"manhwasusu:fetch": "node src/manhwasusu/fetchChapters.cli.js",
|
||||
"manhwasusu:chapter-cbz": "node src/manhwasusu/downloadChapterCbz.js",
|
||||
"manhwasusu:chapters-cbz": "node src/manhwasusu/downloadChaptersQueue.js",
|
||||
"mangaread:chapter-cbz": "node src/downloadChapterCbz.js",
|
||||
"mangaread:chapters-cbz": "node src/downloadChaptersQueue.js",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@fastify/static": "^10.1.2",
|
||||
"@picocss/pico": "^2.1.1",
|
||||
"adm-zip": "^0.5.17",
|
||||
"axios": "^1.16.1",
|
||||
"better-sqlite3": "^13.0.2",
|
||||
"fastify": "^5.11.0",
|
||||
"node-cron": "^4.6.0",
|
||||
"node-html-parser": "^7.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^10.0.1",
|
||||
"@sveltejs/vite-plugin-svelte": "^7.2.0",
|
||||
"eslint": "^10.4.0",
|
||||
"eslint-plugin-jsdoc": "^62.9.0",
|
||||
"globals": "^17.6.0",
|
||||
"jsdoc": "^4.0.5",
|
||||
"svelte": "^5.56.8",
|
||||
"vite": "^8.2.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
const fs = require("fs/promises");
|
||||
const path = require("path");
|
||||
const os = require("os");
|
||||
const AdmZip = require("adm-zip");
|
||||
|
||||
const { downloadWpMangaChapterImages, extractWpMangaComicTitle, fetchPageHtml, mangaSlugFromUrl } =
|
||||
require("./mangareadFetchChapters.js");
|
||||
|
||||
/**
|
||||
* @param {string} chapterPageUrl - Full chapter URL.
|
||||
* @returns {string} Suggested base name without extension.
|
||||
*/
|
||||
function stemForCbzFilename(chapterPageUrl) {
|
||||
try {
|
||||
const { pathname } = new URL(chapterPageUrl);
|
||||
const segs = pathname.replace(/\/+$/, "").split("/").filter(Boolean);
|
||||
const mangaIdx = segs.indexOf("manga");
|
||||
if (mangaIdx !== -1 && segs.length >= mangaIdx + 3) {
|
||||
return `${segs[mangaIdx + 1]}-${segs[mangaIdx + 2]}`;
|
||||
}
|
||||
if (segs.length >= 2) {
|
||||
return segs.slice(-2).join("-");
|
||||
}
|
||||
return segs.at(-1) || "chapter";
|
||||
} catch {
|
||||
return "chapter";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} stem - File base name.
|
||||
* @returns {string} Sanitized segment.
|
||||
*/
|
||||
function sanitizeFsStem(stem) {
|
||||
const cleaned = stem.replace(/[^a-z0-9._-]+/gi, "_").replace(/^_+|_+$/g, "");
|
||||
return cleaned.length > 0 ? cleaned.slice(0, 180) : "chapter";
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} name - Human-readable comic title.
|
||||
* @returns {string} Filesystem-safe directory name.
|
||||
*/
|
||||
function sanitizeComicDirName(name) {
|
||||
const collapsed = name.trim().replace(/\s+/g, " ");
|
||||
const cleaned = [...collapsed]
|
||||
.filter((ch) => {
|
||||
const code = ch.charCodeAt(0);
|
||||
if (code < 32 || code === 127) {
|
||||
return false;
|
||||
}
|
||||
return !/[<>:"/\\|?*]/.test(ch);
|
||||
})
|
||||
.join("")
|
||||
.replace(/\.+$/g, "")
|
||||
.trim();
|
||||
return cleaned.length > 0 ? cleaned.slice(0, 180) : "comic";
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string|undefined} cliOut - Second CLI argument, if any.
|
||||
* @param {string} defaultStem - Used when `cliOut` is a directory or omitted.
|
||||
* @param {string} [comicTitle] - When set, default output goes under this subdirectory.
|
||||
* @returns {string} Absolute path ending in `.cbz`.
|
||||
*/
|
||||
function resolveCliOutputPath(cliOut, defaultStem, comicTitle) {
|
||||
const stem = sanitizeFsStem(defaultStem);
|
||||
const baseDir =
|
||||
typeof comicTitle === "string" && comicTitle.trim().length > 0
|
||||
? path.join(process.cwd(), "data", "comics", sanitizeComicDirName(comicTitle))
|
||||
: path.join(process.cwd(), "data", "comics");
|
||||
if (cliOut === undefined || cliOut.trim() === "") {
|
||||
return path.join(baseDir, `${stem}.cbz`);
|
||||
}
|
||||
const trimmed = cliOut.trim();
|
||||
const abs = path.resolve(trimmed);
|
||||
if (abs.toLowerCase().endsWith(".cbz")) {
|
||||
return abs;
|
||||
}
|
||||
return path.join(abs, `${stem}.cbz`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads chapter images to a temp folder, packs them into a `.cbz` zip, then deletes temp files.
|
||||
* @param {string} chapterUrl - Chapter reader URL.
|
||||
* @param {string} outputCbzPath - Absolute `.cbz` path to write.
|
||||
* @returns {Promise<{ imageCount: number, outputPath: string }>} Summary of written pages and archive path.
|
||||
*/
|
||||
async function chapterUrlToCbz(chapterUrl, outputCbzPath) {
|
||||
const absOut = outputCbzPath.toLowerCase().endsWith(".cbz")
|
||||
? path.resolve(outputCbzPath)
|
||||
: `${path.resolve(outputCbzPath)}.cbz`;
|
||||
|
||||
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "cxyz-chapter-"));
|
||||
|
||||
try {
|
||||
const saved = await downloadWpMangaChapterImages(chapterUrl, tmpRoot);
|
||||
|
||||
if (saved.length === 0) {
|
||||
throw new Error("No chapter images were found (try another URL or chapter).");
|
||||
}
|
||||
|
||||
await fs.mkdir(path.dirname(absOut), { recursive: true });
|
||||
|
||||
const zip = new AdmZip();
|
||||
for (const { filePath } of saved) {
|
||||
zip.addLocalFile(filePath, "", path.basename(filePath));
|
||||
}
|
||||
zip.writeZip(absOut);
|
||||
|
||||
return { imageCount: saved.length, outputPath: absOut };
|
||||
} finally {
|
||||
await fs.rm(tmpRoot, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI: `node src/downloadChapterCbz.js <chapterUrl> [path/to/file.cbz|path/to/dir/]`
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function main() {
|
||||
const chapterUrl = process.argv[2];
|
||||
if (!chapterUrl || chapterUrl.startsWith("-")) {
|
||||
console.error(
|
||||
"Usage: node src/downloadChapterCbz.js <chapterUrl> [output.cbz|outputDirectory]",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const outArg = process.argv[3];
|
||||
const stem = stemForCbzFilename(chapterUrl);
|
||||
let comicTitle = mangaSlugFromUrl(chapterUrl);
|
||||
try {
|
||||
const html = await fetchPageHtml(chapterUrl);
|
||||
comicTitle = extractWpMangaComicTitle(html) || comicTitle;
|
||||
} catch {
|
||||
/* keep slug fallback */
|
||||
}
|
||||
const target = resolveCliOutputPath(outArg, stem, comicTitle);
|
||||
|
||||
try {
|
||||
const { imageCount, outputPath } = await chapterUrlToCbz(chapterUrl, target);
|
||||
console.log(`Wrote ${String(imageCount)} page(s) → ${outputPath}`);
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : err);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
chapterUrlToCbz,
|
||||
sanitizeComicDirName,
|
||||
sanitizeFsStem,
|
||||
stemForCbzFilename,
|
||||
resolveCliOutputPath,
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
main();
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
const path = require("path");
|
||||
|
||||
const { fetchWpMangaChapterLinks } = require("./mangareadFetchChapters.js");
|
||||
const { chapterUrlToCbz, sanitizeComicDirName, stemForCbzFilename } = require("./downloadChapterCbz.js");
|
||||
|
||||
/**
|
||||
* @typedef {object} ParsedCli
|
||||
* @property {boolean} [help] - When set from `--help`, print usage and exit.
|
||||
* @property {string|undefined} listingUrl - Page containing `li.wp-manga-chapter` links.
|
||||
* @property {boolean} listOnly - Print numbered chapters and exit.
|
||||
* @property {boolean} allChapters - Queue every chapter in list order.
|
||||
* @property {string|undefined} selectSpec - Comma-separated **list indices** on the printed menu (oldest first: row 1 ≈ Chapter 1), e.g. `1,3,5-7`.
|
||||
* @property {string|undefined} numbersSpec - Comma-separated **chapter numbers** from titles, e.g. `22,23,24` or `22-24`.
|
||||
* @property {string} outDir - Directory for `.cbz` files.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Parses CLI arguments after `node .../downloadChaptersQueue.js`.
|
||||
* @param {string[]} argv - `process.argv`.
|
||||
* @returns {ParsedCli} Parsed options.
|
||||
*/
|
||||
function parseCli(argv) {
|
||||
/** @type {ParsedCli} */
|
||||
const result = {
|
||||
help: false,
|
||||
listingUrl: undefined,
|
||||
listOnly: false,
|
||||
allChapters: false,
|
||||
selectSpec: undefined,
|
||||
numbersSpec: undefined,
|
||||
outDir: path.join(process.cwd(), "data", "comics"),
|
||||
};
|
||||
|
||||
for (let i = 2; i < argv.length; i += 1) {
|
||||
const a = argv[i];
|
||||
if (a === "--help" || a === "-h") {
|
||||
result.help = true;
|
||||
} else if (a === "--list-only" || a === "-l") {
|
||||
result.listOnly = true;
|
||||
} else if (a === "--all") {
|
||||
result.allChapters = true;
|
||||
} else if (a === "--select" || a === "-s") {
|
||||
const next = argv[i + 1];
|
||||
if (next === undefined || next.startsWith("-")) {
|
||||
throw new Error("Missing value for --select");
|
||||
}
|
||||
result.selectSpec = next;
|
||||
i += 1;
|
||||
} else if (a === "--numbers" || a === "-n") {
|
||||
const next = argv[i + 1];
|
||||
if (next === undefined || next.startsWith("-")) {
|
||||
throw new Error("Missing value for --numbers");
|
||||
}
|
||||
result.numbersSpec = next;
|
||||
i += 1;
|
||||
} else if (a === "--out" || a === "-o") {
|
||||
const next = argv[i + 1];
|
||||
if (next === undefined || next.startsWith("-")) {
|
||||
throw new Error("Missing value for --out");
|
||||
}
|
||||
result.outDir = path.resolve(next);
|
||||
i += 1;
|
||||
} else if (!a.startsWith("-")) {
|
||||
if (result.listingUrl) {
|
||||
throw new Error(`Unexpected extra argument: ${a}`);
|
||||
}
|
||||
result.listingUrl = a;
|
||||
} else {
|
||||
throw new Error(`Unknown option: ${a}`);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* WP‑Manga lists chapters **newest first** in the HTML. We reverse so the numbered
|
||||
* menu and `--select` indices match reading order (line 1 ≈ Chapter 1).
|
||||
* @param {import("./mangareadFetchChapters.js").ChapterLink[]} chapters - Fetched in site order.
|
||||
* @returns {import("./mangareadFetchChapters.js").ChapterLink[]} New array, oldest chapter first.
|
||||
*/
|
||||
function chaptersInReadingOrder(chapters) {
|
||||
return [...chapters].reverse();
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds sorted unique 1-based indices from a spec like `1`, `1,3`, `5-10,2`.
|
||||
* @param {string} spec - Selection string.
|
||||
* @param {number} count - Number of chapters (max index).
|
||||
* @returns {number[]} Ascending unique indices in [1, count].
|
||||
*/
|
||||
function parseChapterSelection(spec, count) {
|
||||
if (count <= 0) {
|
||||
throw new Error("Chapter list is empty; cannot select indices.");
|
||||
}
|
||||
|
||||
const trimmed = spec.trim();
|
||||
if (trimmed.length === 0) {
|
||||
throw new Error("--select value is empty.");
|
||||
}
|
||||
|
||||
/** @type {Set<number>} */
|
||||
const seen = new Set();
|
||||
|
||||
for (const rawPart of trimmed.split(",")) {
|
||||
const part = rawPart.trim();
|
||||
if (part.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (/^\d+$/.test(part)) {
|
||||
const n = Number.parseInt(part, 10);
|
||||
if (n < 1 || n > count) {
|
||||
throw new Error(
|
||||
`Index ${String(n)} is out of range (valid: 1–${String(count)}).`,
|
||||
);
|
||||
}
|
||||
seen.add(n);
|
||||
continue;
|
||||
}
|
||||
|
||||
const rangeMatch = /^(\d+)\s*-\s*(\d+)$/.exec(part);
|
||||
if (rangeMatch) {
|
||||
const lo = Number.parseInt(rangeMatch[1], 10);
|
||||
const hi = Number.parseInt(rangeMatch[2], 10);
|
||||
const a = Math.min(lo, hi);
|
||||
const b = Math.max(lo, hi);
|
||||
for (let i = a; i <= b; i += 1) {
|
||||
if (i < 1 || i > count) {
|
||||
throw new Error(
|
||||
`Range ${part} includes invalid index ${String(i)} (valid: 1–${String(count)}).`,
|
||||
);
|
||||
}
|
||||
seen.add(i);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`Invalid --select segment: "${part}"`);
|
||||
}
|
||||
|
||||
if (seen.size === 0) {
|
||||
throw new Error("No indices parsed from --select.");
|
||||
}
|
||||
|
||||
return [...seen].sort((x, y) => x - y);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses "Chapter 22" / "chapter 160.5" style titles from WP‑Manga lists.
|
||||
* @param {string} title - Anchor text.
|
||||
* @returns {string|null} Numeric suffix or null.
|
||||
*/
|
||||
function chapterNumberFromWpTitle(title) {
|
||||
const m = /^Chapter\s+([\d.]+)\s*$/iu.exec(title.trim().replace(/\s+/g, " "));
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Expands `"22,23,24"` or `"22-24"` into ordered chapter-number strings.
|
||||
* Integer ranges only (22-24 → 22,23,24). Use explicit `160.5` for decimals.
|
||||
* @param {string} spec - User `--numbers` argument.
|
||||
* @returns {string[]} Ordered unique chapter numbers to resolve.
|
||||
*/
|
||||
function parseMangaChapterNumbersSpec(spec) {
|
||||
const trimmed = spec.trim();
|
||||
if (trimmed.length === 0) {
|
||||
throw new Error("--numbers value is empty.");
|
||||
}
|
||||
|
||||
/** @type {string[]} */
|
||||
const ordered = [];
|
||||
/** @type {Set<string>} */
|
||||
const seen = new Set();
|
||||
|
||||
for (const rawPart of trimmed.split(",")) {
|
||||
const part = rawPart.trim();
|
||||
if (part.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const rangeMatch = /^(\d+)\s*-\s*(\d+)$/.exec(part);
|
||||
if (rangeMatch) {
|
||||
const lo = Number.parseInt(rangeMatch[1], 10);
|
||||
const hi = Number.parseInt(rangeMatch[2], 10);
|
||||
const a = Math.min(lo, hi);
|
||||
const b = Math.max(lo, hi);
|
||||
for (let n = a; n <= b; n += 1) {
|
||||
const k = String(n);
|
||||
if (!seen.has(k)) {
|
||||
seen.add(k);
|
||||
ordered.push(k);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (/^\d+(\.\d+)?$/.test(part)) {
|
||||
if (!seen.has(part)) {
|
||||
seen.add(part);
|
||||
ordered.push(part);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`Invalid --numbers segment: "${part}"`);
|
||||
}
|
||||
|
||||
if (ordered.length === 0) {
|
||||
throw new Error("No chapter numbers parsed from --numbers.");
|
||||
}
|
||||
|
||||
return ordered;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves chapter rows by numeric title (e.g. Chapter 22).
|
||||
* @param {string[]} orderedNums - From {@link parseMangaChapterNumbersSpec}.
|
||||
* @param {import("./mangareadFetchChapters.js").ChapterLink[]} chapters - Full list.
|
||||
* @returns {Array<{ url: string, title: string, pickLabel: string, chapterNumber: string }>} Queue rows with URLs and labels.
|
||||
*/
|
||||
function queueItemsByChapterNumbers(orderedNums, chapters) {
|
||||
/** @type {Map<string, import("./mangareadFetchChapters.js").ChapterLink>} */
|
||||
const first = new Map();
|
||||
for (const ch of chapters) {
|
||||
const num = chapterNumberFromWpTitle(ch.title);
|
||||
if (num !== null && !first.has(num)) {
|
||||
first.set(num, ch);
|
||||
}
|
||||
}
|
||||
|
||||
return orderedNums.map((num) => {
|
||||
const ch = first.get(num);
|
||||
if (!ch) {
|
||||
throw new Error(
|
||||
`No chapter titled "Chapter ${num}" in the listing.` +
|
||||
" Use --list-only and check spelling (decimals like 160.5 must be listed explicitly).",
|
||||
);
|
||||
}
|
||||
return {
|
||||
url: ch.url,
|
||||
title: ch.title,
|
||||
pickLabel: `Ch.${num}`,
|
||||
chapterNumber: num,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints numbered chapters for manual `--select` choice.
|
||||
* @param {import("./mangareadFetchChapters.js").ChapterLink[]} chapters - Chapter rows.
|
||||
* @returns {void}
|
||||
*/
|
||||
function printChapterMenu(chapters) {
|
||||
const digits = String(chapters.length).length;
|
||||
for (let i = 0; i < chapters.length; i += 1) {
|
||||
const n = i + 1;
|
||||
const pad = String(n).padStart(digits, " ");
|
||||
const title = chapters[i].title.replace(/\s+/g, " ").trim();
|
||||
console.log(`${pad}. ${title}`);
|
||||
console.log(` ${chapters[i].url}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {number[]} indices1 - Sorted unique 1-based indices.
|
||||
* @param {import("./mangareadFetchChapters.js").ChapterLink[]} chapters - Full list.
|
||||
* @returns {Array<{ url: string, title: string, index: number, pickLabel: string }>} One item per requested index.
|
||||
*/
|
||||
function chaptersByIndices(indices1, chapters) {
|
||||
return indices1.map((idx) => ({
|
||||
index: idx,
|
||||
title: chapters[idx - 1].title,
|
||||
url: chapters[idx - 1].url,
|
||||
pickLabel: `#${String(idx)}`,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches chapter list, queues selected chapters, downloads one `.cbz` at a time.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function main() {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = parseCli(process.argv);
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : err);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (parsed.help) {
|
||||
console.log(`Usage:
|
||||
node src/downloadChaptersQueue.js <listing-page-url> [--list-only|-l]
|
||||
node src/downloadChaptersQueue.js <listing-page-url> --numbers|-n <spec> [--out|-o <dir>]
|
||||
node src/downloadChaptersQueue.js <listing-page-url> --select|-s <spec> [--out|-o <dir>]
|
||||
node src/downloadChaptersQueue.js <listing-page-url> --all [--out|-o <dir>]
|
||||
|
||||
Any chapter URL for that manga usually works as <listing-page-url> (same HTML includes the chapter list).
|
||||
|
||||
--numbers spec: chapter numbers from link titles ("Chapter 22"), e.g. 22,23,24 or 22-24
|
||||
Decimals (e.g. 160.5) must be listed explicitly; integer ranges expand to whole numbers only.
|
||||
|
||||
--select spec: 1-based row numbers on the **printed list** (oldest first; row 1 is Chapter 1),
|
||||
e.g. 1,3,5-10. This is the menu index, not a substring of the title (use --numbers for that).
|
||||
|
||||
Chapters are processed in list order for --select/--all, or in the order given for --numbers.
|
||||
Default --out: data/comics/<comic title>/ (under current working directory).`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!parsed.listingUrl) {
|
||||
console.error(
|
||||
"Usage: node src/downloadChaptersQueue.js <listing-page-url> [options]\nTry --help for details.",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
let chapters;
|
||||
let comicTitle;
|
||||
try {
|
||||
({ chapters, comicTitle } = await fetchWpMangaChapterLinks(parsed.listingUrl));
|
||||
chapters = chaptersInReadingOrder(chapters);
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : err);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Found ${String(chapters.length)} chapter link(s) on listing page.`);
|
||||
printChapterMenu(chapters);
|
||||
|
||||
if (parsed.listOnly) {
|
||||
return;
|
||||
}
|
||||
|
||||
const wantsAll = parsed.allChapters;
|
||||
const wantsSelect = Boolean(parsed.selectSpec);
|
||||
const wantsNumbers = Boolean(parsed.numbersSpec);
|
||||
const pickModes = [wantsAll, wantsSelect, wantsNumbers].filter(Boolean).length;
|
||||
if (pickModes === 0) {
|
||||
console.error(
|
||||
"Choose: --numbers <spec>, --select <spec>, or --all (or use --list-only to exit after listing).",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (pickModes > 1) {
|
||||
console.error("Use only one of --all, --select, or --numbers.");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
/** @type {Array<{ url: string, title: string, pickLabel: string }>} */
|
||||
let queue;
|
||||
try {
|
||||
if (wantsAll) {
|
||||
const indices = Array.from({ length: chapters.length }, (_, i) => i + 1);
|
||||
queue = chaptersByIndices(indices, chapters);
|
||||
} else if (wantsSelect) {
|
||||
const indices = parseChapterSelection(
|
||||
/** @type {string} */ (parsed.selectSpec),
|
||||
chapters.length,
|
||||
);
|
||||
queue = chaptersByIndices(indices, chapters);
|
||||
} else {
|
||||
const orderedNums = parseMangaChapterNumbersSpec(
|
||||
/** @type {string} */ (parsed.numbersSpec),
|
||||
);
|
||||
queue = queueItemsByChapterNumbers(orderedNums, chapters);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : err);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const outDir = path.join(parsed.outDir, sanitizeComicDirName(comicTitle));
|
||||
console.log(`Comic: ${comicTitle}`);
|
||||
console.log(`Output directory: ${outDir}`);
|
||||
|
||||
let failedCount = 0;
|
||||
let done = 0;
|
||||
for (const item of queue) {
|
||||
done += 1;
|
||||
const stem = stemForCbzFilename(item.url);
|
||||
const targetPath = path.join(outDir, `${stem}.cbz`);
|
||||
console.log(
|
||||
`\n[${String(done)}/${String(queue.length)}] ${item.pickLabel} ${item.title}`,
|
||||
);
|
||||
console.log(` → ${item.url}`);
|
||||
try {
|
||||
const { imageCount, outputPath } = await chapterUrlToCbz(item.url, targetPath);
|
||||
console.log(` ✓ ${String(imageCount)} page(s) → ${outputPath}`);
|
||||
} catch (err) {
|
||||
console.error(` ✗ ${err instanceof Error ? err.message : err}`);
|
||||
failedCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (failedCount > 0) {
|
||||
console.error(
|
||||
`\n${String(failedCount)} chapter(s) failed (${String(queue.length - failedCount)} succeeded).`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
chapterNumberFromWpTitle,
|
||||
chaptersByIndices,
|
||||
chaptersInReadingOrder,
|
||||
parseChapterSelection,
|
||||
parseCli,
|
||||
parseMangaChapterNumbersSpec,
|
||||
printChapterMenu,
|
||||
queueItemsByChapterNumbers,
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
const axios = require("axios");
|
||||
|
||||
/**
|
||||
* Turn upstream scrape/HTTP failures into a short UI-friendly message.
|
||||
* @param {unknown} err - Caught error (often Axios).
|
||||
* @param {string} [siteLabel] - e.g. "ManhwaSusu".
|
||||
* @returns {string}
|
||||
*/
|
||||
function formatUpstreamError(err, siteLabel) {
|
||||
const label = siteLabel || "Source site";
|
||||
const status = err && typeof err === "object" && "response" in err
|
||||
? /** @type {{ response?: { status?: number } }} */ (err).response?.status
|
||||
: undefined;
|
||||
const code =
|
||||
err && typeof err === "object" && "code" in err
|
||||
? /** @type {{ code?: string }} */ (err).code
|
||||
: undefined;
|
||||
const raw = err instanceof Error ? err.message : String(err);
|
||||
|
||||
if (status === 403) {
|
||||
return (
|
||||
`${label} blocked the request (HTTP 403), usually Cloudflare. ` +
|
||||
`Set FLARESOLVERR_URL (e.g. http://flaresolverr:8191/v1) and retry.`
|
||||
);
|
||||
}
|
||||
if (status === 404) {
|
||||
return `${label} returned 404 — check the series URL.`;
|
||||
}
|
||||
if (status === 429) {
|
||||
return `${label} rate-limited the request (HTTP 429). Try again later.`;
|
||||
}
|
||||
if (status && status >= 500) {
|
||||
return `${label} is unavailable (HTTP ${String(status)}).`;
|
||||
}
|
||||
if (code === "ECONNABORTED" || /timeout/iu.test(raw)) {
|
||||
return `${label} request timed out.`;
|
||||
}
|
||||
if (code === "ENOTFOUND" || code === "ECONNREFUSED") {
|
||||
return `Could not reach ${label}: ${raw}`;
|
||||
}
|
||||
if (/status code 403/iu.test(raw)) {
|
||||
return (
|
||||
`${label} blocked the request (HTTP 403), usually Cloudflare. ` +
|
||||
`Set FLARESOLVERR_URL (e.g. http://flaresolverr:8191/v1) and retry.`
|
||||
);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch HTML, optionally via FlareSolverr when `FLARESOLVERR_URL` is set.
|
||||
* @param {string} pageUrl - Absolute page URL.
|
||||
* @param {import("axios").AxiosRequestConfig} [directConfig] - Used for direct axios GET.
|
||||
* @returns {Promise<string>} HTML body.
|
||||
*/
|
||||
async function fetchHtmlWithOptionalFlareSolverr(pageUrl, directConfig = {}) {
|
||||
const flareBase = (process.env.FLARESOLVERR_URL || "").trim().replace(/\/+$/u, "");
|
||||
if (flareBase) {
|
||||
const endpoint = flareBase.endsWith("/v1") ? flareBase : `${flareBase}/v1`;
|
||||
const { data } = await axios.post(
|
||||
endpoint,
|
||||
{
|
||||
cmd: "request.get",
|
||||
url: pageUrl,
|
||||
maxTimeout: 60_000,
|
||||
},
|
||||
{
|
||||
timeout: 90_000,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
validateStatus: (s) => s >= 200 && s < 300,
|
||||
},
|
||||
);
|
||||
|
||||
if (!data || data.status !== "ok" || !data.solution) {
|
||||
const msg =
|
||||
data && typeof data.message === "string"
|
||||
? data.message
|
||||
: "FlareSolverr request failed";
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
const status = Number(data.solution.status) || 0;
|
||||
if (status < 200 || status >= 300) {
|
||||
const err = new Error(`Request failed with status code ${String(status)}`);
|
||||
/** @type {any} */ (err).response = { status };
|
||||
throw err;
|
||||
}
|
||||
|
||||
return String(data.solution.response || "");
|
||||
}
|
||||
|
||||
const { data, status } = await axios.get(pageUrl, {
|
||||
responseType: "text",
|
||||
timeout: 30_000,
|
||||
validateStatus: (s) => s >= 200 && s < 300,
|
||||
...directConfig,
|
||||
headers: {
|
||||
...(directConfig.headers || {}),
|
||||
},
|
||||
});
|
||||
|
||||
if (status !== 200) {
|
||||
throw new Error(`Unexpected HTTP status ${String(status)}`);
|
||||
}
|
||||
|
||||
return typeof data === "string" ? data : String(data);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchHtmlWithOptionalFlareSolverr,
|
||||
formatUpstreamError,
|
||||
};
|
||||
@@ -0,0 +1,19 @@
|
||||
const axios = require("axios");
|
||||
const { parse } = require("node-html-parser");
|
||||
|
||||
/**
|
||||
* Fetches HTML from a URL and returns the parsed document root.
|
||||
* @async
|
||||
* @param {string} url - Absolute URL to fetch.
|
||||
* @returns {Promise<object>} Parsed HTML root (node-html-parser tree).
|
||||
*/
|
||||
async function fetchAndParseHtml(url) {
|
||||
const { data } = await axios.get(url, {
|
||||
responseType: "text",
|
||||
validateStatus: (status) => status >= 200 && status < 300,
|
||||
timeout: 15_000,
|
||||
});
|
||||
return parse(data);
|
||||
}
|
||||
|
||||
module.exports = { fetchAndParseHtml };
|
||||
@@ -0,0 +1,426 @@
|
||||
const fs = require("fs/promises");
|
||||
const path = require("path");
|
||||
const axios = require("axios");
|
||||
const { parse } = require("node-html-parser");
|
||||
|
||||
const { fetchHtmlWithOptionalFlareSolverr } = require("./flaresolverr.js");
|
||||
|
||||
const PAGE_URL =
|
||||
"https://www.mangaread.org/manga/the-chaebeols-youngest-son/chapter-560/";
|
||||
const OUTPUT_HTML = path.join(__dirname, "..", "data", "mangaread-chapter-560.html");
|
||||
|
||||
/** @type {import("axios").AxiosRequestConfig} */
|
||||
const DEFAULT_GET_HTML = {
|
||||
responseType: "text",
|
||||
timeout: 30_000,
|
||||
headers: {
|
||||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
},
|
||||
validateStatus: (s) => s >= 200 && s < 300,
|
||||
};
|
||||
|
||||
/**
|
||||
* Madara / WP‑Manga often serves desktop HTML as manga info (`manga-page`) and the reader
|
||||
* (`reading-manga` + `img.wp-manga-chapter-img`) only when the client looks like mobile.
|
||||
* @type {import("axios").AxiosRequestConfig}
|
||||
*/
|
||||
const DEFAULT_GET_HTML_READER_MOBILE = {
|
||||
...DEFAULT_GET_HTML,
|
||||
headers: {
|
||||
...DEFAULT_GET_HTML.headers,
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 " +
|
||||
"(KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* @typedef {object} ChapterLink
|
||||
* @property {string} url - Absolute or relative chapter URL from the anchor `href`.
|
||||
* @property {string} title - Visible chapter label (trimmed).
|
||||
*/
|
||||
|
||||
/**
|
||||
* Returns every `a` inside `li.wp-manga-chapter` with URL and title.
|
||||
* @param {string} html - Raw HTML document.
|
||||
* @returns {ChapterLink[]} Extracted links in document order.
|
||||
*/
|
||||
function extractWpMangaChapters(html) {
|
||||
const root = parse(html);
|
||||
const items = root.querySelectorAll("li.wp-manga-chapter");
|
||||
|
||||
/** @type {ChapterLink[]} */
|
||||
const chapters = [];
|
||||
|
||||
for (const li of items) {
|
||||
const a = li.querySelector("a");
|
||||
if (!a) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const href = a.getAttribute("href");
|
||||
const title = (a.textContent || "").trim().replace(/\s+/g, " ");
|
||||
if (href) {
|
||||
chapters.push({ url: href.trim(), title });
|
||||
}
|
||||
}
|
||||
|
||||
return chapters;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} html - Listing or chapter page HTML.
|
||||
* @returns {string|null} Series title when found in markup.
|
||||
*/
|
||||
function extractWpMangaComicTitle(html) {
|
||||
const root = parse(html);
|
||||
|
||||
const og = root.querySelector('meta[property="og:title"]');
|
||||
if (og) {
|
||||
const t = (og.getAttribute("content") || "").trim();
|
||||
if (t.length > 0) {
|
||||
return t.replace(/\s+/g, " ");
|
||||
}
|
||||
}
|
||||
|
||||
const h1 = root.querySelector(".profile-manga .post-title h1");
|
||||
if (h1) {
|
||||
const t = (h1.textContent || "").trim().replace(/\s+/g, " ");
|
||||
if (t.length > 0) {
|
||||
return t;
|
||||
}
|
||||
}
|
||||
|
||||
const titleEl = root.querySelector("title");
|
||||
if (titleEl) {
|
||||
const m = /^Read\s+(.+?)\s+-\s+manga/iu.exec((titleEl.textContent || "").trim());
|
||||
if (m) {
|
||||
return m[1].trim().replace(/\s+/g, " ");
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} pageUrl - Manga or chapter URL.
|
||||
* @returns {string} Slug segment from `/manga/{slug}/`, or `"comic"`.
|
||||
*/
|
||||
function mangaSlugFromUrl(pageUrl) {
|
||||
try {
|
||||
const { pathname } = new URL(pageUrl);
|
||||
const parts = pathname.split("/").filter(Boolean);
|
||||
const mi = parts.indexOf("manga");
|
||||
if (mi !== -1 && parts[mi + 1] !== undefined) {
|
||||
return parts[mi + 1];
|
||||
}
|
||||
} catch {
|
||||
/* invalid URL */
|
||||
}
|
||||
return "comic";
|
||||
}
|
||||
|
||||
/**
|
||||
* Derives `https://host/manga/{slug}/` from a listing or chapter URL.
|
||||
* @param {string} pageUrl - Any URL under a manga (chapter or series page).
|
||||
* @returns {string} Normalized manga base URL with trailing slash.
|
||||
*/
|
||||
function mangaBaseUrlFromListingUrl(pageUrl) {
|
||||
const u = new URL(pageUrl);
|
||||
const parts = u.pathname.split("/").filter(Boolean);
|
||||
const mi = parts.indexOf("manga");
|
||||
if (mi === -1 || parts[mi + 1] === undefined) {
|
||||
throw new Error(
|
||||
'Could not find /manga/{slug}/ in URL; use a manga or chapter link from this site.',
|
||||
);
|
||||
}
|
||||
const slug = parts[mi + 1];
|
||||
return new URL(`/manga/${slug}/`, u.origin).href;
|
||||
}
|
||||
|
||||
/**
|
||||
* WP‑Manga / Madara fills `#manga-chapters-holder` via POST to `ajax/chapters/` when
|
||||
* the initial GET does not include chapter rows.
|
||||
* @param {string} mangaBaseUrl - Output of {@link mangaBaseUrlFromListingUrl}.
|
||||
* @param {string} refererUrl - Original page URL (for Referer).
|
||||
* @returns {Promise<string>} HTML fragment containing `li.wp-manga-chapter` items.
|
||||
*/
|
||||
async function fetchMangaChapterListAjaxHtml(mangaBaseUrl, refererUrl) {
|
||||
const base = mangaBaseUrl.endsWith("/") ? mangaBaseUrl : `${mangaBaseUrl}/`;
|
||||
const ajaxUrl = new URL("ajax/chapters/", base).href;
|
||||
const origin = new URL(refererUrl).origin;
|
||||
|
||||
const { data, status } = await axios.post(ajaxUrl, "", {
|
||||
...DEFAULT_GET_HTML,
|
||||
timeout: 45_000,
|
||||
headers: {
|
||||
...DEFAULT_GET_HTML.headers,
|
||||
Accept: "*/*",
|
||||
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
|
||||
Referer: refererUrl,
|
||||
Origin: origin,
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
},
|
||||
validateStatus: (s) => s >= 200 && s < 300,
|
||||
});
|
||||
|
||||
if (status !== 200) {
|
||||
throw new Error(`ajax/chapters/ returned HTTP ${String(status)}`);
|
||||
}
|
||||
|
||||
return typeof data === "string" ? data : String(data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches HTML for {@link pageUrl}.
|
||||
* @param {string} pageUrl - URL to load.
|
||||
* @param {object} [options] - Request shape.
|
||||
* @param {"desktop"|"reader"} [options.variant] - `reader` requests mobile reader markup when the theme separates info vs reader.
|
||||
* @returns {Promise<string>} Response body as text.
|
||||
*/
|
||||
async function fetchPageHtml(pageUrl, options = {}) {
|
||||
/** @type {import("axios").AxiosRequestConfig} */
|
||||
const cfg =
|
||||
options.variant === "reader" ? DEFAULT_GET_HTML_READER_MOBILE : DEFAULT_GET_HTML;
|
||||
|
||||
return fetchHtmlWithOptionalFlareSolverr(pageUrl, cfg);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a page and extracts each chapter anchor URL and title under `li.wp-manga-chapter`.
|
||||
* If the first response has no such rows (common on series-only URLs), POSTs to `ajax/chapters/`.
|
||||
* @param {string} pageUrl - Any page containing that markup (e.g. Mangaread chapter URL).
|
||||
* @returns {Promise<{ chapters: ChapterLink[], comicTitle: string }>} Chapter rows and series title.
|
||||
*/
|
||||
async function fetchWpMangaChapterLinks(pageUrl) {
|
||||
const html = await fetchPageHtml(pageUrl);
|
||||
let chapters = extractWpMangaChapters(html);
|
||||
|
||||
if (chapters.length === 0) {
|
||||
const base = mangaBaseUrlFromListingUrl(pageUrl);
|
||||
const ajaxHtml = await fetchMangaChapterListAjaxHtml(base, pageUrl);
|
||||
chapters = extractWpMangaChapters(ajaxHtml);
|
||||
}
|
||||
|
||||
const comicTitle =
|
||||
extractWpMangaComicTitle(html) || mangaSlugFromUrl(pageUrl);
|
||||
|
||||
return { chapters, comicTitle };
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {object} SavedChapterImage
|
||||
* @property {string} imageUrl - Absolute URL that was downloaded.
|
||||
* @property {string} filePath - Absolute path of the saved file.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {import("node-html-parser").HTMLElement} img - Image element.
|
||||
* @returns {string|null} First usable image URL attribute, or null.
|
||||
*/
|
||||
function pickChapterImageSrc(img) {
|
||||
/** @type {(string|null)[]} */
|
||||
const attrs = [
|
||||
img.getAttribute("data-src"),
|
||||
img.getAttribute("data-lazy-src"),
|
||||
img.getAttribute("data-original"),
|
||||
img.getAttribute("data-full-url"),
|
||||
img.getAttribute("src"),
|
||||
];
|
||||
|
||||
for (const raw of attrs) {
|
||||
if (raw && typeof raw === "string") {
|
||||
const t = raw.trim();
|
||||
if (t.length > 0 && !t.startsWith("data:")) {
|
||||
return t;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} pageUrl - Chapter or site URL (for folder name).
|
||||
* @returns {string} Filesystem-safe single path segment.
|
||||
*/
|
||||
function slugFromUrlForDir(pageUrl) {
|
||||
try {
|
||||
const { pathname } = new URL(pageUrl);
|
||||
const parts = pathname.split("/").filter(Boolean);
|
||||
const last = parts[parts.length - 1] || "chapter";
|
||||
const safe = last.replace(/[^a-z0-9._-]+/gi, "_").slice(0, 96);
|
||||
return safe.length > 0 ? safe : "chapter";
|
||||
} catch {
|
||||
return "chapter";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} contentType - `Content-Type` header value.
|
||||
* @returns {string} File extension including dot, or empty string.
|
||||
*/
|
||||
function extensionFromContentType(contentType) {
|
||||
if (!contentType || typeof contentType !== "string") {
|
||||
return "";
|
||||
}
|
||||
const ct = contentType.split(";")[0].trim().toLowerCase();
|
||||
/** @type {Record<string, string>} */
|
||||
const map = {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/jpg": ".jpg",
|
||||
"image/pjpeg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/webp": ".webp",
|
||||
"image/gif": ".gif",
|
||||
"image/avif": ".avif",
|
||||
"image/svg+xml": ".svg",
|
||||
};
|
||||
return map[ct] || "";
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} imageUrl - Absolute image URL.
|
||||
* @returns {string} Lowercase extension with dot, or empty string.
|
||||
*/
|
||||
function extensionFromImageUrl(imageUrl) {
|
||||
try {
|
||||
const p = new URL(imageUrl).pathname;
|
||||
const ext = path.extname(p).toLowerCase();
|
||||
if (ext.length > 0 && ext.length <= 8 && /^\.[a-z0-9]+$/u.test(ext)) {
|
||||
return ext;
|
||||
}
|
||||
} catch {
|
||||
/* invalid URL */
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects image URLs from `img.wp-manga-chapter-img` in document order.
|
||||
* @param {string} html - Document HTML.
|
||||
* @param {string} baseHref - URL used to resolve relative `src` values.
|
||||
* @returns {string[]} Absolute image URLs (deduped sequential).
|
||||
*/
|
||||
function collectWpMangaChapterImageUrls(html, baseHref) {
|
||||
const root = parse(html);
|
||||
const imgs = root.querySelectorAll("img.wp-manga-chapter-img");
|
||||
const baseUrl = new URL(baseHref);
|
||||
|
||||
/** @type {string[]} */
|
||||
const urls = [];
|
||||
|
||||
for (const img of imgs) {
|
||||
const raw = pickChapterImageSrc(img);
|
||||
if (!raw) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
urls.push(new URL(raw, baseUrl).href);
|
||||
} catch {
|
||||
/* skip bad href */
|
||||
}
|
||||
}
|
||||
|
||||
return urls;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a chapter page, selects `img.wp-manga-chapter-img`, downloads each image to disk.
|
||||
* Uses a mobile UA first (Madara reader HTML), then falls back to desktop if no images appear.
|
||||
*
|
||||
* Pure client-side lazy sites may still return an empty list.
|
||||
* @param {string} chapterPageUrl - Reader/chapter page URL.
|
||||
* @param {string} [outDir] - Directory for saved files (default: `data/chapter-images/<slug>`).
|
||||
* @returns {Promise<SavedChapterImage[]>} One record per image, in DOM order.
|
||||
*/
|
||||
async function downloadWpMangaChapterImages(chapterPageUrl, outDir) {
|
||||
let urls = collectWpMangaChapterImageUrls(
|
||||
await fetchPageHtml(chapterPageUrl, { variant: "reader" }),
|
||||
chapterPageUrl,
|
||||
);
|
||||
|
||||
if (urls.length === 0) {
|
||||
urls = collectWpMangaChapterImageUrls(await fetchPageHtml(chapterPageUrl), chapterPageUrl);
|
||||
}
|
||||
|
||||
const targetDir =
|
||||
typeof outDir === "string" && outDir.trim().length > 0
|
||||
? path.resolve(outDir.trim())
|
||||
: path.join(__dirname, "..", "data", "chapter-images", slugFromUrlForDir(chapterPageUrl));
|
||||
|
||||
await fs.mkdir(targetDir, { recursive: true });
|
||||
|
||||
/** @type {import("axios").AxiosRequestConfig} */
|
||||
const imageRequest = {
|
||||
responseType: "arraybuffer",
|
||||
timeout: 120_000,
|
||||
maxContentLength: 50 * 1024 * 1024,
|
||||
maxBodyLength: 50 * 1024 * 1024,
|
||||
headers: {
|
||||
...DEFAULT_GET_HTML.headers,
|
||||
Accept: "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
|
||||
Referer: chapterPageUrl,
|
||||
},
|
||||
validateStatus: (s) => s >= 200 && s < 300,
|
||||
};
|
||||
|
||||
/** @type {SavedChapterImage[]} */
|
||||
const saved = [];
|
||||
let index = 0;
|
||||
|
||||
for (const imageUrl of urls) {
|
||||
index += 1;
|
||||
const { data, headers } = await axios.get(imageUrl, imageRequest);
|
||||
const ct = headers["content-type"];
|
||||
let ext = extensionFromImageUrl(imageUrl);
|
||||
if (ext.length === 0) {
|
||||
ext = extensionFromContentType(typeof ct === "string" ? ct : "") || ".bin";
|
||||
}
|
||||
const filename = `page-${String(index).padStart(4, "0")}${ext}`;
|
||||
const filePath = path.join(targetDir, filename);
|
||||
await fs.writeFile(filePath, Buffer.from(data));
|
||||
saved.push({ imageUrl, filePath });
|
||||
}
|
||||
|
||||
return saved;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the page, writes HTML to disk, parses chapter rows.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function main() {
|
||||
const html = await fetchPageHtml(PAGE_URL);
|
||||
|
||||
await fs.mkdir(path.dirname(OUTPUT_HTML), { recursive: true });
|
||||
await fs.writeFile(OUTPUT_HTML, html, "utf8");
|
||||
|
||||
const chapters = extractWpMangaChapters(html);
|
||||
|
||||
console.log(`Saved HTML: ${OUTPUT_HTML}`);
|
||||
console.log(`Found ${String(chapters.length)} li.wp-manga-chapter link(s):`);
|
||||
console.log(JSON.stringify(chapters, null, 2));
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectWpMangaChapterImageUrls,
|
||||
downloadWpMangaChapterImages,
|
||||
extractWpMangaChapters,
|
||||
extractWpMangaComicTitle,
|
||||
fetchMangaChapterListAjaxHtml,
|
||||
fetchPageHtml,
|
||||
fetchWpMangaChapterLinks,
|
||||
mangaBaseUrlFromListingUrl,
|
||||
mangaSlugFromUrl,
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
const { parse } = require("node-html-parser");
|
||||
|
||||
const { CHAPTER_PATHNAME_PATTERN, MANHWASUSU_HOSTS } = require("./constants.js");
|
||||
const { fetchManhwaSusuHtml } = require("./io.js");
|
||||
|
||||
/**
|
||||
* @typedef {object} ChapterLink
|
||||
* @property {string} url - Absolute chapter URL.
|
||||
* @property {string} title - Chapter label (trimmed).
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {string} pathname - URL pathname.
|
||||
* @returns {boolean} Whether this path is a ManhwaSusu chapter reader URL.
|
||||
*/
|
||||
function isChapterPathname(pathname) {
|
||||
const p = pathname.replace(/\/+$/u, "");
|
||||
return CHAPTER_PATHNAME_PATTERN.test(p);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} url - Any http(s) URL.
|
||||
* @returns {boolean} True when hostname matches known ManhwaSusu hosts (hint only).
|
||||
*/
|
||||
function isLikelyManhwaSusuSite(url) {
|
||||
try {
|
||||
const { hostname } = new URL(url);
|
||||
return MANHWASUSU_HOSTS.some((h) => h === hostname.toLowerCase());
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Skip duplicate “First / Last chapter” buttons; the full list still contains the same targets.
|
||||
* @param {import("node-html-parser").HTMLElement} anchor - `<a>`.
|
||||
* @returns {boolean} True when the link text is only “First chapter” or “Last chapter”.
|
||||
*/
|
||||
function isFirstLastNavOnlyLink(anchor) {
|
||||
const t = (anchor.textContent || "").trim().replace(/\s+/gu, " ");
|
||||
return /^first chapter$/iu.test(t) || /^last chapter$/iu.test(t);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses chapter anchors from a series / listing HTML document.
|
||||
* @param {string} html - Raw HTML.
|
||||
* @param {string} pageUrl - Base URL for resolving relative `href`s.
|
||||
* @returns {ChapterLink[]} Unique links in **document order** (site lists newest first).
|
||||
*/
|
||||
function extractChapterLinksFromListingHtml(html, pageUrl) {
|
||||
const root = parse(html);
|
||||
const base = new URL(pageUrl);
|
||||
|
||||
/** @type {Map<string, ChapterLink>} */
|
||||
const byUrl = new Map();
|
||||
|
||||
for (const a of root.querySelectorAll("a[href]")) {
|
||||
const href = (a.getAttribute("href") || "").trim();
|
||||
if (href.length === 0 || href.startsWith("#")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let abs;
|
||||
try {
|
||||
abs = new URL(href, base).href;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
const { pathname } = new URL(abs);
|
||||
if (!isChapterPathname(pathname)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isFirstLastNavOnlyLink(a)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (byUrl.has(abs)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let title = "";
|
||||
const pEl = a.querySelector("p");
|
||||
if (pEl) {
|
||||
title = (pEl.textContent || "").trim().replace(/\s+/gu, " ");
|
||||
}
|
||||
if (title.length === 0) {
|
||||
title = (a.textContent || "").trim().replace(/\s+/gu, " ");
|
||||
}
|
||||
title = title
|
||||
.replace(/\s+\d+\s*(min|hour|day|week|mth|month|year)s?\s+ago$/iu, "")
|
||||
.trim();
|
||||
|
||||
const fallback = pathname.split("/").filter(Boolean).pop() || "chapter";
|
||||
byUrl.set(abs, { url: abs, title: title.length > 0 ? title : fallback });
|
||||
}
|
||||
|
||||
return [...byUrl.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} html - Series listing HTML.
|
||||
* @returns {string|null} Comic title when found in markup.
|
||||
*/
|
||||
function extractComicTitleFromListingHtml(html) {
|
||||
const root = parse(html);
|
||||
|
||||
const stripSiteSuffix = (raw) =>
|
||||
raw.replace(/\s*-\s*ManhwaSusu\s*$/iu, "").trim().replace(/\s+/gu, " ");
|
||||
|
||||
const og = root.querySelector('meta[property="og:title"]');
|
||||
if (og) {
|
||||
const t = stripSiteSuffix(og.getAttribute("content") || "");
|
||||
if (t.length > 0) {
|
||||
return t;
|
||||
}
|
||||
}
|
||||
|
||||
const titleEl = root.querySelector("title");
|
||||
if (titleEl) {
|
||||
const t = stripSiteSuffix(titleEl.textContent || "");
|
||||
if (t.length > 0) {
|
||||
return t;
|
||||
}
|
||||
}
|
||||
|
||||
for (const h1 of root.querySelectorAll("h1")) {
|
||||
const t = (h1.textContent || "").trim().replace(/\s+/gu, " ");
|
||||
if (
|
||||
t.length > 0 &&
|
||||
!/^ManhwaSusu$/iu.test(t) &&
|
||||
!/chapter\s*list/iu.test(t)
|
||||
) {
|
||||
return t;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} pageUrl - Series or chapter URL under `/read/{slug}/`.
|
||||
* @returns {string} Series slug segment, or `"comic"`.
|
||||
*/
|
||||
function seriesSlugFromListingUrl(pageUrl) {
|
||||
try {
|
||||
const { pathname } = new URL(pageUrl);
|
||||
const parts = pathname.split("/").filter(Boolean);
|
||||
const readIdx = parts.indexOf("read");
|
||||
if (readIdx !== -1 && parts[readIdx + 1] !== undefined) {
|
||||
return parts[readIdx + 1];
|
||||
}
|
||||
} catch {
|
||||
/* invalid URL */
|
||||
}
|
||||
return "comic";
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} listingPageUrl - e.g. `https://manhwasusu.com/read/{slug}/`
|
||||
* @returns {Promise<{ chapters: ChapterLink[], comicTitle: string }>} Chapter rows and series title.
|
||||
*/
|
||||
async function fetchChapterLinks(listingPageUrl) {
|
||||
const html = await fetchManhwaSusuHtml(listingPageUrl);
|
||||
const chapters = extractChapterLinksFromListingHtml(html, listingPageUrl);
|
||||
const comicTitle =
|
||||
extractComicTitleFromListingHtml(html) || seriesSlugFromListingUrl(listingPageUrl);
|
||||
return { chapters, comicTitle };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
extractChapterLinksFromListingHtml,
|
||||
extractComicTitleFromListingHtml,
|
||||
fetchChapterLinks,
|
||||
isChapterPathname,
|
||||
isFirstLastNavOnlyLink,
|
||||
isLikelyManhwaSusuSite,
|
||||
seriesSlugFromListingUrl,
|
||||
};
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* ManhwaSusu-specific URLs and DOM selectors. Adjust here if routes or markup change.
|
||||
* @see https://manhwasusu.com/read/{slug}/ — series + chapter index
|
||||
*/
|
||||
|
||||
/** Hostnames that this adapter is intended for (for friendly checks only). */
|
||||
const MANHWASUSU_HOSTS = /** @type {const} */ (["manhwasusu.com", "www.manhwasusu.com"]);
|
||||
|
||||
/** Canonical site origin used as Referer for CDN panel downloads. */
|
||||
const MANHWASUSU_ORIGIN = "https://manhwasusu.com/";
|
||||
|
||||
/** Default sample listing used by the fetch CLI when no URL is overridden. */
|
||||
const DEFAULT_LISTING_URL = "https://manhwasusu.com/read/switch-on-season-2/";
|
||||
|
||||
/** Path shape: `/read/{seriesSlug}/chapter-{id}/` (SSR uses trailing slash optional). */
|
||||
const CHAPTER_PATHNAME_PATTERN = /^\/read\/[^/]+\/chapter-[^/]+$/iu;
|
||||
|
||||
/**
|
||||
* Reader lazily fills `src` from `data-src`; class `imgku` wraps panel images on chapter pages.
|
||||
* @type {readonly [string, ...string[]]}
|
||||
*/
|
||||
const READER_IMG_SELECTORS = ["img.imgku"];
|
||||
|
||||
/**
|
||||
* Browser-like headers for CDN image GETs (captured from manhwasusu.com reader → manhwature.com).
|
||||
* Referer must be the site origin; chapter URLs are often rejected by the CDN.
|
||||
* @type {Readonly<Record<string, string>>}
|
||||
*/
|
||||
const CDN_IMAGE_HEADERS = {
|
||||
Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9,id;q=0.8",
|
||||
"Cache-Control": "no-cache",
|
||||
Pragma: "no-cache",
|
||||
"Sec-CH-UA":
|
||||
'"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"',
|
||||
"Sec-CH-UA-Mobile": "?0",
|
||||
"Sec-CH-UA-Platform": '"Windows"',
|
||||
"Sec-Fetch-Dest": "image",
|
||||
"Sec-Fetch-Mode": "no-cors",
|
||||
"Sec-Fetch-Site": "cross-site",
|
||||
"Sec-Fetch-Storage-Access": "active",
|
||||
Referer: MANHWASUSU_ORIGIN,
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
||||
"(KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0",
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
CDN_IMAGE_HEADERS,
|
||||
CHAPTER_PATHNAME_PATTERN,
|
||||
DEFAULT_LISTING_URL,
|
||||
MANHWASUSU_HOSTS,
|
||||
MANHWASUSU_ORIGIN,
|
||||
READER_IMG_SELECTORS,
|
||||
};
|
||||
@@ -0,0 +1,169 @@
|
||||
const fs = require("fs/promises");
|
||||
const path = require("path");
|
||||
const os = require("os");
|
||||
const AdmZip = require("adm-zip");
|
||||
|
||||
const { downloadChapterImages } = require("./images.js");
|
||||
const { extractComicTitleFromListingHtml, seriesSlugFromListingUrl } = require("./chapters.js");
|
||||
const { fetchManhwaSusuHtml } = require("./io.js");
|
||||
|
||||
/**
|
||||
* `/read/my-series/chapter-12/` → `my-series-chapter-12`.
|
||||
* @param {string} chapterPageUrl - Full chapter URL.
|
||||
* @returns {string} Suggested `.cbz` stem.
|
||||
*/
|
||||
function stemForManhwaSusuChapterCbz(chapterPageUrl) {
|
||||
try {
|
||||
const { pathname } = new URL(chapterPageUrl);
|
||||
const segs = pathname.replace(/\/+$/u, "").split("/").filter(Boolean);
|
||||
const readIdx = segs.indexOf("read");
|
||||
if (readIdx !== -1 && segs.length >= readIdx + 3) {
|
||||
const slug = segs[readIdx + 1];
|
||||
const chapSeg = segs[readIdx + 2];
|
||||
return `${slug}-${chapSeg}`;
|
||||
}
|
||||
if (segs.length >= 2) {
|
||||
return segs.slice(-2).join("-");
|
||||
}
|
||||
return segs.at(-1) || "chapter";
|
||||
} catch {
|
||||
return "chapter";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} stem - File base segment.
|
||||
* @returns {string} Sanitized stem.
|
||||
*/
|
||||
function sanitizeFsStem(stem) {
|
||||
const cleaned = stem.replace(/[^a-z0-9._-]+/giu, "_").replace(/^_+|_+$/gu, "");
|
||||
return cleaned.length > 0 ? cleaned.slice(0, 180) : "chapter";
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} name - Human-readable comic title.
|
||||
* @returns {string} Filesystem-safe directory name.
|
||||
*/
|
||||
function sanitizeComicDirName(name) {
|
||||
const collapsed = name.trim().replace(/\s+/gu, " ");
|
||||
const cleaned = [...collapsed]
|
||||
.filter((ch) => {
|
||||
const code = ch.charCodeAt(0);
|
||||
if (code < 32 || code === 127) {
|
||||
return false;
|
||||
}
|
||||
return !/[<>:"/\\|?*]/.test(ch);
|
||||
})
|
||||
.join("")
|
||||
.replace(/\.+$/g, "")
|
||||
.trim();
|
||||
return cleaned.length > 0 ? cleaned.slice(0, 180) : "comic";
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string|undefined} cliOut - Positional output path/dir.
|
||||
* @param {string} defaultStem - When `cliOut` is omitted or directory-like.
|
||||
* @param {string} [comicTitle] - When set, default output goes under this subdirectory.
|
||||
* @returns {string} Absolute `.cbz` path.
|
||||
*/
|
||||
function resolveCliOutputPath(cliOut, defaultStem, comicTitle) {
|
||||
const stem = sanitizeFsStem(defaultStem);
|
||||
const baseDir =
|
||||
typeof comicTitle === "string" && comicTitle.trim().length > 0
|
||||
? path.join(
|
||||
process.cwd(),
|
||||
"data",
|
||||
"manhwasusu",
|
||||
"comics",
|
||||
sanitizeComicDirName(comicTitle),
|
||||
)
|
||||
: path.join(process.cwd(), "data", "manhwasusu", "comics");
|
||||
if (cliOut === undefined || cliOut.trim() === "") {
|
||||
return path.join(baseDir, `${stem}.cbz`);
|
||||
}
|
||||
const trimmed = cliOut.trim();
|
||||
const abs = path.resolve(trimmed);
|
||||
if (abs.toLowerCase().endsWith(".cbz")) {
|
||||
return abs;
|
||||
}
|
||||
return path.join(abs, `${stem}.cbz`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads ManhwaSusu chapter panels to temp storage, packs `.cbz`, removes temp folder.
|
||||
* @param {string} chapterUrl - Chapter reader URL.
|
||||
* @param {string} outputCbzPath - Destination `.cbz`.
|
||||
* @returns {Promise<{ imageCount: number, outputPath: string }>} Page count and written archive path.
|
||||
*/
|
||||
async function chapterUrlToCbz(chapterUrl, outputCbzPath) {
|
||||
const absOut = outputCbzPath.toLowerCase().endsWith(".cbz")
|
||||
? path.resolve(outputCbzPath)
|
||||
: `${path.resolve(outputCbzPath)}.cbz`;
|
||||
|
||||
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "cxyz-manhwasusu-chapter-"));
|
||||
|
||||
try {
|
||||
const saved = await downloadChapterImages(chapterUrl, tmpRoot);
|
||||
|
||||
if (saved.length === 0) {
|
||||
throw new Error("No chapter images were found for this ManhwaSusu URL.");
|
||||
}
|
||||
|
||||
await fs.mkdir(path.dirname(absOut), { recursive: true });
|
||||
|
||||
const zip = new AdmZip();
|
||||
for (const { filePath } of saved) {
|
||||
zip.addLocalFile(filePath, "", path.basename(filePath));
|
||||
}
|
||||
zip.writeZip(absOut);
|
||||
|
||||
return { imageCount: saved.length, outputPath: absOut };
|
||||
} finally {
|
||||
await fs.rm(tmpRoot, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI: `node src/manhwasusu/downloadChapterCbz.js <chapterUrl> [.cbz|dir]`
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function main() {
|
||||
const chapterUrl = process.argv[2];
|
||||
if (!chapterUrl || chapterUrl.startsWith("-")) {
|
||||
console.error(
|
||||
"Usage: node src/manhwasusu/downloadChapterCbz.js <chapterUrl> [output.cbz|outputDirectory]",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const stem = stemForManhwaSusuChapterCbz(chapterUrl);
|
||||
let comicTitle = seriesSlugFromListingUrl(chapterUrl);
|
||||
try {
|
||||
const html = await fetchManhwaSusuHtml(chapterUrl);
|
||||
comicTitle = extractComicTitleFromListingHtml(html) || comicTitle;
|
||||
} catch {
|
||||
/* keep slug fallback */
|
||||
}
|
||||
const target = resolveCliOutputPath(process.argv[3], stem, comicTitle);
|
||||
|
||||
try {
|
||||
const { imageCount, outputPath } = await chapterUrlToCbz(chapterUrl, target);
|
||||
console.log(`Wrote ${String(imageCount)} page(s) → ${outputPath}`);
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : err);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
chapterUrlToCbz,
|
||||
resolveCliOutputPath,
|
||||
sanitizeComicDirName,
|
||||
sanitizeFsStem,
|
||||
stemForManhwaSusuChapterCbz,
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
main();
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
const path = require("path");
|
||||
|
||||
const { fetchChapterLinks, isLikelyManhwaSusuSite } = require("./chapters.js");
|
||||
const { stemForManhwaSusuChapterCbz, chapterUrlToCbz, sanitizeComicDirName } =
|
||||
require("./downloadChapterCbz.js");
|
||||
|
||||
/**
|
||||
* @typedef {object} ParsedCli
|
||||
* @property {boolean} [help] - Print usage and exit when true.
|
||||
* @property {string|undefined} listingUrl - Series listing URL (required unless help).
|
||||
* @property {boolean} listOnly - Only print the chapter menu.
|
||||
* @property {boolean} allChapters - Queue every chapter after reversal to reading order.
|
||||
* @property {string|undefined} selectSpec - `--select` string (1-based menu indices).
|
||||
* @property {string|undefined} numbersSpec - `--numbers` string (chapter numbers from titles).
|
||||
* @property {string} outDir - Output directory for `.cbz` files.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {string[]} argv - `process.argv`.
|
||||
* @returns {ParsedCli} Parsed flags and positional URL.
|
||||
*/
|
||||
function parseCli(argv) {
|
||||
/** @type {ParsedCli} */
|
||||
const result = {
|
||||
help: false,
|
||||
listingUrl: undefined,
|
||||
listOnly: false,
|
||||
allChapters: false,
|
||||
selectSpec: undefined,
|
||||
numbersSpec: undefined,
|
||||
outDir: path.join(process.cwd(), "data", "manhwasusu", "comics"),
|
||||
};
|
||||
|
||||
for (let i = 2; i < argv.length; i += 1) {
|
||||
const a = argv[i];
|
||||
if (a === "--help" || a === "-h") {
|
||||
result.help = true;
|
||||
} else if (a === "--list-only" || a === "-l") {
|
||||
result.listOnly = true;
|
||||
} else if (a === "--all") {
|
||||
result.allChapters = true;
|
||||
} else if (a === "--select" || a === "-s") {
|
||||
const next = argv[i + 1];
|
||||
if (next === undefined || next.startsWith("-")) {
|
||||
throw new Error("Missing value for --select");
|
||||
}
|
||||
result.selectSpec = next;
|
||||
i += 1;
|
||||
} else if (a === "--numbers" || a === "-n") {
|
||||
const next = argv[i + 1];
|
||||
if (next === undefined || next.startsWith("-")) {
|
||||
throw new Error("Missing value for --numbers");
|
||||
}
|
||||
result.numbersSpec = next;
|
||||
i += 1;
|
||||
} else if (a === "--out" || a === "-o") {
|
||||
const next = argv[i + 1];
|
||||
if (next === undefined || next.startsWith("-")) {
|
||||
throw new Error("Missing value for --out");
|
||||
}
|
||||
result.outDir = path.resolve(next);
|
||||
i += 1;
|
||||
} else if (!a.startsWith("-")) {
|
||||
if (result.listingUrl) {
|
||||
throw new Error(`Unexpected extra argument: ${a}`);
|
||||
}
|
||||
result.listingUrl = a;
|
||||
} else {
|
||||
throw new Error(`Unknown option: ${a}`);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Listing HTML is newest-first; reverse so `--select` line 1 is Chapter 1.
|
||||
* @param {import("./chapters.js").ChapterLink[]} chapters - Site order.
|
||||
* @returns {import("./chapters.js").ChapterLink[]} New array, oldest chapter first.
|
||||
*/
|
||||
function chaptersInReadingOrder(chapters) {
|
||||
return [...chapters].reverse();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} spec - `--select` value.
|
||||
* @param {number} count - Chapter count.
|
||||
* @returns {number[]} Sorted unique 1-based row indices into the reversed list.
|
||||
*/
|
||||
function parseChapterSelection(spec, count) {
|
||||
if (count <= 0) {
|
||||
throw new Error("Chapter list is empty; cannot select indices.");
|
||||
}
|
||||
|
||||
const trimmed = spec.trim();
|
||||
if (trimmed.length === 0) {
|
||||
throw new Error("--select value is empty.");
|
||||
}
|
||||
|
||||
/** @type {Set<number>} */
|
||||
const seen = new Set();
|
||||
|
||||
for (const rawPart of trimmed.split(",")) {
|
||||
const part = rawPart.trim();
|
||||
if (part.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (/^\d+$/u.test(part)) {
|
||||
const n = Number.parseInt(part, 10);
|
||||
if (n < 1 || n > count) {
|
||||
throw new Error(
|
||||
`Index ${String(n)} is out of range (valid: 1–${String(count)}).`,
|
||||
);
|
||||
}
|
||||
seen.add(n);
|
||||
continue;
|
||||
}
|
||||
|
||||
const rangeMatch = /^(\d+)\s*-\s*(\d+)$/u.exec(part);
|
||||
if (rangeMatch) {
|
||||
const lo = Number.parseInt(rangeMatch[1], 10);
|
||||
const hi = Number.parseInt(rangeMatch[2], 10);
|
||||
const a = Math.min(lo, hi);
|
||||
const b = Math.max(lo, hi);
|
||||
for (let j = a; j <= b; j += 1) {
|
||||
if (j < 1 || j > count) {
|
||||
throw new Error(
|
||||
`Range ${part} includes invalid index ${String(j)} (valid: 1–${String(count)}).`,
|
||||
);
|
||||
}
|
||||
seen.add(j);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`Invalid --select segment: "${part}"`);
|
||||
}
|
||||
|
||||
if (seen.size === 0) {
|
||||
throw new Error("No indices parsed from --select.");
|
||||
}
|
||||
|
||||
return [...seen].sort((x, y) => x - y);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} title - e.g. `Chapter 22`.
|
||||
* @returns {string|null} Chapter number string from title, or null.
|
||||
*/
|
||||
function chapterNumberFromListingTitle(title) {
|
||||
const m = /^Chapter\s+([\d.]+)\s*$/iu.exec(title.trim().replace(/\s+/gu, " "));
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} spec - `--numbers` argument.
|
||||
* @returns {string[]} Ordered unique chapter number strings as entered.
|
||||
*/
|
||||
function parseChapterNumbersSpec(spec) {
|
||||
const trimmed = spec.trim();
|
||||
if (trimmed.length === 0) {
|
||||
throw new Error("--numbers value is empty.");
|
||||
}
|
||||
|
||||
/** @type {string[]} */
|
||||
const ordered = [];
|
||||
/** @type {Set<string>} */
|
||||
const seen = new Set();
|
||||
|
||||
for (const rawPart of trimmed.split(",")) {
|
||||
const part = rawPart.trim();
|
||||
if (part.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const rangeMatch = /^(\d+)\s*-\s*(\d+)$/u.exec(part);
|
||||
if (rangeMatch) {
|
||||
const lo = Number.parseInt(rangeMatch[1], 10);
|
||||
const hi = Number.parseInt(rangeMatch[2], 10);
|
||||
const a = Math.min(lo, hi);
|
||||
const b = Math.max(lo, hi);
|
||||
for (let n = a; n <= b; n += 1) {
|
||||
const k = String(n);
|
||||
if (!seen.has(k)) {
|
||||
seen.add(k);
|
||||
ordered.push(k);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (/^\d+(\.\d+)?$/u.test(part)) {
|
||||
if (!seen.has(part)) {
|
||||
seen.add(part);
|
||||
ordered.push(part);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`Invalid --numbers segment: "${part}"`);
|
||||
}
|
||||
|
||||
if (ordered.length === 0) {
|
||||
throw new Error("No chapter numbers parsed from --numbers.");
|
||||
}
|
||||
|
||||
return ordered;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string[]} orderedNums - Chapter numbers requested.
|
||||
* @param {import("./chapters.js").ChapterLink[]} chapters - Ordered oldest→newest after reverse.
|
||||
* @returns {Array<{ url: string, title: string, pickLabel: string, chapterNumber: string }>}
|
||||
* Rows to download.
|
||||
*/
|
||||
function queueItemsByChapterNumbers(orderedNums, chapters) {
|
||||
/** @type {Map<string, import("./chapters.js").ChapterLink>} */
|
||||
const first = new Map();
|
||||
for (const ch of chapters) {
|
||||
const num = chapterNumberFromListingTitle(ch.title);
|
||||
if (num !== null && !first.has(num)) {
|
||||
first.set(num, ch);
|
||||
}
|
||||
}
|
||||
|
||||
return orderedNums.map((num) => {
|
||||
const ch = first.get(num);
|
||||
if (!ch) {
|
||||
throw new Error(
|
||||
`No chapter titled "Chapter ${num}" in the ManhwaSusu listing. ` +
|
||||
"Use --list-only and check titles.",
|
||||
);
|
||||
}
|
||||
return {
|
||||
url: ch.url,
|
||||
title: ch.title,
|
||||
pickLabel: `Ch.${num}`,
|
||||
chapterNumber: num,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first list.
|
||||
* @returns {void}
|
||||
*/
|
||||
function printChapterMenu(chapters) {
|
||||
const digits = String(chapters.length).length;
|
||||
for (let i = 0; i < chapters.length; i += 1) {
|
||||
const n = i + 1;
|
||||
const pad = String(n).padStart(digits, " ");
|
||||
const title = chapters[i].title.replace(/\s+/gu, " ").trim();
|
||||
console.log(`${pad}. ${title}`);
|
||||
console.log(` ${chapters[i].url}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {number[]} indices1 - Sorted 1-based indices.
|
||||
* @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first.
|
||||
*/
|
||||
/**
|
||||
* @param {number[]} indices1 - Sorted 1-based indices.
|
||||
* @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first chapter list.
|
||||
* @returns {Array<{ index: number, title: string, url: string, pickLabel: string }>} Queue rows.
|
||||
*/
|
||||
function chaptersByIndices(indices1, chapters) {
|
||||
return indices1.map((idx) => ({
|
||||
index: idx,
|
||||
title: chapters[idx - 1].title,
|
||||
url: chapters[idx - 1].url,
|
||||
pickLabel: `#${String(idx)}`,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* CLI entry: fetch listing, optionally build `.cbz` archives.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function main() {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = parseCli(process.argv);
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : err);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (parsed.help) {
|
||||
console.log(`ManhwaSusu chapter queue (dedicated adapter; URLs like /read/{slug}/):
|
||||
|
||||
Usage:
|
||||
node src/manhwasusu/downloadChaptersQueue.js <listing-url> [--list-only|-l]
|
||||
node src/manhwasusu/downloadChaptersQueue.js <listing-url> --numbers|-n <spec> [--out|-o <dir>]
|
||||
node src/manhwasusu/downloadChaptersQueue.js <listing-url> --select|-s <spec> [--out|-o <dir>]
|
||||
node src/manhwasusu/downloadChaptersQueue.js <listing-url> --all [--out|-o <dir>]
|
||||
|
||||
Listing URL: series page ending in /read/{slug}/ (or chapter URL — site embeds chapter list markup).
|
||||
|
||||
Default --out: data/manhwasusu/comics/<comic title>/
|
||||
|
||||
--numbers: titles like "Chapter 22", e.g. 22,23,24 or 22-24
|
||||
--select: 1-based rows on printed list after reversal (line 1 = Chapter 1)
|
||||
`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!parsed.listingUrl) {
|
||||
console.error(
|
||||
"Usage: node src/manhwasusu/downloadChaptersQueue.js <listing-url> [options]",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isLikelyManhwaSusuSite(parsed.listingUrl)) {
|
||||
console.warn(
|
||||
"Warning: URL hostname does not look like ManhwaSusu; parsers may fail if structure differs.",
|
||||
);
|
||||
}
|
||||
|
||||
let chapters;
|
||||
let comicTitle;
|
||||
try {
|
||||
({ chapters, comicTitle } = await fetchChapterLinks(parsed.listingUrl));
|
||||
chapters = chaptersInReadingOrder(chapters);
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : err);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Found ${String(chapters.length)} ManhwaSusu chapter link(s).`);
|
||||
printChapterMenu(chapters);
|
||||
|
||||
if (parsed.listOnly) {
|
||||
return;
|
||||
}
|
||||
|
||||
const wantsAll = parsed.allChapters;
|
||||
const wantsSelect = Boolean(parsed.selectSpec);
|
||||
const wantsNumbers = Boolean(parsed.numbersSpec);
|
||||
const pickModes = [wantsAll, wantsSelect, wantsNumbers].filter(Boolean).length;
|
||||
if (pickModes === 0) {
|
||||
console.error("Choose --numbers, --select, or --all (or --list-only).");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (pickModes > 1) {
|
||||
console.error("Use only one of --all, --select, or --numbers.");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
/** @type {Array<{ url: string, title: string, pickLabel: string }>} */
|
||||
let queue;
|
||||
try {
|
||||
if (wantsAll) {
|
||||
const indices = Array.from({ length: chapters.length }, (_, i) => i + 1);
|
||||
queue = chaptersByIndices(indices, chapters);
|
||||
} else if (wantsSelect) {
|
||||
const indices = parseChapterSelection(
|
||||
/** @type {string} */ (parsed.selectSpec),
|
||||
chapters.length,
|
||||
);
|
||||
queue = chaptersByIndices(indices, chapters);
|
||||
} else {
|
||||
const orderedNums = parseChapterNumbersSpec(
|
||||
/** @type {string} */ (parsed.numbersSpec),
|
||||
);
|
||||
queue = queueItemsByChapterNumbers(orderedNums, chapters);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : err);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const outDir = path.join(parsed.outDir, sanitizeComicDirName(comicTitle));
|
||||
console.log(`Comic: ${comicTitle}`);
|
||||
console.log(`Output directory: ${outDir}`);
|
||||
|
||||
let failedCount = 0;
|
||||
let done = 0;
|
||||
for (const item of queue) {
|
||||
done += 1;
|
||||
const stem = stemForManhwaSusuChapterCbz(item.url);
|
||||
const targetPath = path.join(outDir, `${stem}.cbz`);
|
||||
console.log(`\n[${String(done)}/${String(queue.length)}] ${item.pickLabel} ${item.title}`);
|
||||
console.log(` → ${item.url}`);
|
||||
try {
|
||||
const { imageCount, outputPath } = await chapterUrlToCbz(item.url, targetPath);
|
||||
console.log(` ✓ ${String(imageCount)} page(s) → ${outputPath}`);
|
||||
} catch (err) {
|
||||
console.error(` ✗ ${err instanceof Error ? err.message : err}`);
|
||||
failedCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (failedCount > 0) {
|
||||
console.error(
|
||||
`\n${String(failedCount)} chapter(s) failed (${String(queue.length - failedCount)} succeeded).`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
chapterNumberFromListingTitle,
|
||||
chaptersByIndices,
|
||||
chaptersInReadingOrder,
|
||||
parseChapterNumbersSpec,
|
||||
parseChapterSelection,
|
||||
parseCli,
|
||||
printChapterMenu,
|
||||
queueItemsByChapterNumbers,
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
const fs = require("fs/promises");
|
||||
const path = require("path");
|
||||
|
||||
const { DEFAULT_LISTING_URL } = require("./constants.js");
|
||||
const { extractChapterLinksFromListingHtml } = require("./chapters.js");
|
||||
const { fetchManhwaSusuHtml } = require("./io.js");
|
||||
|
||||
const OUTPUT_HTML = path.join(
|
||||
__dirname,
|
||||
"..",
|
||||
"..",
|
||||
"data",
|
||||
"manhwasusu",
|
||||
"debug-listing.html",
|
||||
);
|
||||
|
||||
/**
|
||||
* CLI: snapshot listing HTML + print chapter JSON. Optional `MANHWASUSU_LISTING_URL` env overrides default.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function main() {
|
||||
const pageUrl = process.env.MANHWASUSU_LISTING_URL || DEFAULT_LISTING_URL;
|
||||
const html = await fetchManhwaSusuHtml(pageUrl);
|
||||
|
||||
await fs.mkdir(path.dirname(OUTPUT_HTML), { recursive: true });
|
||||
await fs.writeFile(OUTPUT_HTML, html, "utf8");
|
||||
|
||||
const chapters = extractChapterLinksFromListingHtml(html, pageUrl);
|
||||
|
||||
console.log(`Source: ${pageUrl}`);
|
||||
console.log(`Saved HTML: ${OUTPUT_HTML}`);
|
||||
console.log(
|
||||
`Found ${String(chapters.length)} chapter link(s) (newest-first on page; reverse for reading order):`,
|
||||
);
|
||||
console.log(JSON.stringify(chapters, null, 2));
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { main };
|
||||
@@ -0,0 +1,100 @@
|
||||
const fs = require("fs/promises");
|
||||
const path = require("path");
|
||||
const axios = require("axios");
|
||||
const { parse } = require("node-html-parser");
|
||||
|
||||
const { READER_IMG_SELECTORS } = require("./constants.js");
|
||||
const {
|
||||
extensionFromContentType,
|
||||
extensionFromImageUrl,
|
||||
fetchManhwaSusuHtml,
|
||||
imageGetConfig,
|
||||
pickReaderImgSrc,
|
||||
slugFromChapterUrl,
|
||||
} = require("./io.js");
|
||||
|
||||
/**
|
||||
* Collects panel URLs from chapter HTML (lazy `data-src` on reader images).
|
||||
* @param {string} html - Chapter document.
|
||||
* @param {string} baseHref - Used to resolve relative attributes.
|
||||
* @returns {string[]} Absolute image URLs in DOM order.
|
||||
*/
|
||||
function collectChapterImageUrls(html, baseHref) {
|
||||
const root = parse(html);
|
||||
const baseUrl = new URL(baseHref);
|
||||
const selector = READER_IMG_SELECTORS.join(", ");
|
||||
|
||||
/** @type {string[]} */
|
||||
const urls = [];
|
||||
|
||||
for (const img of root.querySelectorAll(selector)) {
|
||||
const raw = pickReaderImgSrc(img);
|
||||
if (!raw) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
urls.push(new URL(raw, baseUrl).href);
|
||||
} catch {
|
||||
/* skip bad href */
|
||||
}
|
||||
}
|
||||
|
||||
return urls;
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {object} SavedChapterImage
|
||||
* @property {string} imageUrl - Remote URL that was downloaded.
|
||||
* @property {string} filePath - Path of the saved file.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Downloads reader panels into `data/manhwasusu/chapters/{slug}/` by default.
|
||||
* @param {string} chapterPageUrl - Full `/read/{slug}/chapter-N/` URL.
|
||||
* @param {string} [outDir] - Override output directory.
|
||||
* @returns {Promise<SavedChapterImage[]>} One record per saved panel file.
|
||||
*/
|
||||
async function downloadChapterImages(chapterPageUrl, outDir) {
|
||||
const html = await fetchManhwaSusuHtml(chapterPageUrl);
|
||||
const urls = collectChapterImageUrls(html, chapterPageUrl);
|
||||
|
||||
const targetDir =
|
||||
typeof outDir === "string" && outDir.trim().length > 0
|
||||
? path.resolve(outDir.trim())
|
||||
: path.join(
|
||||
path.join(__dirname, "..", ".."),
|
||||
"data",
|
||||
"manhwasusu",
|
||||
"chapters",
|
||||
slugFromChapterUrl(chapterPageUrl),
|
||||
);
|
||||
|
||||
await fs.mkdir(targetDir, { recursive: true });
|
||||
|
||||
const cfg = imageGetConfig(chapterPageUrl);
|
||||
|
||||
/** @type {SavedChapterImage[]} */
|
||||
const saved = [];
|
||||
let index = 0;
|
||||
|
||||
for (const imageUrl of urls) {
|
||||
index += 1;
|
||||
const { data, headers } = await axios.get(imageUrl, cfg);
|
||||
const ct = headers["content-type"];
|
||||
let ext = extensionFromImageUrl(imageUrl);
|
||||
if (ext.length === 0) {
|
||||
ext = extensionFromContentType(typeof ct === "string" ? ct : "") || ".bin";
|
||||
}
|
||||
const filename = `page-${String(index).padStart(4, "0")}${ext}`;
|
||||
const filePath = path.join(targetDir, filename);
|
||||
await fs.writeFile(filePath, Buffer.from(data));
|
||||
saved.push({ imageUrl, filePath });
|
||||
}
|
||||
|
||||
return saved;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectChapterImageUrls,
|
||||
downloadChapterImages,
|
||||
};
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* ManhwaSusu adapter: HTTP, listing/chapter parsing, image download, and CBZ CLIs live under this folder.
|
||||
* Tweak {@link ./constants.js} when routes or reader markup change.
|
||||
*/
|
||||
|
||||
const {
|
||||
CDN_IMAGE_HEADERS,
|
||||
CHAPTER_PATHNAME_PATTERN,
|
||||
DEFAULT_LISTING_URL,
|
||||
MANHWASUSU_HOSTS,
|
||||
MANHWASUSU_ORIGIN,
|
||||
READER_IMG_SELECTORS,
|
||||
} = require("./constants.js");
|
||||
|
||||
const {
|
||||
extractChapterLinksFromListingHtml,
|
||||
extractComicTitleFromListingHtml,
|
||||
fetchChapterLinks,
|
||||
isChapterPathname,
|
||||
isFirstLastNavOnlyLink,
|
||||
isLikelyManhwaSusuSite,
|
||||
seriesSlugFromListingUrl,
|
||||
} = require("./chapters.js");
|
||||
|
||||
const {
|
||||
collectChapterImageUrls,
|
||||
downloadChapterImages,
|
||||
} = require("./images.js");
|
||||
|
||||
const {
|
||||
DEFAULT_GET_HTML,
|
||||
extensionFromContentType,
|
||||
extensionFromImageUrl,
|
||||
fetchManhwaSusuHtml,
|
||||
imageGetConfig,
|
||||
pickReaderImgSrc,
|
||||
slugFromChapterUrl,
|
||||
} = require("./io.js");
|
||||
|
||||
const {
|
||||
chapterUrlToCbz,
|
||||
resolveCliOutputPath,
|
||||
sanitizeComicDirName,
|
||||
sanitizeFsStem,
|
||||
stemForManhwaSusuChapterCbz,
|
||||
} = require("./downloadChapterCbz.js");
|
||||
|
||||
module.exports = {
|
||||
CDN_IMAGE_HEADERS,
|
||||
CHAPTER_PATHNAME_PATTERN,
|
||||
DEFAULT_GET_HTML,
|
||||
DEFAULT_LISTING_URL,
|
||||
MANHWASUSU_HOSTS,
|
||||
MANHWASUSU_ORIGIN,
|
||||
READER_IMG_SELECTORS,
|
||||
chapterUrlToCbz,
|
||||
collectChapterImageUrls,
|
||||
/** @deprecated Use {@link collectChapterImageUrls} */
|
||||
collectManhwaSusuChapterImageUrls: collectChapterImageUrls,
|
||||
downloadChapterImages,
|
||||
/** @deprecated Use {@link downloadChapterImages} */
|
||||
downloadManhwaSusuChapterImages: downloadChapterImages,
|
||||
extensionFromContentType,
|
||||
extensionFromImageUrl,
|
||||
extractChapterLinksFromListingHtml,
|
||||
extractComicTitleFromListingHtml,
|
||||
/** @deprecated Use {@link extractChapterLinksFromListingHtml} */
|
||||
extractManhwaSusuChapters: extractChapterLinksFromListingHtml,
|
||||
fetchChapterLinks,
|
||||
fetchManhwaSusuHtml,
|
||||
/** @deprecated Use {@link fetchChapterLinks} */
|
||||
fetchManhwaSusuChapterLinks: fetchChapterLinks,
|
||||
imageGetConfig,
|
||||
isChapterPathname,
|
||||
isFirstLastNavOnlyLink,
|
||||
/** @deprecated Use {@link isFirstLastNavOnlyLink} */
|
||||
isFirstLastChapterNavLink: isFirstLastNavOnlyLink,
|
||||
isLikelyManhwaSusuSite,
|
||||
/** @deprecated Use {@link isChapterPathname} */
|
||||
isManhwaSusuChapterPath: isChapterPathname,
|
||||
pickReaderImgSrc,
|
||||
resolveCliOutputPath,
|
||||
sanitizeComicDirName,
|
||||
sanitizeFsStem,
|
||||
seriesSlugFromListingUrl,
|
||||
slugFromChapterUrl,
|
||||
stemForManhwaSusuChapterCbz,
|
||||
};
|
||||
@@ -0,0 +1,141 @@
|
||||
const path = require("path");
|
||||
const axios = require("axios");
|
||||
|
||||
const { fetchHtmlWithOptionalFlareSolverr } = require("../flaresolverr.js");
|
||||
const { CDN_IMAGE_HEADERS, MANHWASUSU_ORIGIN } = require("./constants.js");
|
||||
|
||||
/** @type {import("axios").AxiosRequestConfig} */
|
||||
const DEFAULT_GET_HTML = {
|
||||
responseType: "text",
|
||||
timeout: 30_000,
|
||||
headers: {
|
||||
Accept:
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9,id;q=0.8",
|
||||
Referer: MANHWASUSU_ORIGIN,
|
||||
"User-Agent": CDN_IMAGE_HEADERS["User-Agent"],
|
||||
},
|
||||
validateStatus: (s) => s >= 200 && s < 300,
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {string} pageUrl - URL to load.
|
||||
* @returns {Promise<string>} Response body text.
|
||||
*/
|
||||
async function fetchManhwaSusuHtml(pageUrl) {
|
||||
return fetchHtmlWithOptionalFlareSolverr(pageUrl, DEFAULT_GET_HTML);
|
||||
}
|
||||
|
||||
/**
|
||||
* Chooses a usable Lazy-load / CDN attribute from a reader `<img>` (same order as many WP‑style lazy themes).
|
||||
* @param {import("node-html-parser").HTMLElement} img - Image element.
|
||||
* @returns {string|null} First non-placeholder image URL, or null.
|
||||
*/
|
||||
function pickReaderImgSrc(img) {
|
||||
/** @type {(string|null)[]} */
|
||||
const attrs = [
|
||||
img.getAttribute("data-src"),
|
||||
img.getAttribute("data-lazy-src"),
|
||||
img.getAttribute("data-original"),
|
||||
img.getAttribute("data-full-url"),
|
||||
img.getAttribute("src"),
|
||||
];
|
||||
|
||||
for (const raw of attrs) {
|
||||
if (raw && typeof raw === "string") {
|
||||
const t = raw.trim();
|
||||
if (
|
||||
t.length > 0 &&
|
||||
!t.startsWith("data:") &&
|
||||
t !== "/readerarea.svg" &&
|
||||
!t.endsWith("readerarea.svg")
|
||||
) {
|
||||
return t;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} contentType - `Content-Type` header value.
|
||||
* @returns {string} File extension including dot, or empty string.
|
||||
*/
|
||||
function extensionFromContentType(contentType) {
|
||||
if (!contentType || typeof contentType !== "string") {
|
||||
return "";
|
||||
}
|
||||
const ct = contentType.split(";")[0].trim().toLowerCase();
|
||||
/** @type {Record<string, string>} */
|
||||
const map = {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/jpg": ".jpg",
|
||||
"image/pjpeg": ".jpg",
|
||||
"image/png": ".png",
|
||||
"image/webp": ".webp",
|
||||
"image/gif": ".gif",
|
||||
"image/avif": ".avif",
|
||||
"image/svg+xml": ".svg",
|
||||
};
|
||||
return map[ct] || "";
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} imageUrl - Absolute image URL.
|
||||
* @returns {string} Lowercase extension with dot, or empty string.
|
||||
*/
|
||||
function extensionFromImageUrl(imageUrl) {
|
||||
try {
|
||||
const p = new URL(imageUrl).pathname;
|
||||
const ext = path.extname(p).toLowerCase();
|
||||
if (ext.length > 0 && ext.length <= 8 && /^\.[a-z0-9]+$/u.test(ext)) {
|
||||
return ext;
|
||||
}
|
||||
} catch {
|
||||
/* invalid URL */
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} pageUrl - Chapter URL (directory name fragment).
|
||||
* @returns {string} Single filesystem-safe segment.
|
||||
*/
|
||||
function slugFromChapterUrl(pageUrl) {
|
||||
try {
|
||||
const { pathname } = new URL(pageUrl);
|
||||
const parts = pathname.split("/").filter(Boolean);
|
||||
const last = parts[parts.length - 1] || "chapter";
|
||||
const safe = last.replace(/[^a-z0-9._-]+/giu, "_").slice(0, 96);
|
||||
return safe.length > 0 ? safe : "chapter";
|
||||
} catch {
|
||||
return "chapter";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Axios config for CDN panel bytes (browser-like headers from manhwasusu reader).
|
||||
* @param {string} [_referrerChapterUrl] - Kept for call-site compatibility; CDN expects site-origin Referer.
|
||||
* @returns {import("axios").AxiosRequestConfig} GET config for binary image responses.
|
||||
*/
|
||||
function imageGetConfig(_referrerChapterUrl) {
|
||||
return {
|
||||
responseType: "arraybuffer",
|
||||
timeout: 120_000,
|
||||
maxContentLength: 50 * 1024 * 1024,
|
||||
maxBodyLength: 50 * 1024 * 1024,
|
||||
headers: { ...CDN_IMAGE_HEADERS },
|
||||
validateStatus: (s) => s >= 200 && s < 300,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_GET_HTML,
|
||||
extensionFromContentType,
|
||||
extensionFromImageUrl,
|
||||
fetchManhwaSusuHtml,
|
||||
imageGetConfig,
|
||||
pickReaderImgSrc,
|
||||
slugFromChapterUrl,
|
||||
};
|
||||
@@ -0,0 +1,5 @@
|
||||
/**
|
||||
* Re-export for older imports. Prefer `require("./manhwasusu")` or `require("./manhwasusu/index.js")`.
|
||||
* @see {@link ./manhwasusu/index.js}
|
||||
*/
|
||||
module.exports = require("./manhwasusu/index.js");
|
||||
@@ -0,0 +1,473 @@
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const Database = require("better-sqlite3");
|
||||
|
||||
/**
|
||||
* @param {string} dataDir - Absolute data directory (SQLite + CBZ roots live under this).
|
||||
* @returns {import("better-sqlite3").Database} Opened database with schema applied.
|
||||
*/
|
||||
function openDatabase(dataDir) {
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
const dbPath = path.join(dataDir, "app.db");
|
||||
const db = new Database(dbPath);
|
||||
db.pragma("journal_mode = WAL");
|
||||
db.pragma("foreign_keys = ON");
|
||||
migrate(db);
|
||||
return db;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db - Database handle.
|
||||
*/
|
||||
function migrate(db) {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS titles (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source TEXT NOT NULL CHECK (source IN ('mangaread', 'manhwasusu')),
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
slug TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
last_checked_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chapters (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title_id INTEGER NOT NULL REFERENCES titles(id) ON DELETE CASCADE,
|
||||
chapter_key TEXT NOT NULL,
|
||||
chapter_number TEXT,
|
||||
label TEXT NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'downloading', 'downloaded', 'failed')),
|
||||
cbz_path TEXT,
|
||||
downloaded_at TEXT,
|
||||
error TEXT,
|
||||
UNIQUE (title_id, chapter_key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS jobs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
type TEXT NOT NULL CHECK (type IN ('watch_sync', 'redownload', 'download')),
|
||||
title_id INTEGER REFERENCES titles(id) ON DELETE SET NULL,
|
||||
chapter_id INTEGER REFERENCES chapters(id) ON DELETE SET NULL,
|
||||
status TEXT NOT NULL DEFAULT 'queued'
|
||||
CHECK (status IN ('queued', 'running', 'done', 'failed')),
|
||||
message TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
finished_at TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_chapters_title_id ON chapters(title_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_chapters_status ON chapters(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status);
|
||||
`);
|
||||
|
||||
// Drop active jobs left behind after a title was removed (ON DELETE SET NULL).
|
||||
db.prepare(
|
||||
`
|
||||
DELETE FROM jobs
|
||||
WHERE status IN ('queued', 'running')
|
||||
AND (title_id IS NULL OR title_id NOT IN (SELECT id FROM titles))
|
||||
`,
|
||||
).run();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @returns {object[]}
|
||||
*/
|
||||
function listTitlesWithCounts(db) {
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
t.*,
|
||||
COUNT(c.id) AS total,
|
||||
SUM(CASE WHEN c.status = 'downloaded' THEN 1 ELSE 0 END) AS downloaded,
|
||||
SUM(CASE WHEN c.status = 'pending' THEN 1 ELSE 0 END) AS pending,
|
||||
SUM(CASE WHEN c.status = 'downloading' THEN 1 ELSE 0 END) AS downloading,
|
||||
SUM(CASE WHEN c.status = 'failed' THEN 1 ELSE 0 END) AS failed
|
||||
FROM titles t
|
||||
LEFT JOIN chapters c ON c.title_id = t.id
|
||||
GROUP BY t.id
|
||||
ORDER BY t.id DESC
|
||||
`,
|
||||
)
|
||||
.all()
|
||||
.map(normalizeTitleRow);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {object} row
|
||||
* @returns {object}
|
||||
*/
|
||||
function normalizeTitleRow(row) {
|
||||
return {
|
||||
...row,
|
||||
enabled: Boolean(row.enabled),
|
||||
total: Number(row.total) || 0,
|
||||
downloaded: Number(row.downloaded) || 0,
|
||||
pending: Number(row.pending) || 0,
|
||||
downloading: Number(row.downloading) || 0,
|
||||
failed: Number(row.failed) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {number} id
|
||||
* @returns {object|undefined}
|
||||
*/
|
||||
function getTitle(db, id) {
|
||||
const row = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
t.*,
|
||||
COUNT(c.id) AS total,
|
||||
SUM(CASE WHEN c.status = 'downloaded' THEN 1 ELSE 0 END) AS downloaded,
|
||||
SUM(CASE WHEN c.status = 'pending' THEN 1 ELSE 0 END) AS pending,
|
||||
SUM(CASE WHEN c.status = 'downloading' THEN 1 ELSE 0 END) AS downloading,
|
||||
SUM(CASE WHEN c.status = 'failed' THEN 1 ELSE 0 END) AS failed
|
||||
FROM titles t
|
||||
LEFT JOIN chapters c ON c.title_id = t.id
|
||||
WHERE t.id = ?
|
||||
GROUP BY t.id
|
||||
`,
|
||||
)
|
||||
.get(id);
|
||||
return row ? normalizeTitleRow(row) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {number} titleId
|
||||
* @returns {object[]}
|
||||
*/
|
||||
function listChapters(db, titleId) {
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT * FROM chapters
|
||||
WHERE title_id = ?
|
||||
ORDER BY
|
||||
CASE WHEN chapter_number GLOB '[0-9]*' THEN CAST(chapter_number AS REAL) ELSE 1e18 END,
|
||||
id
|
||||
`,
|
||||
)
|
||||
.all(titleId);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {{ source: string, url: string, slug: string, title: string }} data
|
||||
* @returns {object}
|
||||
*/
|
||||
function insertTitle(db, data) {
|
||||
const info = db
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO titles (source, url, slug, title, enabled)
|
||||
VALUES (@source, @url, @slug, @title, 1)
|
||||
`,
|
||||
)
|
||||
.run(data);
|
||||
return getTitle(db, Number(info.lastInsertRowid));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {number} id
|
||||
* @param {{ enabled?: boolean, title?: string }} patch
|
||||
* @returns {object|undefined}
|
||||
*/
|
||||
function updateTitle(db, id, patch) {
|
||||
const current = db.prepare("SELECT * FROM titles WHERE id = ?").get(id);
|
||||
if (!current) {
|
||||
return undefined;
|
||||
}
|
||||
const enabled =
|
||||
patch.enabled === undefined ? current.enabled : patch.enabled ? 1 : 0;
|
||||
const title = patch.title === undefined ? current.title : patch.title;
|
||||
db.prepare("UPDATE titles SET enabled = ?, title = ? WHERE id = ?").run(
|
||||
enabled,
|
||||
title,
|
||||
id,
|
||||
);
|
||||
return getTitle(db, id);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {number} id
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function deleteTitle(db, id) {
|
||||
const info = db.prepare("DELETE FROM titles WHERE id = ?").run(id);
|
||||
return info.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upsert a chapter row; does not overwrite downloading/downloaded unless forcePending.
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {object} chapter
|
||||
* @returns {{ id: number, inserted: boolean }}
|
||||
*/
|
||||
function upsertChapter(db, chapter) {
|
||||
const existing = db
|
||||
.prepare(
|
||||
"SELECT id, status, cbz_path FROM chapters WHERE title_id = ? AND chapter_key = ?",
|
||||
)
|
||||
.get(chapter.title_id, chapter.chapter_key);
|
||||
|
||||
if (existing) {
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE chapters
|
||||
SET label = @label, url = @url, chapter_number = @chapter_number,
|
||||
cbz_path = COALESCE(@cbz_path, cbz_path)
|
||||
WHERE id = @id
|
||||
`,
|
||||
).run({
|
||||
id: existing.id,
|
||||
label: chapter.label,
|
||||
url: chapter.url,
|
||||
chapter_number: chapter.chapter_number,
|
||||
cbz_path: chapter.cbz_path ?? null,
|
||||
});
|
||||
return { id: existing.id, inserted: false };
|
||||
}
|
||||
|
||||
const info = db
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO chapters (
|
||||
title_id, chapter_key, chapter_number, label, url, status, cbz_path
|
||||
) VALUES (
|
||||
@title_id, @chapter_key, @chapter_number, @label, @url, @status, @cbz_path
|
||||
)
|
||||
`,
|
||||
)
|
||||
.run(chapter);
|
||||
return { id: Number(info.lastInsertRowid), inserted: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {number} id
|
||||
* @returns {object|undefined}
|
||||
*/
|
||||
function getChapter(db, id) {
|
||||
return db.prepare("SELECT * FROM chapters WHERE id = ?").get(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {number} id
|
||||
* @param {object} fields
|
||||
*/
|
||||
function updateChapter(db, id, fields) {
|
||||
const allowed = [
|
||||
"status",
|
||||
"cbz_path",
|
||||
"downloaded_at",
|
||||
"error",
|
||||
"label",
|
||||
"url",
|
||||
];
|
||||
const sets = [];
|
||||
const values = {};
|
||||
for (const key of allowed) {
|
||||
if (Object.prototype.hasOwnProperty.call(fields, key)) {
|
||||
sets.push(`${key} = @${key}`);
|
||||
values[key] = fields[key];
|
||||
}
|
||||
}
|
||||
if (sets.length === 0) {
|
||||
return;
|
||||
}
|
||||
values.id = id;
|
||||
db.prepare(`UPDATE chapters SET ${sets.join(", ")} WHERE id = @id`).run(values);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {{ type: string, title_id?: number|null, chapter_id?: number|null, message?: string|null }} data
|
||||
* @returns {number} Job id
|
||||
*/
|
||||
function createJob(db, data) {
|
||||
const info = db
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO jobs (type, title_id, chapter_id, status, message)
|
||||
VALUES (@type, @title_id, @chapter_id, 'queued', @message)
|
||||
`,
|
||||
)
|
||||
.run({
|
||||
type: data.type,
|
||||
title_id: data.title_id ?? null,
|
||||
chapter_id: data.chapter_id ?? null,
|
||||
message: data.message ?? null,
|
||||
});
|
||||
return Number(info.lastInsertRowid);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {number} id
|
||||
* @param {object} fields
|
||||
*/
|
||||
function updateJob(db, id, fields) {
|
||||
const allowed = ["status", "message", "finished_at"];
|
||||
const sets = [];
|
||||
const values = {};
|
||||
for (const key of allowed) {
|
||||
if (Object.prototype.hasOwnProperty.call(fields, key)) {
|
||||
sets.push(`${key} = @${key}`);
|
||||
values[key] = fields[key];
|
||||
}
|
||||
}
|
||||
if (sets.length === 0) {
|
||||
return;
|
||||
}
|
||||
values.id = id;
|
||||
db.prepare(`UPDATE jobs SET ${sets.join(", ")} WHERE id = @id`).run(values);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {{ active?: boolean, limit?: number }} opts
|
||||
* @returns {object[]}
|
||||
*/
|
||||
function listJobs(db, opts = {}) {
|
||||
const limit = opts.limit ?? 50;
|
||||
if (opts.active) {
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT j.*, t.title AS title_name, c.label AS chapter_label
|
||||
FROM jobs j
|
||||
INNER JOIN titles t ON t.id = j.title_id
|
||||
LEFT JOIN chapters c ON c.id = j.chapter_id
|
||||
WHERE j.status IN ('queued', 'running')
|
||||
ORDER BY
|
||||
CASE j.status WHEN 'running' THEN 0 ELSE 1 END,
|
||||
j.id ASC
|
||||
LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(limit);
|
||||
}
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT j.*, t.title AS title_name, c.label AS chapter_label
|
||||
FROM jobs j
|
||||
LEFT JOIN titles t ON t.id = j.title_id
|
||||
LEFT JOIN chapters c ON c.id = j.chapter_id
|
||||
ORDER BY j.id DESC
|
||||
LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @returns {object[]}
|
||||
*/
|
||||
function listEnabledTitles(db) {
|
||||
return db
|
||||
.prepare("SELECT * FROM titles WHERE enabled = 1 ORDER BY id ASC")
|
||||
.all();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {number} titleId
|
||||
*/
|
||||
function touchTitleChecked(db, titleId) {
|
||||
db.prepare(
|
||||
"UPDATE titles SET last_checked_at = datetime('now') WHERE id = ?",
|
||||
).run(titleId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Chapters that need a download (pending/failed, or downloaded but file missing).
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {number} titleId
|
||||
* @returns {object[]}
|
||||
*/
|
||||
function listChaptersNeedingDownload(db, titleId) {
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT * FROM chapters
|
||||
WHERE title_id = ?
|
||||
AND status IN ('pending', 'failed')
|
||||
ORDER BY id ASC
|
||||
`,
|
||||
)
|
||||
.all(titleId);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db - Database handle.
|
||||
* @param {string} url - Exact listing URL.
|
||||
* @returns {object|undefined} Title row when found.
|
||||
*/
|
||||
function findTitleByUrl(db, url) {
|
||||
return db.prepare("SELECT * FROM titles WHERE url = ?").get(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the next enabled title due for a watch sync (never-checked first, then oldest).
|
||||
* @param {import("better-sqlite3").Database} db - Database handle.
|
||||
* @param {number} minAgeHours - Hours since last check before a title is due again.
|
||||
* @returns {object|undefined} Title row, or undefined when nothing is due.
|
||||
*/
|
||||
function pickNextTitleDue(db, minAgeHours) {
|
||||
const hours = Math.max(1, Math.floor(Number(minAgeHours) || 24));
|
||||
const modifier = `-${hours} hours`;
|
||||
return db
|
||||
.prepare(
|
||||
`
|
||||
SELECT *
|
||||
FROM titles
|
||||
WHERE enabled = 1
|
||||
AND (
|
||||
last_checked_at IS NULL
|
||||
OR last_checked_at <= datetime('now', '${modifier}')
|
||||
)
|
||||
ORDER BY
|
||||
CASE WHEN last_checked_at IS NULL THEN 0 ELSE 1 END,
|
||||
last_checked_at ASC,
|
||||
id ASC
|
||||
LIMIT 1
|
||||
`,
|
||||
)
|
||||
.get();
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
openDatabase,
|
||||
listTitlesWithCounts,
|
||||
getTitle,
|
||||
listChapters,
|
||||
insertTitle,
|
||||
updateTitle,
|
||||
deleteTitle,
|
||||
upsertChapter,
|
||||
getChapter,
|
||||
updateChapter,
|
||||
createJob,
|
||||
updateJob,
|
||||
listJobs,
|
||||
listEnabledTitles,
|
||||
touchTitleChecked,
|
||||
listChaptersNeedingDownload,
|
||||
findTitleByUrl,
|
||||
pickNextTitleDue,
|
||||
};
|
||||
@@ -0,0 +1,92 @@
|
||||
const path = require("path");
|
||||
const fs = require("fs");
|
||||
const Fastify = require("fastify");
|
||||
const fastifyStatic = require("@fastify/static");
|
||||
|
||||
const { openDatabase } = require("./db.js");
|
||||
const { DownloadQueue } = require("./queue.js");
|
||||
const { startWatcher } = require("./watcher.js");
|
||||
const routesPlugin = require("./routes.js");
|
||||
|
||||
const PORT = Number(process.env.PORT) || 3000;
|
||||
const DATA_DIR = path.resolve(process.env.DATA_DIR || path.join(process.cwd(), "data"));
|
||||
/** How often the watcher wakes (default: every hour). */
|
||||
const WATCH_CRON = process.env.WATCH_CRON || "0 * * * *";
|
||||
/** Only sync a title if it has not been checked for this many hours (default: daily). */
|
||||
const WATCH_TITLE_INTERVAL_HOURS = Number(process.env.WATCH_TITLE_INTERVAL_HOURS) || 24;
|
||||
const PUBLIC_DIR = path.join(process.cwd(), "public");
|
||||
|
||||
/**
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function main() {
|
||||
fs.mkdirSync(DATA_DIR, { recursive: true });
|
||||
fs.mkdirSync(PUBLIC_DIR, { recursive: true });
|
||||
|
||||
const db = openDatabase(DATA_DIR);
|
||||
const queue = new DownloadQueue(db);
|
||||
const app = Fastify({ logger: true });
|
||||
|
||||
// Allow POST/PATCH with Content-Type: application/json and an empty body
|
||||
app.addContentTypeParser(
|
||||
"application/json",
|
||||
{ parseAs: "string" },
|
||||
(request, body, done) => {
|
||||
if (body === "" || body == null) {
|
||||
done(null, {});
|
||||
return;
|
||||
}
|
||||
try {
|
||||
done(null, JSON.parse(body));
|
||||
} catch (err) {
|
||||
done(err instanceof Error ? err : new Error(String(err)), undefined);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
await app.register(routesPlugin, { db, queue, dataDir: DATA_DIR });
|
||||
|
||||
await app.register(fastifyStatic, {
|
||||
root: PUBLIC_DIR,
|
||||
prefix: "/",
|
||||
wildcard: false,
|
||||
});
|
||||
|
||||
// SPA fallback for non-API routes
|
||||
app.setNotFoundHandler((request, reply) => {
|
||||
if (request.url.startsWith("/api/")) {
|
||||
return reply.code(404).send({ error: "Not found" });
|
||||
}
|
||||
const indexPath = path.join(PUBLIC_DIR, "index.html");
|
||||
if (fs.existsSync(indexPath)) {
|
||||
return reply.type("text/html").send(fs.readFileSync(indexPath));
|
||||
}
|
||||
return reply
|
||||
.code(503)
|
||||
.type("text/plain")
|
||||
.send("Frontend not built. Run: npm run build:web");
|
||||
});
|
||||
|
||||
const watcher = startWatcher(db, queue, DATA_DIR, WATCH_CRON, {
|
||||
titleIntervalHours: WATCH_TITLE_INTERVAL_HOURS,
|
||||
});
|
||||
|
||||
const shutdown = async () => {
|
||||
watcher.stop();
|
||||
await app.close();
|
||||
db.close();
|
||||
process.exit(0);
|
||||
};
|
||||
process.on("SIGINT", () => void shutdown());
|
||||
process.on("SIGTERM", () => void shutdown());
|
||||
|
||||
await app.listen({ port: PORT, host: "0.0.0.0" });
|
||||
app.log.info(`Data dir: ${DATA_DIR}`);
|
||||
app.log.info(`Watch cron: ${WATCH_CRON}`);
|
||||
app.log.info(`Watch title interval: ${String(WATCH_TITLE_INTERVAL_HOURS)}h (one title per tick)`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,359 @@
|
||||
const dbApi = require("./db.js");
|
||||
const {
|
||||
downloadChapter,
|
||||
cbzExists,
|
||||
removeCbzIfExists,
|
||||
} = require("./sources.js");
|
||||
|
||||
/**
|
||||
* Serial in-process download queue.
|
||||
*/
|
||||
class DownloadQueue {
|
||||
/**
|
||||
* @param {import("better-sqlite3").Database} db
|
||||
* @param {{ onChange?: () => void }} [opts]
|
||||
*/
|
||||
constructor(db, opts = {}) {
|
||||
this.db = db;
|
||||
this.onChange = opts.onChange;
|
||||
/** @type {number[]} */
|
||||
this.pendingJobIds = [];
|
||||
this.running = false;
|
||||
/** @type {Set<number>} Chapters cancelled while a download may still be in flight. */
|
||||
this.cancelledChapterIds = new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue a chapter download job.
|
||||
* @param {number} chapterId
|
||||
* @param {{ type?: 'download'|'redownload', force?: boolean }} [opts]
|
||||
* @returns {number|null} Job id, or null if skipped (already queued/running for same chapter)
|
||||
*/
|
||||
enqueueChapter(chapterId, opts = {}) {
|
||||
const type = opts.type || (opts.force ? "redownload" : "download");
|
||||
const chapter = dbApi.getChapter(this.db, chapterId);
|
||||
if (!chapter) {
|
||||
throw new Error(`Chapter ${String(chapterId)} not found.`);
|
||||
}
|
||||
|
||||
const active = dbApi
|
||||
.listJobs(this.db, { active: true, limit: 500 })
|
||||
.find((j) => j.chapter_id === chapterId);
|
||||
if (active) {
|
||||
return active.id;
|
||||
}
|
||||
|
||||
if (opts.force) {
|
||||
removeCbzIfExists(chapter.cbz_path);
|
||||
dbApi.updateChapter(this.db, chapterId, {
|
||||
status: "pending",
|
||||
error: null,
|
||||
downloaded_at: null,
|
||||
});
|
||||
}
|
||||
|
||||
this.cancelledChapterIds.delete(chapterId);
|
||||
|
||||
const jobId = dbApi.createJob(this.db, {
|
||||
type,
|
||||
title_id: chapter.title_id,
|
||||
chapter_id: chapterId,
|
||||
message: `Queued ${chapter.label}`,
|
||||
});
|
||||
this.pendingJobIds.push(jobId);
|
||||
this.#notify();
|
||||
void this.#pump();
|
||||
return jobId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue all chapters that need downloading for a title.
|
||||
* @param {number} titleId
|
||||
* @param {{ alsoMissingFiles?: boolean }} [opts]
|
||||
* @returns {number} Number of jobs enqueued
|
||||
*/
|
||||
enqueueMissingForTitle(titleId, opts = {}) {
|
||||
const chapters = dbApi.listChapters(this.db, titleId);
|
||||
let count = 0;
|
||||
for (const ch of chapters) {
|
||||
const missingFile =
|
||||
Boolean(opts.alsoMissingFiles) &&
|
||||
ch.status === "downloaded" &&
|
||||
!cbzExists(ch.cbz_path);
|
||||
const needs =
|
||||
ch.status === "pending" ||
|
||||
ch.status === "failed" ||
|
||||
missingFile;
|
||||
if (!needs) {
|
||||
continue;
|
||||
}
|
||||
if (missingFile) {
|
||||
dbApi.updateChapter(this.db, ch.id, { status: "pending", error: "CBZ missing on disk" });
|
||||
}
|
||||
this.enqueueChapter(ch.id, { type: "download" });
|
||||
count += 1;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop all waiting download jobs. The in-flight job (if any) is left to finish.
|
||||
* @returns {{ cleared: number }}
|
||||
*/
|
||||
clear() {
|
||||
this.pendingJobIds = [];
|
||||
const info = this.db
|
||||
.prepare(
|
||||
`
|
||||
UPDATE jobs
|
||||
SET status = 'failed',
|
||||
message = 'Cleared from queue',
|
||||
finished_at = datetime('now')
|
||||
WHERE status = 'queued'
|
||||
`,
|
||||
)
|
||||
.run();
|
||||
this.#notify();
|
||||
return { cleared: Number(info.changes) || 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel a stuck/queued/downloading chapter: drop its jobs and reset status to pending.
|
||||
* If a download is mid-flight, ignore its success when it finishes.
|
||||
* @param {number} chapterId - Chapter row id.
|
||||
* @returns {{ ok: boolean, chapter: object|undefined }}
|
||||
*/
|
||||
cancelChapter(chapterId) {
|
||||
const chapter = dbApi.getChapter(this.db, chapterId);
|
||||
if (!chapter) {
|
||||
return { ok: false, chapter: undefined };
|
||||
}
|
||||
|
||||
this.cancelledChapterIds.add(chapterId);
|
||||
|
||||
this.pendingJobIds = this.pendingJobIds.filter((jobId) => {
|
||||
const job = this.db.prepare("SELECT chapter_id FROM jobs WHERE id = ?").get(jobId);
|
||||
return !job || job.chapter_id !== chapterId;
|
||||
});
|
||||
|
||||
this.db
|
||||
.prepare(
|
||||
`
|
||||
UPDATE jobs
|
||||
SET status = 'failed',
|
||||
message = 'Cancelled by user',
|
||||
finished_at = datetime('now')
|
||||
WHERE chapter_id = ?
|
||||
AND status IN ('queued', 'running')
|
||||
`,
|
||||
)
|
||||
.run(chapterId);
|
||||
|
||||
dbApi.updateChapter(this.db, chapterId, {
|
||||
status: "pending",
|
||||
error: "Cancelled by user",
|
||||
downloaded_at: null,
|
||||
});
|
||||
|
||||
this.#notify();
|
||||
return { ok: true, chapter: dbApi.getChapter(this.db, chapterId) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a watched title: cancel its jobs/queue entries, then delete the row.
|
||||
* @param {number} titleId - Title id to remove.
|
||||
* @returns {boolean} True when the title existed and was deleted.
|
||||
*/
|
||||
removeTitle(titleId) {
|
||||
const title = this.db.prepare("SELECT id FROM titles WHERE id = ?").get(titleId);
|
||||
if (!title) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const chapters = dbApi.listChapters(this.db, titleId);
|
||||
for (const ch of chapters) {
|
||||
this.cancelledChapterIds.add(ch.id);
|
||||
}
|
||||
|
||||
this.pendingJobIds = this.pendingJobIds.filter((jobId) => {
|
||||
const job = this.db.prepare("SELECT title_id FROM jobs WHERE id = ?").get(jobId);
|
||||
return !job || job.title_id !== titleId;
|
||||
});
|
||||
|
||||
this.db
|
||||
.prepare(
|
||||
`
|
||||
UPDATE jobs
|
||||
SET status = 'failed',
|
||||
message = 'Title removed',
|
||||
finished_at = datetime('now')
|
||||
WHERE title_id = ?
|
||||
AND status IN ('queued', 'running')
|
||||
`,
|
||||
)
|
||||
.run(titleId);
|
||||
|
||||
this.db.prepare("DELETE FROM jobs WHERE title_id = ?").run(titleId);
|
||||
dbApi.deleteTitle(this.db, titleId);
|
||||
this.#notify();
|
||||
return true;
|
||||
}
|
||||
|
||||
async #pump() {
|
||||
if (this.running) {
|
||||
return;
|
||||
}
|
||||
this.running = true;
|
||||
try {
|
||||
while (this.pendingJobIds.length > 0) {
|
||||
const jobId = this.pendingJobIds.shift();
|
||||
if (jobId === undefined) {
|
||||
break;
|
||||
}
|
||||
await this.#runJob(jobId);
|
||||
}
|
||||
} finally {
|
||||
this.running = false;
|
||||
this.#notify();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {number} jobId
|
||||
*/
|
||||
async #runJob(jobId) {
|
||||
const job = this.db.prepare("SELECT * FROM jobs WHERE id = ?").get(jobId);
|
||||
if (!job || job.status !== "queued") {
|
||||
return;
|
||||
}
|
||||
|
||||
const chapter = job.chapter_id
|
||||
? dbApi.getChapter(this.db, job.chapter_id)
|
||||
: null;
|
||||
if (!chapter) {
|
||||
dbApi.updateJob(this.db, jobId, {
|
||||
status: "failed",
|
||||
message: "Chapter missing",
|
||||
finished_at: new Date().toISOString(),
|
||||
});
|
||||
this.#notify();
|
||||
return;
|
||||
}
|
||||
|
||||
const title = dbApi.getTitle(this.db, chapter.title_id);
|
||||
if (!title) {
|
||||
dbApi.updateJob(this.db, jobId, {
|
||||
status: "failed",
|
||||
message: "Title missing",
|
||||
finished_at: new Date().toISOString(),
|
||||
});
|
||||
this.#notify();
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip if already present (unless redownload)
|
||||
if (
|
||||
job.type !== "redownload" &&
|
||||
chapter.status === "downloaded" &&
|
||||
cbzExists(chapter.cbz_path)
|
||||
) {
|
||||
dbApi.updateJob(this.db, jobId, {
|
||||
status: "done",
|
||||
message: "Already downloaded",
|
||||
finished_at: new Date().toISOString(),
|
||||
});
|
||||
this.#notify();
|
||||
return;
|
||||
}
|
||||
|
||||
dbApi.updateJob(this.db, jobId, {
|
||||
status: "running",
|
||||
message: `Downloading ${chapter.label}`,
|
||||
});
|
||||
dbApi.updateChapter(this.db, chapter.id, {
|
||||
status: "downloading",
|
||||
error: null,
|
||||
});
|
||||
this.#notify();
|
||||
|
||||
const cbzPath = chapter.cbz_path;
|
||||
if (!cbzPath) {
|
||||
dbApi.updateChapter(this.db, chapter.id, {
|
||||
status: "failed",
|
||||
error: "No cbz_path set",
|
||||
});
|
||||
dbApi.updateJob(this.db, jobId, {
|
||||
status: "failed",
|
||||
message: "No cbz_path set",
|
||||
finished_at: new Date().toISOString(),
|
||||
});
|
||||
this.#notify();
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const { imageCount, outputPath } = await downloadChapter(
|
||||
title.source,
|
||||
chapter.url,
|
||||
cbzPath,
|
||||
);
|
||||
|
||||
if (this.cancelledChapterIds.has(chapter.id)) {
|
||||
this.cancelledChapterIds.delete(chapter.id);
|
||||
dbApi.updateJob(this.db, jobId, {
|
||||
status: "failed",
|
||||
message: "Cancelled by user",
|
||||
finished_at: new Date().toISOString(),
|
||||
});
|
||||
// Keep chapter as pending if cancel already reset it; otherwise force pending.
|
||||
const latest = dbApi.getChapter(this.db, chapter.id);
|
||||
if (latest && latest.status === "downloading") {
|
||||
dbApi.updateChapter(this.db, chapter.id, {
|
||||
status: "pending",
|
||||
error: "Cancelled by user",
|
||||
});
|
||||
}
|
||||
this.#notify();
|
||||
return;
|
||||
}
|
||||
|
||||
dbApi.updateChapter(this.db, chapter.id, {
|
||||
status: "downloaded",
|
||||
cbz_path: outputPath,
|
||||
downloaded_at: new Date().toISOString(),
|
||||
error: null,
|
||||
});
|
||||
dbApi.updateJob(this.db, jobId, {
|
||||
status: "done",
|
||||
message: `${chapter.label}: ${String(imageCount)} page(s)`,
|
||||
finished_at: new Date().toISOString(),
|
||||
});
|
||||
} catch (err) {
|
||||
if (this.cancelledChapterIds.has(chapter.id)) {
|
||||
this.cancelledChapterIds.delete(chapter.id);
|
||||
}
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
const latest = dbApi.getChapter(this.db, chapter.id);
|
||||
if (!latest || latest.status === "downloading") {
|
||||
dbApi.updateChapter(this.db, chapter.id, {
|
||||
status: "failed",
|
||||
error: msg,
|
||||
});
|
||||
}
|
||||
dbApi.updateJob(this.db, jobId, {
|
||||
status: "failed",
|
||||
message: msg,
|
||||
finished_at: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
this.#notify();
|
||||
}
|
||||
|
||||
#notify() {
|
||||
if (typeof this.onChange === "function") {
|
||||
this.onChange();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { DownloadQueue };
|
||||
@@ -0,0 +1,226 @@
|
||||
const dbApi = require("./db.js");
|
||||
const {
|
||||
detectSource,
|
||||
normalizeListingUrl,
|
||||
fetchSeriesChapters,
|
||||
resolveCbzPath,
|
||||
cbzExists,
|
||||
} = require("./sources.js");
|
||||
const { syncTitle } = require("./watcher.js");
|
||||
const { formatUpstreamError } = require("../flaresolverr.js");
|
||||
|
||||
/**
|
||||
* @param {import("fastify").FastifyInstance} fastify - Fastify app.
|
||||
* @param {{ db: import("better-sqlite3").Database, queue: import("./queue.js").DownloadQueue, dataDir: string }} opts - Shared deps.
|
||||
*/
|
||||
async function routesPlugin(fastify, opts) {
|
||||
const { db, queue, dataDir } = opts;
|
||||
|
||||
fastify.get("/api/titles", async () => {
|
||||
return { titles: dbApi.listTitlesWithCounts(db) };
|
||||
});
|
||||
|
||||
fastify.get("/api/titles/:id", async (request, reply) => {
|
||||
const id = Number(request.params.id);
|
||||
const title = dbApi.getTitle(db, id);
|
||||
if (!title) {
|
||||
return reply.code(404).send({ error: "Title not found" });
|
||||
}
|
||||
return { title };
|
||||
});
|
||||
|
||||
fastify.post("/api/titles", async (request, reply) => {
|
||||
const body = request.body || {};
|
||||
const url = typeof body.url === "string" ? body.url.trim() : "";
|
||||
if (!url) {
|
||||
return reply.code(400).send({ error: "url is required" });
|
||||
}
|
||||
|
||||
let source;
|
||||
let listingUrl;
|
||||
try {
|
||||
source = detectSource(url);
|
||||
listingUrl = normalizeListingUrl(url, source);
|
||||
} catch (err) {
|
||||
return reply
|
||||
.code(400)
|
||||
.send({ error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
|
||||
const siteLabel = source === "manhwasusu" ? "ManhwaSusu" : "Mangaread";
|
||||
|
||||
const existing = dbApi.findTitleByUrl(db, listingUrl);
|
||||
if (existing) {
|
||||
try {
|
||||
const result = await syncTitle(db, queue, existing.id, dataDir, {
|
||||
enqueue: true,
|
||||
});
|
||||
return {
|
||||
title: dbApi.getTitle(db, existing.id),
|
||||
created: false,
|
||||
...result,
|
||||
};
|
||||
} catch (err) {
|
||||
return reply.code(502).send({
|
||||
error: formatUpstreamError(err, siteLabel),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let series;
|
||||
try {
|
||||
series = await fetchSeriesChapters(listingUrl, source);
|
||||
} catch (err) {
|
||||
return reply.code(502).send({
|
||||
error: formatUpstreamError(err, siteLabel),
|
||||
});
|
||||
}
|
||||
|
||||
const title = dbApi.insertTitle(db, {
|
||||
source: series.source,
|
||||
url: series.listingUrl,
|
||||
slug: series.slug,
|
||||
title: series.comicTitle,
|
||||
});
|
||||
|
||||
for (const ch of series.chapters) {
|
||||
const cbzPath = resolveCbzPath(
|
||||
series.source,
|
||||
series.comicTitle,
|
||||
ch.url,
|
||||
dataDir,
|
||||
);
|
||||
const exists = cbzExists(cbzPath);
|
||||
dbApi.upsertChapter(db, {
|
||||
title_id: title.id,
|
||||
chapter_key: ch.chapter_key,
|
||||
chapter_number: ch.chapter_number,
|
||||
label: ch.title,
|
||||
url: ch.url,
|
||||
status: exists ? "downloaded" : "pending",
|
||||
cbz_path: cbzPath,
|
||||
});
|
||||
if (exists) {
|
||||
const row = db
|
||||
.prepare(
|
||||
"SELECT id FROM chapters WHERE title_id = ? AND chapter_key = ?",
|
||||
)
|
||||
.get(title.id, ch.chapter_key);
|
||||
if (row) {
|
||||
dbApi.updateChapter(db, row.id, {
|
||||
status: "downloaded",
|
||||
downloaded_at: new Date().toISOString(),
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dbApi.touchTitleChecked(db, title.id);
|
||||
const enqueued = queue.enqueueMissingForTitle(title.id, {
|
||||
alsoMissingFiles: true,
|
||||
});
|
||||
|
||||
return {
|
||||
title: dbApi.getTitle(db, title.id),
|
||||
created: true,
|
||||
added: series.chapters.length,
|
||||
enqueued,
|
||||
total: series.chapters.length,
|
||||
};
|
||||
});
|
||||
|
||||
fastify.patch("/api/titles/:id", async (request, reply) => {
|
||||
const id = Number(request.params.id);
|
||||
const body = request.body || {};
|
||||
const patch = {};
|
||||
if (typeof body.enabled === "boolean") {
|
||||
patch.enabled = body.enabled;
|
||||
}
|
||||
if (typeof body.title === "string" && body.title.trim()) {
|
||||
patch.title = body.title.trim();
|
||||
}
|
||||
const title = dbApi.updateTitle(db, id, patch);
|
||||
if (!title) {
|
||||
return reply.code(404).send({ error: "Title not found" });
|
||||
}
|
||||
return { title };
|
||||
});
|
||||
|
||||
fastify.delete("/api/titles/:id", async (request, reply) => {
|
||||
const id = Number(request.params.id);
|
||||
const ok = queue.removeTitle(id);
|
||||
if (!ok) {
|
||||
return reply.code(404).send({ error: "Title not found" });
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
jobs: dbApi.listJobs(db, { active: true, limit: 20 }),
|
||||
};
|
||||
});
|
||||
|
||||
fastify.get("/api/titles/:id/chapters", async (request, reply) => {
|
||||
const id = Number(request.params.id);
|
||||
const title = dbApi.getTitle(db, id);
|
||||
if (!title) {
|
||||
return reply.code(404).send({ error: "Title not found" });
|
||||
}
|
||||
return { chapters: dbApi.listChapters(db, id) };
|
||||
});
|
||||
|
||||
fastify.post("/api/titles/:id/sync", async (request, reply) => {
|
||||
const id = Number(request.params.id);
|
||||
const title = dbApi.getTitle(db, id);
|
||||
if (!title) {
|
||||
return reply.code(404).send({ error: "Title not found" });
|
||||
}
|
||||
const siteLabel = title.source === "manhwasusu" ? "ManhwaSusu" : "Mangaread";
|
||||
try {
|
||||
const result = await syncTitle(db, queue, id, dataDir, { enqueue: true });
|
||||
return { title: dbApi.getTitle(db, id), ...result };
|
||||
} catch (err) {
|
||||
return reply.code(502).send({
|
||||
error: formatUpstreamError(err, siteLabel),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
fastify.post("/api/chapters/:id/redownload", async (request, reply) => {
|
||||
const id = Number(request.params.id);
|
||||
const chapter = dbApi.getChapter(db, id);
|
||||
if (!chapter) {
|
||||
return reply.code(404).send({ error: "Chapter not found" });
|
||||
}
|
||||
const jobId = queue.enqueueChapter(id, { force: true, type: "redownload" });
|
||||
return { ok: true, jobId, chapter: dbApi.getChapter(db, id) };
|
||||
});
|
||||
|
||||
fastify.post("/api/chapters/:id/cancel", async (request, reply) => {
|
||||
const id = Number(request.params.id);
|
||||
const result = queue.cancelChapter(id);
|
||||
if (!result.ok) {
|
||||
return reply.code(404).send({ error: "Chapter not found" });
|
||||
}
|
||||
return { ok: true, chapter: result.chapter };
|
||||
});
|
||||
|
||||
fastify.get("/api/jobs", async (request) => {
|
||||
const active =
|
||||
request.query.active === "1" ||
|
||||
request.query.active === "true" ||
|
||||
request.query.active === true;
|
||||
const limit = request.query.limit ? Number(request.query.limit) : 50;
|
||||
return { jobs: dbApi.listJobs(db, { active, limit }) };
|
||||
});
|
||||
|
||||
fastify.post("/api/jobs/clear", async () => {
|
||||
const result = queue.clear();
|
||||
return {
|
||||
ok: true,
|
||||
cleared: result.cleared,
|
||||
jobs: dbApi.listJobs(db, { active: true, limit: 20 }),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = routesPlugin;
|
||||
@@ -0,0 +1,258 @@
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const {
|
||||
fetchWpMangaChapterLinks,
|
||||
mangaSlugFromUrl,
|
||||
} = require("../mangareadFetchChapters.js");
|
||||
const {
|
||||
chapterUrlToCbz: mangareadChapterUrlToCbz,
|
||||
sanitizeComicDirName: mangareadSanitizeDir,
|
||||
sanitizeFsStem: mangareadSanitizeStem,
|
||||
stemForCbzFilename: mangareadStem,
|
||||
} = require("../downloadChapterCbz.js");
|
||||
const { chaptersInReadingOrder: mangareadReadingOrder } = require("../downloadChaptersQueue.js");
|
||||
const { chapterNumberFromWpTitle } = require("../downloadChaptersQueue.js");
|
||||
|
||||
const manhwa = require("../manhwasusu/index.js");
|
||||
const {
|
||||
chaptersInReadingOrder: manhwaReadingOrder,
|
||||
} = require("../manhwasusu/downloadChaptersQueue.js");
|
||||
|
||||
/**
|
||||
* @typedef {'mangaread'|'manhwasusu'} SourceId
|
||||
*/
|
||||
|
||||
/**
|
||||
* Detect source from a series listing URL.
|
||||
* @param {string} rawUrl
|
||||
* @returns {SourceId}
|
||||
*/
|
||||
function detectSource(rawUrl) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(rawUrl);
|
||||
} catch {
|
||||
throw new Error("Invalid URL.");
|
||||
}
|
||||
|
||||
const host = parsed.hostname.toLowerCase();
|
||||
const pathname = parsed.pathname.replace(/\/+$/u, "");
|
||||
|
||||
if (
|
||||
host.includes("manhwasusu") ||
|
||||
manhwa.isLikelyManhwaSusuSite(rawUrl) ||
|
||||
/^\/read\/[^/]+/iu.test(pathname)
|
||||
) {
|
||||
if (manhwa.isLikelyManhwaSusuSite(rawUrl) || host.includes("manhwasusu")) {
|
||||
return "manhwasusu";
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
host.includes("mangaread") ||
|
||||
/^\/manga\/[^/]+/iu.test(pathname)
|
||||
) {
|
||||
return "mangaread";
|
||||
}
|
||||
|
||||
// Heuristic fallbacks by path shape
|
||||
if (/^\/read\/[^/]+/iu.test(pathname)) {
|
||||
return "manhwasusu";
|
||||
}
|
||||
if (/^\/manga\/[^/]+/iu.test(pathname)) {
|
||||
return "mangaread";
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
"Unsupported URL. Use a Mangaread `/manga/{slug}/` or ManhwaSusu `/read/{slug}/` series page.",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize listing URL (strip chapter segment if pasted a chapter URL).
|
||||
* @param {string} rawUrl
|
||||
* @param {SourceId} source
|
||||
* @returns {string}
|
||||
*/
|
||||
function normalizeListingUrl(rawUrl, source) {
|
||||
const u = new URL(rawUrl);
|
||||
const segs = u.pathname.replace(/\/+$/u, "").split("/").filter(Boolean);
|
||||
|
||||
if (source === "mangaread") {
|
||||
const mangaIdx = segs.indexOf("manga");
|
||||
if (mangaIdx !== -1 && segs[mangaIdx + 1]) {
|
||||
u.pathname = `/manga/${segs[mangaIdx + 1]}/`;
|
||||
u.search = "";
|
||||
u.hash = "";
|
||||
return u.toString();
|
||||
}
|
||||
}
|
||||
|
||||
if (source === "manhwasusu") {
|
||||
const readIdx = segs.indexOf("read");
|
||||
if (readIdx !== -1 && segs[readIdx + 1]) {
|
||||
u.pathname = `/read/${segs[readIdx + 1]}/`;
|
||||
u.search = "";
|
||||
u.hash = "";
|
||||
return u.toString();
|
||||
}
|
||||
}
|
||||
|
||||
if (!u.pathname.endsWith("/")) {
|
||||
u.pathname = `${u.pathname}/`;
|
||||
}
|
||||
return u.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} chapterUrl
|
||||
* @returns {string} e.g. chapter-22
|
||||
*/
|
||||
function chapterKeyFromUrl(chapterUrl) {
|
||||
try {
|
||||
const segs = new URL(chapterUrl).pathname.replace(/\/+$/u, "").split("/").filter(Boolean);
|
||||
return segs.at(-1) || "chapter";
|
||||
} catch {
|
||||
return "chapter";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} title
|
||||
* @param {SourceId} source
|
||||
* @returns {string|null}
|
||||
*/
|
||||
function chapterNumberFromTitle(title, source) {
|
||||
if (source === "mangaread") {
|
||||
return chapterNumberFromWpTitle(title);
|
||||
}
|
||||
const m = /^Chapter\s+([\d.]+)/iu.exec(title.trim().replace(/\s+/gu, " "));
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {SourceId} source - Site adapter id.
|
||||
* @param {string} dataDir - Absolute data root.
|
||||
* @returns {string} Comics output directory for the source.
|
||||
*/
|
||||
function comicsRoot(source, dataDir) {
|
||||
if (source === "manhwasusu") {
|
||||
return path.join(dataDir, "manhwasusu", "comics");
|
||||
}
|
||||
return path.join(dataDir, "comics");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {SourceId} source
|
||||
* @param {string} comicTitle
|
||||
* @param {string} chapterUrl
|
||||
* @param {string} dataDir
|
||||
* @returns {string} Absolute CBZ path
|
||||
*/
|
||||
function resolveCbzPath(source, comicTitle, chapterUrl, dataDir) {
|
||||
const dirName =
|
||||
source === "manhwasusu"
|
||||
? manhwa.sanitizeComicDirName(comicTitle)
|
||||
: mangareadSanitizeDir(comicTitle);
|
||||
const stem =
|
||||
source === "manhwasusu"
|
||||
? manhwa.sanitizeFsStem(manhwa.stemForManhwaSusuChapterCbz(chapterUrl))
|
||||
: mangareadSanitizeStem(mangareadStem(chapterUrl));
|
||||
return path.join(comicsRoot(source, dataDir), dirName, `${stem}.cbz`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch chapter list for a series listing URL.
|
||||
* @param {string} listingUrl
|
||||
* @param {SourceId} [sourceHint]
|
||||
* @returns {Promise<{ source: SourceId, listingUrl: string, slug: string, comicTitle: string, chapters: Array<{ url: string, title: string, chapter_key: string, chapter_number: string|null }> }>}
|
||||
*/
|
||||
async function fetchSeriesChapters(listingUrl, sourceHint) {
|
||||
const source = sourceHint || detectSource(listingUrl);
|
||||
const normalized = normalizeListingUrl(listingUrl, source);
|
||||
|
||||
if (source === "manhwasusu") {
|
||||
const { chapters, comicTitle } = await manhwa.fetchChapterLinks(normalized);
|
||||
const ordered = manhwaReadingOrder(chapters);
|
||||
const slug = manhwa.seriesSlugFromListingUrl(normalized);
|
||||
return {
|
||||
source,
|
||||
listingUrl: normalized,
|
||||
slug,
|
||||
comicTitle,
|
||||
chapters: ordered.map((ch) => ({
|
||||
url: ch.url,
|
||||
title: ch.title,
|
||||
chapter_key: chapterKeyFromUrl(ch.url),
|
||||
chapter_number: chapterNumberFromTitle(ch.title, source),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const { chapters, comicTitle } = await fetchWpMangaChapterLinks(normalized);
|
||||
const ordered = mangareadReadingOrder(chapters);
|
||||
const slug = mangaSlugFromUrl(normalized);
|
||||
return {
|
||||
source,
|
||||
listingUrl: normalized,
|
||||
slug,
|
||||
comicTitle,
|
||||
chapters: ordered.map((ch) => ({
|
||||
url: ch.url,
|
||||
title: ch.title,
|
||||
chapter_key: chapterKeyFromUrl(ch.url),
|
||||
chapter_number: chapterNumberFromTitle(ch.title, source),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Download one chapter to CBZ.
|
||||
* @param {SourceId} source
|
||||
* @param {string} chapterUrl
|
||||
* @param {string} outputCbzPath
|
||||
* @returns {Promise<{ imageCount: number, outputPath: string }>}
|
||||
*/
|
||||
async function downloadChapter(source, chapterUrl, outputCbzPath) {
|
||||
if (source === "manhwasusu") {
|
||||
return manhwa.chapterUrlToCbz(chapterUrl, outputCbzPath);
|
||||
}
|
||||
return mangareadChapterUrlToCbz(chapterUrl, outputCbzPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string|null|undefined} cbzPath
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function cbzExists(cbzPath) {
|
||||
if (!cbzPath) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return fs.existsSync(cbzPath) && fs.statSync(cbzPath).isFile();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string|null|undefined} cbzPath
|
||||
*/
|
||||
function removeCbzIfExists(cbzPath) {
|
||||
if (cbzPath && fs.existsSync(cbzPath)) {
|
||||
fs.unlinkSync(cbzPath);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
detectSource,
|
||||
normalizeListingUrl,
|
||||
chapterKeyFromUrl,
|
||||
resolveCbzPath,
|
||||
fetchSeriesChapters,
|
||||
downloadChapter,
|
||||
cbzExists,
|
||||
removeCbzIfExists,
|
||||
comicsRoot,
|
||||
};
|
||||
@@ -0,0 +1,172 @@
|
||||
const cron = require("node-cron");
|
||||
const dbApi = require("./db.js");
|
||||
const {
|
||||
fetchSeriesChapters,
|
||||
resolveCbzPath,
|
||||
cbzExists,
|
||||
} = require("./sources.js");
|
||||
|
||||
/**
|
||||
* Sync a title's chapter list from the source site and enqueue missing downloads.
|
||||
* @param {import("better-sqlite3").Database} db - Database handle.
|
||||
* @param {import("./queue.js").DownloadQueue} queue - Download queue.
|
||||
* @param {number} titleId - Title to sync.
|
||||
* @param {string} dataDir - Absolute data root for CBZ paths.
|
||||
* @param {{ enqueue?: boolean }} [opts] - Options; `enqueue` defaults to true.
|
||||
* @returns {Promise<{ added: number, enqueued: number, total: number }>} Sync summary.
|
||||
*/
|
||||
async function syncTitle(db, queue, titleId, dataDir, opts = {}) {
|
||||
const title = dbApi.getTitle(db, titleId);
|
||||
if (!title) {
|
||||
throw new Error(`Title ${String(titleId)} not found.`);
|
||||
}
|
||||
|
||||
const syncJobId = dbApi.createJob(db, {
|
||||
type: "watch_sync",
|
||||
title_id: titleId,
|
||||
message: `Syncing ${title.title}`,
|
||||
});
|
||||
dbApi.updateJob(db, syncJobId, { status: "running" });
|
||||
|
||||
try {
|
||||
const series = await fetchSeriesChapters(title.url, title.source);
|
||||
let added = 0;
|
||||
|
||||
for (const ch of series.chapters) {
|
||||
const cbzPath = resolveCbzPath(
|
||||
title.source,
|
||||
series.comicTitle || title.title,
|
||||
ch.url,
|
||||
dataDir,
|
||||
);
|
||||
const exists = cbzExists(cbzPath);
|
||||
const { inserted } = dbApi.upsertChapter(db, {
|
||||
title_id: titleId,
|
||||
chapter_key: ch.chapter_key,
|
||||
chapter_number: ch.chapter_number,
|
||||
label: ch.title,
|
||||
url: ch.url,
|
||||
status: exists ? "downloaded" : "pending",
|
||||
cbz_path: cbzPath,
|
||||
});
|
||||
if (inserted) {
|
||||
added += 1;
|
||||
}
|
||||
|
||||
// Heal statuses without interrupting an in-flight download
|
||||
const row = db
|
||||
.prepare(
|
||||
"SELECT id, status, cbz_path FROM chapters WHERE title_id = ? AND chapter_key = ?",
|
||||
)
|
||||
.get(titleId, ch.chapter_key);
|
||||
if (row && row.status === "downloading") {
|
||||
continue;
|
||||
}
|
||||
if (row && row.status === "downloaded" && !cbzExists(row.cbz_path)) {
|
||||
dbApi.updateChapter(db, row.id, {
|
||||
status: "pending",
|
||||
error: "CBZ missing on disk",
|
||||
});
|
||||
}
|
||||
if (row && exists && row.status !== "downloaded") {
|
||||
dbApi.updateChapter(db, row.id, {
|
||||
status: "downloaded",
|
||||
cbz_path: cbzPath,
|
||||
downloaded_at: new Date().toISOString(),
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh comic title if site provides a better one
|
||||
if (series.comicTitle && series.comicTitle !== title.title) {
|
||||
dbApi.updateTitle(db, titleId, { title: series.comicTitle });
|
||||
}
|
||||
|
||||
dbApi.touchTitleChecked(db, titleId);
|
||||
|
||||
let enqueued = 0;
|
||||
if (opts.enqueue !== false) {
|
||||
enqueued = queue.enqueueMissingForTitle(titleId, { alsoMissingFiles: true });
|
||||
}
|
||||
|
||||
dbApi.updateJob(db, syncJobId, {
|
||||
status: "done",
|
||||
message: `Sync complete: +${String(added)} chapter(s), ${String(enqueued)} queued`,
|
||||
finished_at: new Date().toISOString(),
|
||||
});
|
||||
|
||||
return {
|
||||
added,
|
||||
enqueued,
|
||||
total: series.chapters.length,
|
||||
};
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
dbApi.updateJob(db, syncJobId, {
|
||||
status: "failed",
|
||||
message: msg,
|
||||
finished_at: new Date().toISOString(),
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Watches titles on a schedule. Each tick syncs at most one due title
|
||||
* (oldest `last_checked_at` first), so titles are staggered instead of
|
||||
* all hitting the source sites at once. Default cadence is hourly wake +
|
||||
* 24h per-title cooldown → each title roughly once per day, unlimited count.
|
||||
* @param {import("better-sqlite3").Database} db - Database handle.
|
||||
* @param {import("./queue.js").DownloadQueue} queue - Download queue.
|
||||
* @param {string} dataDir - Absolute data root.
|
||||
* @param {string} cronExpr - node-cron expression.
|
||||
* @param {{ titleIntervalHours?: number }} [opts] - Per-title cooldown in hours.
|
||||
* @returns {{ stop: () => void }} Handle to stop the cron task.
|
||||
*/
|
||||
function startWatcher(db, queue, dataDir, cronExpr, opts = {}) {
|
||||
if (!cron.validate(cronExpr)) {
|
||||
throw new Error(`Invalid WATCH_CRON expression: ${cronExpr}`);
|
||||
}
|
||||
|
||||
const titleIntervalHours =
|
||||
Number(opts.titleIntervalHours) > 0 ? Number(opts.titleIntervalHours) : 24;
|
||||
|
||||
let busy = false;
|
||||
|
||||
const task = cron.schedule(cronExpr, () => {
|
||||
if (busy) {
|
||||
console.warn("[watcher] previous sync still running; skipping this tick");
|
||||
return;
|
||||
}
|
||||
busy = true;
|
||||
void (async () => {
|
||||
try {
|
||||
const title = dbApi.pickNextTitleDue(db, titleIntervalHours);
|
||||
if (!title) {
|
||||
console.info("[watcher] no titles due this tick");
|
||||
return;
|
||||
}
|
||||
console.info(
|
||||
`[watcher] syncing 1 title: ${title.title} (${title.source})`,
|
||||
);
|
||||
await syncTitle(db, queue, title.id, dataDir, { enqueue: true });
|
||||
} catch (err) {
|
||||
console.error(
|
||||
"[watcher] sync failed:",
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
} finally {
|
||||
busy = false;
|
||||
}
|
||||
})();
|
||||
});
|
||||
|
||||
return {
|
||||
stop() {
|
||||
task.stop();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { syncTitle, startWatcher };
|
||||
@@ -0,0 +1,12 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Watch & Download</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
<script type="module" src="/src/main.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,378 @@
|
||||
<script>
|
||||
let titles = $state([]);
|
||||
let jobs = $state([]);
|
||||
let chaptersByTitle = $state({});
|
||||
let url = $state("");
|
||||
let error = $state("");
|
||||
let busy = $state(false);
|
||||
let loadingChapters = $state({});
|
||||
let expanded = $state({});
|
||||
|
||||
async function api(path, options = {}) {
|
||||
const method = (options.method || "GET").toUpperCase();
|
||||
const headers = { ...(options.headers || {}) };
|
||||
let body = options.body;
|
||||
|
||||
if (body !== undefined && body !== null) {
|
||||
if (!headers["Content-Type"]) {
|
||||
headers["Content-Type"] = "application/json";
|
||||
}
|
||||
} else if (method === "POST" || method === "PUT" || method === "PATCH") {
|
||||
// Fastify rejects Content-Type: application/json with an empty body
|
||||
headers["Content-Type"] = "application/json";
|
||||
body = "{}";
|
||||
}
|
||||
|
||||
const res = await fetch(path, {
|
||||
...options,
|
||||
method,
|
||||
headers,
|
||||
body,
|
||||
});
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
throw new Error(data.error || data.message || res.statusText || "Request failed");
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
const [t, j] = await Promise.all([
|
||||
api("/api/titles"),
|
||||
api("/api/jobs?active=1&limit=20"),
|
||||
]);
|
||||
titles = t.titles || [];
|
||||
jobs = j.jobs || [];
|
||||
}
|
||||
|
||||
async function loadChapters(titleId) {
|
||||
loadingChapters = { ...loadingChapters, [titleId]: true };
|
||||
try {
|
||||
const data = await api(`/api/titles/${titleId}/chapters`);
|
||||
chaptersByTitle = { ...chaptersByTitle, [titleId]: data.chapters || [] };
|
||||
} finally {
|
||||
loadingChapters = { ...loadingChapters, [titleId]: false };
|
||||
}
|
||||
}
|
||||
|
||||
async function toggleExpand(titleId) {
|
||||
const next = !expanded[titleId];
|
||||
expanded = { ...expanded, [titleId]: next };
|
||||
if (next && !chaptersByTitle[titleId]) {
|
||||
await loadChapters(titleId);
|
||||
}
|
||||
}
|
||||
|
||||
async function addTitle(event) {
|
||||
event.preventDefault();
|
||||
error = "";
|
||||
busy = true;
|
||||
try {
|
||||
await api("/api/titles", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ url: url.trim() }),
|
||||
});
|
||||
url = "";
|
||||
await refresh();
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : String(err);
|
||||
} finally {
|
||||
busy = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function syncTitle(id) {
|
||||
error = "";
|
||||
busy = true;
|
||||
try {
|
||||
await api(`/api/titles/${id}/sync`, { method: "POST" });
|
||||
await refresh();
|
||||
if (expanded[id]) {
|
||||
await loadChapters(id);
|
||||
}
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : String(err);
|
||||
} finally {
|
||||
busy = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function toggleEnabled(title) {
|
||||
error = "";
|
||||
try {
|
||||
await api(`/api/titles/${title.id}`, {
|
||||
method: "PATCH",
|
||||
body: JSON.stringify({ enabled: !title.enabled }),
|
||||
});
|
||||
await refresh();
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}
|
||||
|
||||
async function removeTitle(id) {
|
||||
if (!confirm("Remove this title from the watch list?")) {
|
||||
return;
|
||||
}
|
||||
error = "";
|
||||
try {
|
||||
const data = await api(`/api/titles/${id}`, { method: "DELETE" });
|
||||
const nextCh = { ...chaptersByTitle };
|
||||
delete nextCh[id];
|
||||
chaptersByTitle = nextCh;
|
||||
if (data.jobs) {
|
||||
jobs = data.jobs;
|
||||
}
|
||||
await refresh();
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}
|
||||
|
||||
async function redownload(chapterId, titleId) {
|
||||
error = "";
|
||||
try {
|
||||
await api(`/api/chapters/${chapterId}/redownload`, { method: "POST" });
|
||||
await refresh();
|
||||
await loadChapters(titleId);
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}
|
||||
|
||||
async function cancelChapter(chapterId, titleId) {
|
||||
error = "";
|
||||
try {
|
||||
await api(`/api/chapters/${chapterId}/cancel`, { method: "POST" });
|
||||
await refresh();
|
||||
if (titleId && expanded[titleId]) {
|
||||
await loadChapters(titleId);
|
||||
}
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}
|
||||
|
||||
async function clearQueue() {
|
||||
if (!confirm("Clear all waiting download jobs? The current download (if any) will finish.")) {
|
||||
return;
|
||||
}
|
||||
error = "";
|
||||
try {
|
||||
const data = await api("/api/jobs/clear", { method: "POST" });
|
||||
jobs = data.jobs || [];
|
||||
await refresh();
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
void refresh();
|
||||
const id = setInterval(() => {
|
||||
void (async () => {
|
||||
await refresh();
|
||||
for (const [titleId, isOpen] of Object.entries(expanded)) {
|
||||
if (isOpen) {
|
||||
await loadChapters(Number(titleId));
|
||||
}
|
||||
}
|
||||
})();
|
||||
}, 2000);
|
||||
return () => clearInterval(id);
|
||||
});
|
||||
</script>
|
||||
|
||||
<main class="app-shell">
|
||||
<header>
|
||||
<h1>Watch & Download</h1>
|
||||
</header>
|
||||
|
||||
{#if error}
|
||||
<p class="error-banner" role="alert">{error}</p>
|
||||
{/if}
|
||||
|
||||
<form class="add-title" onsubmit={addTitle}>
|
||||
<label>
|
||||
Series URL
|
||||
<input
|
||||
type="url"
|
||||
bind:value={url}
|
||||
placeholder=""
|
||||
required
|
||||
disabled={busy}
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={busy || !url.trim()}>
|
||||
{busy ? "Working…" : "Add & watch"}
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<section class="jobs-strip" aria-live="polite">
|
||||
<div class="section-heading">
|
||||
<h2>Activity</h2>
|
||||
<button
|
||||
type="button"
|
||||
class="outline contrast"
|
||||
disabled={jobs.length === 0}
|
||||
onclick={clearQueue}
|
||||
>
|
||||
Clear queue
|
||||
</button>
|
||||
</div>
|
||||
{#if jobs.length === 0}
|
||||
<p class="muted">No active jobs.</p>
|
||||
{:else}
|
||||
<ul class="job-list">
|
||||
{#each jobs as job (job.id)}
|
||||
<li class="job-row">
|
||||
<div class="job-info">
|
||||
<strong>{job.status}</strong>
|
||||
— {job.message || job.type}
|
||||
{#if job.title_name}
|
||||
<span class="muted">({job.title_name})</span>
|
||||
{/if}
|
||||
</div>
|
||||
{#if job.chapter_id}
|
||||
<button
|
||||
type="button"
|
||||
class="outline contrast job-cancel"
|
||||
onclick={() => cancelChapter(job.chapter_id, job.title_id)}
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
{/if}
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Titles</h2>
|
||||
{#if titles.length === 0}
|
||||
<p class="muted">No titles yet. Add a series URL above.</p>
|
||||
{:else}
|
||||
{#each titles as title (title.id)}
|
||||
<article>
|
||||
<header>
|
||||
<div class="title-meta">
|
||||
<strong>{title.title}</strong>
|
||||
<span class="muted">{title.source}</span>
|
||||
<span class="progress-text">
|
||||
{title.downloaded} / {title.total} downloaded
|
||||
{#if title.downloading > 0}
|
||||
· {title.downloading} downloading
|
||||
{/if}
|
||||
{#if title.failed > 0}
|
||||
· {title.failed} failed
|
||||
{/if}
|
||||
</span>
|
||||
</div>
|
||||
<p class="muted url-line">
|
||||
<a href={title.url} target="_blank" rel="noreferrer">{title.url}</a>
|
||||
{#if title.last_checked_at}
|
||||
· last checked {title.last_checked_at}
|
||||
{/if}
|
||||
· watch {title.enabled ? "on" : "off"}
|
||||
</p>
|
||||
<div class="row-actions">
|
||||
<button type="button" class="outline" onclick={() => toggleExpand(title.id)}>
|
||||
{expanded[title.id] ? "Hide chapters" : "Show chapters"}
|
||||
</button>
|
||||
<button type="button" class="outline" disabled={busy} onclick={() => syncTitle(title.id)}>
|
||||
Sync now
|
||||
</button>
|
||||
<button type="button" class="outline secondary" onclick={() => toggleEnabled(title)}>
|
||||
{title.enabled ? "Disable watch" : "Enable watch"}
|
||||
</button>
|
||||
<button type="button" class="outline contrast" onclick={() => removeTitle(title.id)}>
|
||||
Remove
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{#if expanded[title.id]}
|
||||
{#if loadingChapters[title.id] && !chaptersByTitle[title.id]}
|
||||
<p class="muted">Loading chapters…</p>
|
||||
{:else}
|
||||
<ul class="chapter-list">
|
||||
{#each chaptersByTitle[title.id] || [] as ch (ch.id)}
|
||||
<li class="chapter-card">
|
||||
<div class="chapter-label">{ch.label}</div>
|
||||
<span class={`status status-${ch.status}`}>{ch.status}</span>
|
||||
{#if ch.error}
|
||||
<div class="muted chapter-error">{ch.error}</div>
|
||||
{/if}
|
||||
<div class="chapter-actions">
|
||||
{#if ch.status === "downloading"}
|
||||
<button
|
||||
type="button"
|
||||
class="outline contrast"
|
||||
onclick={() => cancelChapter(ch.id, title.id)}
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
{/if}
|
||||
<button
|
||||
type="button"
|
||||
class="outline"
|
||||
onclick={() => redownload(ch.id, title.id)}
|
||||
>
|
||||
Redownload
|
||||
</button>
|
||||
</div>
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Chapter</th>
|
||||
<th>Status</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each chaptersByTitle[title.id] || [] as ch (ch.id)}
|
||||
<tr>
|
||||
<td>{ch.label}</td>
|
||||
<td>
|
||||
<span class={`status status-${ch.status}`}>{ch.status}</span>
|
||||
{#if ch.error}
|
||||
<div class="muted">{ch.error}</div>
|
||||
{/if}
|
||||
</td>
|
||||
<td>
|
||||
<div class="chapter-actions">
|
||||
{#if ch.status === "downloading"}
|
||||
<button
|
||||
type="button"
|
||||
class="outline contrast"
|
||||
onclick={() => cancelChapter(ch.id, title.id)}
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
{/if}
|
||||
<button
|
||||
type="button"
|
||||
class="outline"
|
||||
onclick={() => redownload(ch.id, title.id)}
|
||||
>
|
||||
Redownload
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{/if}
|
||||
{/if}
|
||||
</article>
|
||||
{/each}
|
||||
{/if}
|
||||
</section>
|
||||
</main>
|
||||
+332
@@ -0,0 +1,332 @@
|
||||
:root {
|
||||
--pico-font-family: "Segoe UI", system-ui, sans-serif;
|
||||
}
|
||||
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
html {
|
||||
overflow-x: clip;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
overflow-x: clip;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
#app {
|
||||
width: 100%;
|
||||
min-width: 0;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
/* Fluid layout — avoid Pico .container stepped max-widths (510/700/950px) */
|
||||
.app-shell {
|
||||
display: block;
|
||||
width: 100%;
|
||||
max-width: min(960px, 100%);
|
||||
margin-inline: auto;
|
||||
padding-inline: max(1rem, env(safe-area-inset-left, 0px), env(safe-area-inset-right, 0px));
|
||||
padding-block: 1.25rem;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: clamp(1.5rem, 5vw, 2rem);
|
||||
line-height: 1.2;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.muted {
|
||||
color: var(--pico-muted-color);
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.url-line,
|
||||
.url-line a {
|
||||
overflow-wrap: anywhere;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.row-actions {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 0.5rem;
|
||||
margin-top: 0.75rem;
|
||||
}
|
||||
|
||||
.row-actions button {
|
||||
width: 100%;
|
||||
margin-bottom: 0;
|
||||
white-space: normal;
|
||||
line-height: 1.2;
|
||||
padding-inline: 0.5rem;
|
||||
}
|
||||
|
||||
.status {
|
||||
text-transform: uppercase;
|
||||
font-size: 0.75rem;
|
||||
letter-spacing: 0.04em;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.status-pending {
|
||||
color: #a67c00;
|
||||
}
|
||||
|
||||
.status-downloading {
|
||||
color: #0b6bcb;
|
||||
}
|
||||
|
||||
.status-downloaded {
|
||||
color: #1b7f3a;
|
||||
}
|
||||
|
||||
.status-failed {
|
||||
color: #c62828;
|
||||
}
|
||||
|
||||
.progress-text {
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.error-banner {
|
||||
color: #c62828;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.jobs-strip {
|
||||
margin-bottom: 1.5rem;
|
||||
}
|
||||
|
||||
.section-heading {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 0.75rem;
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
.section-heading h2 {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.section-heading button {
|
||||
width: auto;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.jobs-strip ul {
|
||||
margin-bottom: 0;
|
||||
padding-inline-start: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.jobs-strip li {
|
||||
overflow-wrap: anywhere;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.job-list {
|
||||
display: grid;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.job-row {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 0.5rem 0.75rem;
|
||||
padding: 0.5rem 0.65rem;
|
||||
border: 1px solid var(--pico-muted-border-color);
|
||||
border-radius: var(--pico-border-radius);
|
||||
}
|
||||
|
||||
.job-info {
|
||||
flex: 1 1 12rem;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.job-cancel {
|
||||
width: auto;
|
||||
margin-bottom: 0;
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
@media (max-width: 639px) {
|
||||
.job-cancel {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
.title-meta {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.25rem;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.title-meta strong {
|
||||
overflow-wrap: anywhere;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
article {
|
||||
min-width: 0;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
article header {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
section {
|
||||
min-width: 0;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
form.add-title {
|
||||
display: grid;
|
||||
gap: 0.75rem;
|
||||
margin-bottom: 2rem;
|
||||
min-width: 0;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
form.add-title label {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
form.add-title input {
|
||||
max-width: 100%;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
form.add-title button {
|
||||
width: 100%;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.table-wrap {
|
||||
width: 100%;
|
||||
max-width: 100%;
|
||||
overflow-x: auto;
|
||||
-webkit-overflow-scrolling: touch;
|
||||
margin-top: 0.75rem;
|
||||
}
|
||||
|
||||
.table-wrap table {
|
||||
margin-bottom: 0;
|
||||
min-width: 28rem;
|
||||
}
|
||||
|
||||
.table-wrap th,
|
||||
.table-wrap td {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.table-wrap td:first-child {
|
||||
white-space: normal;
|
||||
overflow-wrap: anywhere;
|
||||
max-width: 12rem;
|
||||
}
|
||||
|
||||
.chapter-list {
|
||||
display: grid;
|
||||
gap: 0.75rem;
|
||||
margin-top: 0.75rem;
|
||||
list-style: none;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.chapter-card {
|
||||
border: 1px solid var(--pico-muted-border-color);
|
||||
border-radius: var(--pico-border-radius);
|
||||
padding: 0.75rem;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.chapter-card .chapter-label {
|
||||
font-weight: 600;
|
||||
overflow-wrap: anywhere;
|
||||
margin-bottom: 0.35rem;
|
||||
}
|
||||
|
||||
.chapter-card .chapter-error {
|
||||
margin-top: 0.25rem;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.chapter-card button {
|
||||
width: 100%;
|
||||
margin-top: 0.5rem;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.chapter-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.5rem;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.chapter-actions button {
|
||||
margin-top: 0;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
@media (max-width: 639px) {
|
||||
.chapter-actions button {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
@media (min-width: 640px) {
|
||||
.app-shell {
|
||||
padding-inline: max(1.5rem, env(safe-area-inset-left, 0px), env(safe-area-inset-right, 0px));
|
||||
padding-block: 1.5rem;
|
||||
}
|
||||
|
||||
form.add-title {
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
align-items: end;
|
||||
}
|
||||
|
||||
form.add-title button {
|
||||
width: auto;
|
||||
}
|
||||
|
||||
.title-meta {
|
||||
flex-direction: row;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.75rem 1.25rem;
|
||||
align-items: baseline;
|
||||
}
|
||||
|
||||
.row-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
grid-template-columns: unset;
|
||||
}
|
||||
|
||||
.row-actions button {
|
||||
width: auto;
|
||||
}
|
||||
|
||||
.chapter-list {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 639px) {
|
||||
.table-wrap {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import "@picocss/pico/css/pico.min.css";
|
||||
import "./app.css";
|
||||
import { mount } from "svelte";
|
||||
import App from "./App.svelte";
|
||||
|
||||
const target = document.getElementById("app");
|
||||
if (target) {
|
||||
mount(App, { target });
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { vitePreprocess } from "@sveltejs/vite-plugin-svelte";
|
||||
|
||||
/** @type {import('@sveltejs/vite-plugin-svelte').SvelteConfig} */
|
||||
const config = {
|
||||
preprocess: vitePreprocess(),
|
||||
};
|
||||
|
||||
export default config;
|
||||
@@ -0,0 +1,20 @@
|
||||
import { defineConfig } from "vite";
|
||||
import { svelte } from "@sveltejs/vite-plugin-svelte";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
plugins: [svelte()],
|
||||
build: {
|
||||
outDir: path.resolve(__dirname, "../public"),
|
||||
emptyOutDir: true,
|
||||
},
|
||||
server: {
|
||||
proxy: {
|
||||
"/api": "http://127.0.0.1:3000",
|
||||
},
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user