Add initial project structure with Docker support and chapter downloading functionality

- Created .dockerignore and .gitignore files to exclude unnecessary files from Docker builds.
- Added Dockerfile for multi-stage build setup, including dependencies and production environment configuration.
- Introduced docker-compose.yml for service orchestration, including FlareSolverr and the main application.
- Implemented core functionality for downloading manga chapters, including utilities for handling file paths and chapter metadata.
- Added ESLint configuration for code quality and consistency.
- Included JSDoc configuration for documentation generation.
- Established initial package.json and package-lock.json with necessary dependencies for the project.
This commit is contained in:
2026-08-03 07:35:37 +07:00
commit 3799ad59f8
35 changed files with 8809 additions and 0 deletions
Vendored
BIN
View File
Binary file not shown.
+8
View File
@@ -0,0 +1,8 @@
node_modules
.git
docs
data
public
.env
.env.*
*.md
+5
View File
@@ -0,0 +1,5 @@
node_modules/
docs/
data/
public/*
!public/.gitkeep
+44
View File
@@ -0,0 +1,44 @@
# syntax=docker/dockerfile:1
FROM node:22-bookworm-slim AS web-build
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends python3 make g++ \
&& rm -rf /var/lib/apt/lists/*
COPY package.json package-lock.json ./
RUN npm ci
COPY web ./web
COPY src ./src
RUN npm run build:web
FROM node:22-bookworm-slim AS runtime
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends python3 make g++ \
&& rm -rf /var/lib/apt/lists/*
ENV NODE_ENV=production
ENV DATA_DIR=/app/data
ENV PORT=3000
ENV WATCH_CRON="0 * * * *"
ENV WATCH_TITLE_INTERVAL_HOURS="24"
COPY package.json package-lock.json ./
RUN npm ci --omit=dev \
&& apt-get purge -y python3 make g++ \
&& apt-get autoremove -y \
&& rm -rf /var/lib/apt/lists/*
COPY src ./src
COPY --from=web-build /app/public ./public
RUN mkdir -p /app/data
EXPOSE 3000
VOLUME ["/app/data"]
CMD ["node", "src/server/index.js"]
+26
View File
@@ -0,0 +1,26 @@
services:
flaresolverr:
image: ghcr.io/flaresolverr/flaresolverr:latest
environment:
LOG_LEVEL: info
ports:
- "8191:8191"
restart: unless-stopped
cxyz:
build: .
ports:
- "3000:3000"
environment:
PORT: "3000"
DATA_DIR: /app/data
# Wake hourly; sync at most one due title per wake.
WATCH_CRON: "0 * * * *"
# Each title is eligible again after this many hours (daily).
WATCH_TITLE_INTERVAL_HOURS: "24"
FLARESOLVERR_URL: http://flaresolverr:8191/v1
volumes:
- ./data:/app/data
depends_on:
- flaresolverr
restart: unless-stopped
+18
View File
@@ -0,0 +1,18 @@
import js from "@eslint/js";
import globals from "globals";
import jsdoc from "eslint-plugin-jsdoc";
export default [
{
ignores: ["node_modules/**", "docs/**", "public/**", "web/**", "data/**"],
},
js.configs.recommended,
jsdoc.configs["flat/recommended"],
{
files: ["**/*.js"],
languageOptions: {
ecmaVersion: 2024,
globals: globals.node,
},
},
];
+11
View File
@@ -0,0 +1,11 @@
{
"source": {
"include": ["./src"],
"includePattern": "\\.js$",
"excludePattern": "(node_modules/|docs)"
},
"opts": {
"destination": "./docs",
"recurse": true
}
}
+3961
View File
File diff suppressed because it is too large Load Diff
+45
View File
@@ -0,0 +1,45 @@
{
"name": "cxyz",
"version": "1.0.0",
"description": "",
"main": "src/index.js",
"scripts": {
"start": "node src/index.js",
"web": "node src/server/index.js",
"dev:web": "vite --config web/vite.config.mjs",
"build:web": "vite build --config web/vite.config.mjs",
"lint": "eslint .",
"lint:fix": "eslint . --fix",
"docs": "jsdoc -c jsdoc.conf.json",
"mangaread:fetch": "node src/mangareadFetchChapters.js",
"manhwasusu:fetch": "node src/manhwasusu/fetchChapters.cli.js",
"manhwasusu:chapter-cbz": "node src/manhwasusu/downloadChapterCbz.js",
"manhwasusu:chapters-cbz": "node src/manhwasusu/downloadChaptersQueue.js",
"mangaread:chapter-cbz": "node src/downloadChapterCbz.js",
"mangaread:chapters-cbz": "node src/downloadChaptersQueue.js",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@fastify/static": "^10.1.2",
"@picocss/pico": "^2.1.1",
"adm-zip": "^0.5.17",
"axios": "^1.16.1",
"better-sqlite3": "^13.0.2",
"fastify": "^5.11.0",
"node-cron": "^4.6.0",
"node-html-parser": "^7.1.0"
},
"devDependencies": {
"@eslint/js": "^10.0.1",
"@sveltejs/vite-plugin-svelte": "^7.2.0",
"eslint": "^10.4.0",
"eslint-plugin-jsdoc": "^62.9.0",
"globals": "^17.6.0",
"jsdoc": "^4.0.5",
"svelte": "^5.56.8",
"vite": "^8.2.0"
}
}
+160
View File
@@ -0,0 +1,160 @@
const fs = require("fs/promises");
const path = require("path");
const os = require("os");
const AdmZip = require("adm-zip");
const { downloadWpMangaChapterImages, extractWpMangaComicTitle, fetchPageHtml, mangaSlugFromUrl } =
require("./mangareadFetchChapters.js");
/**
* @param {string} chapterPageUrl - Full chapter URL.
* @returns {string} Suggested base name without extension.
*/
function stemForCbzFilename(chapterPageUrl) {
try {
const { pathname } = new URL(chapterPageUrl);
const segs = pathname.replace(/\/+$/, "").split("/").filter(Boolean);
const mangaIdx = segs.indexOf("manga");
if (mangaIdx !== -1 && segs.length >= mangaIdx + 3) {
return `${segs[mangaIdx + 1]}-${segs[mangaIdx + 2]}`;
}
if (segs.length >= 2) {
return segs.slice(-2).join("-");
}
return segs.at(-1) || "chapter";
} catch {
return "chapter";
}
}
/**
* @param {string} stem - File base name.
* @returns {string} Sanitized segment.
*/
function sanitizeFsStem(stem) {
const cleaned = stem.replace(/[^a-z0-9._-]+/gi, "_").replace(/^_+|_+$/g, "");
return cleaned.length > 0 ? cleaned.slice(0, 180) : "chapter";
}
/**
* @param {string} name - Human-readable comic title.
* @returns {string} Filesystem-safe directory name.
*/
function sanitizeComicDirName(name) {
const collapsed = name.trim().replace(/\s+/g, " ");
const cleaned = [...collapsed]
.filter((ch) => {
const code = ch.charCodeAt(0);
if (code < 32 || code === 127) {
return false;
}
return !/[<>:"/\\|?*]/.test(ch);
})
.join("")
.replace(/\.+$/g, "")
.trim();
return cleaned.length > 0 ? cleaned.slice(0, 180) : "comic";
}
/**
* @param {string|undefined} cliOut - Second CLI argument, if any.
* @param {string} defaultStem - Used when `cliOut` is a directory or omitted.
* @param {string} [comicTitle] - When set, default output goes under this subdirectory.
* @returns {string} Absolute path ending in `.cbz`.
*/
function resolveCliOutputPath(cliOut, defaultStem, comicTitle) {
const stem = sanitizeFsStem(defaultStem);
const baseDir =
typeof comicTitle === "string" && comicTitle.trim().length > 0
? path.join(process.cwd(), "data", "comics", sanitizeComicDirName(comicTitle))
: path.join(process.cwd(), "data", "comics");
if (cliOut === undefined || cliOut.trim() === "") {
return path.join(baseDir, `${stem}.cbz`);
}
const trimmed = cliOut.trim();
const abs = path.resolve(trimmed);
if (abs.toLowerCase().endsWith(".cbz")) {
return abs;
}
return path.join(abs, `${stem}.cbz`);
}
/**
* Downloads chapter images to a temp folder, packs them into a `.cbz` zip, then deletes temp files.
* @param {string} chapterUrl - Chapter reader URL.
* @param {string} outputCbzPath - Absolute `.cbz` path to write.
* @returns {Promise<{ imageCount: number, outputPath: string }>} Summary of written pages and archive path.
*/
async function chapterUrlToCbz(chapterUrl, outputCbzPath) {
const absOut = outputCbzPath.toLowerCase().endsWith(".cbz")
? path.resolve(outputCbzPath)
: `${path.resolve(outputCbzPath)}.cbz`;
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "cxyz-chapter-"));
try {
const saved = await downloadWpMangaChapterImages(chapterUrl, tmpRoot);
if (saved.length === 0) {
throw new Error("No chapter images were found (try another URL or chapter).");
}
await fs.mkdir(path.dirname(absOut), { recursive: true });
const zip = new AdmZip();
for (const { filePath } of saved) {
zip.addLocalFile(filePath, "", path.basename(filePath));
}
zip.writeZip(absOut);
return { imageCount: saved.length, outputPath: absOut };
} finally {
await fs.rm(tmpRoot, { recursive: true, force: true }).catch(() => {});
}
}
/**
* CLI: `node src/downloadChapterCbz.js <chapterUrl> [path/to/file.cbz|path/to/dir/]`
* @returns {Promise<void>}
*/
async function main() {
const chapterUrl = process.argv[2];
if (!chapterUrl || chapterUrl.startsWith("-")) {
console.error(
"Usage: node src/downloadChapterCbz.js <chapterUrl> [output.cbz|outputDirectory]",
);
process.exitCode = 1;
return;
}
const outArg = process.argv[3];
const stem = stemForCbzFilename(chapterUrl);
let comicTitle = mangaSlugFromUrl(chapterUrl);
try {
const html = await fetchPageHtml(chapterUrl);
comicTitle = extractWpMangaComicTitle(html) || comicTitle;
} catch {
/* keep slug fallback */
}
const target = resolveCliOutputPath(outArg, stem, comicTitle);
try {
const { imageCount, outputPath } = await chapterUrlToCbz(chapterUrl, target);
console.log(`Wrote ${String(imageCount)} page(s) → ${outputPath}`);
} catch (err) {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
}
}
module.exports = {
chapterUrlToCbz,
sanitizeComicDirName,
sanitizeFsStem,
stemForCbzFilename,
resolveCliOutputPath,
};
if (require.main === module) {
main();
}
+426
View File
@@ -0,0 +1,426 @@
const path = require("path");
const { fetchWpMangaChapterLinks } = require("./mangareadFetchChapters.js");
const { chapterUrlToCbz, sanitizeComicDirName, stemForCbzFilename } = require("./downloadChapterCbz.js");
/**
* @typedef {object} ParsedCli
* @property {boolean} [help] - When set from `--help`, print usage and exit.
* @property {string|undefined} listingUrl - Page containing `li.wp-manga-chapter` links.
* @property {boolean} listOnly - Print numbered chapters and exit.
* @property {boolean} allChapters - Queue every chapter in list order.
* @property {string|undefined} selectSpec - Comma-separated **list indices** on the printed menu (oldest first: row 1 ≈ Chapter 1), e.g. `1,3,5-7`.
* @property {string|undefined} numbersSpec - Comma-separated **chapter numbers** from titles, e.g. `22,23,24` or `22-24`.
* @property {string} outDir - Directory for `.cbz` files.
*/
/**
* Parses CLI arguments after `node .../downloadChaptersQueue.js`.
* @param {string[]} argv - `process.argv`.
* @returns {ParsedCli} Parsed options.
*/
function parseCli(argv) {
/** @type {ParsedCli} */
const result = {
help: false,
listingUrl: undefined,
listOnly: false,
allChapters: false,
selectSpec: undefined,
numbersSpec: undefined,
outDir: path.join(process.cwd(), "data", "comics"),
};
for (let i = 2; i < argv.length; i += 1) {
const a = argv[i];
if (a === "--help" || a === "-h") {
result.help = true;
} else if (a === "--list-only" || a === "-l") {
result.listOnly = true;
} else if (a === "--all") {
result.allChapters = true;
} else if (a === "--select" || a === "-s") {
const next = argv[i + 1];
if (next === undefined || next.startsWith("-")) {
throw new Error("Missing value for --select");
}
result.selectSpec = next;
i += 1;
} else if (a === "--numbers" || a === "-n") {
const next = argv[i + 1];
if (next === undefined || next.startsWith("-")) {
throw new Error("Missing value for --numbers");
}
result.numbersSpec = next;
i += 1;
} else if (a === "--out" || a === "-o") {
const next = argv[i + 1];
if (next === undefined || next.startsWith("-")) {
throw new Error("Missing value for --out");
}
result.outDir = path.resolve(next);
i += 1;
} else if (!a.startsWith("-")) {
if (result.listingUrl) {
throw new Error(`Unexpected extra argument: ${a}`);
}
result.listingUrl = a;
} else {
throw new Error(`Unknown option: ${a}`);
}
}
return result;
}
/**
* WPManga lists chapters **newest first** in the HTML. We reverse so the numbered
* menu and `--select` indices match reading order (line 1 ≈ Chapter 1).
* @param {import("./mangareadFetchChapters.js").ChapterLink[]} chapters - Fetched in site order.
* @returns {import("./mangareadFetchChapters.js").ChapterLink[]} New array, oldest chapter first.
*/
function chaptersInReadingOrder(chapters) {
return [...chapters].reverse();
}
/**
* Builds sorted unique 1-based indices from a spec like `1`, `1,3`, `5-10,2`.
* @param {string} spec - Selection string.
* @param {number} count - Number of chapters (max index).
* @returns {number[]} Ascending unique indices in [1, count].
*/
function parseChapterSelection(spec, count) {
if (count <= 0) {
throw new Error("Chapter list is empty; cannot select indices.");
}
const trimmed = spec.trim();
if (trimmed.length === 0) {
throw new Error("--select value is empty.");
}
/** @type {Set<number>} */
const seen = new Set();
for (const rawPart of trimmed.split(",")) {
const part = rawPart.trim();
if (part.length === 0) {
continue;
}
if (/^\d+$/.test(part)) {
const n = Number.parseInt(part, 10);
if (n < 1 || n > count) {
throw new Error(
`Index ${String(n)} is out of range (valid: 1${String(count)}).`,
);
}
seen.add(n);
continue;
}
const rangeMatch = /^(\d+)\s*-\s*(\d+)$/.exec(part);
if (rangeMatch) {
const lo = Number.parseInt(rangeMatch[1], 10);
const hi = Number.parseInt(rangeMatch[2], 10);
const a = Math.min(lo, hi);
const b = Math.max(lo, hi);
for (let i = a; i <= b; i += 1) {
if (i < 1 || i > count) {
throw new Error(
`Range ${part} includes invalid index ${String(i)} (valid: 1${String(count)}).`,
);
}
seen.add(i);
}
continue;
}
throw new Error(`Invalid --select segment: "${part}"`);
}
if (seen.size === 0) {
throw new Error("No indices parsed from --select.");
}
return [...seen].sort((x, y) => x - y);
}
/**
* Parses "Chapter 22" / "chapter 160.5" style titles from WPManga lists.
* @param {string} title - Anchor text.
* @returns {string|null} Numeric suffix or null.
*/
function chapterNumberFromWpTitle(title) {
const m = /^Chapter\s+([\d.]+)\s*$/iu.exec(title.trim().replace(/\s+/g, " "));
return m ? m[1] : null;
}
/**
* Expands `"22,23,24"` or `"22-24"` into ordered chapter-number strings.
* Integer ranges only (22-24 → 22,23,24). Use explicit `160.5` for decimals.
* @param {string} spec - User `--numbers` argument.
* @returns {string[]} Ordered unique chapter numbers to resolve.
*/
function parseMangaChapterNumbersSpec(spec) {
const trimmed = spec.trim();
if (trimmed.length === 0) {
throw new Error("--numbers value is empty.");
}
/** @type {string[]} */
const ordered = [];
/** @type {Set<string>} */
const seen = new Set();
for (const rawPart of trimmed.split(",")) {
const part = rawPart.trim();
if (part.length === 0) {
continue;
}
const rangeMatch = /^(\d+)\s*-\s*(\d+)$/.exec(part);
if (rangeMatch) {
const lo = Number.parseInt(rangeMatch[1], 10);
const hi = Number.parseInt(rangeMatch[2], 10);
const a = Math.min(lo, hi);
const b = Math.max(lo, hi);
for (let n = a; n <= b; n += 1) {
const k = String(n);
if (!seen.has(k)) {
seen.add(k);
ordered.push(k);
}
}
continue;
}
if (/^\d+(\.\d+)?$/.test(part)) {
if (!seen.has(part)) {
seen.add(part);
ordered.push(part);
}
continue;
}
throw new Error(`Invalid --numbers segment: "${part}"`);
}
if (ordered.length === 0) {
throw new Error("No chapter numbers parsed from --numbers.");
}
return ordered;
}
/**
* Resolves chapter rows by numeric title (e.g. Chapter 22).
* @param {string[]} orderedNums - From {@link parseMangaChapterNumbersSpec}.
* @param {import("./mangareadFetchChapters.js").ChapterLink[]} chapters - Full list.
* @returns {Array<{ url: string, title: string, pickLabel: string, chapterNumber: string }>} Queue rows with URLs and labels.
*/
function queueItemsByChapterNumbers(orderedNums, chapters) {
/** @type {Map<string, import("./mangareadFetchChapters.js").ChapterLink>} */
const first = new Map();
for (const ch of chapters) {
const num = chapterNumberFromWpTitle(ch.title);
if (num !== null && !first.has(num)) {
first.set(num, ch);
}
}
return orderedNums.map((num) => {
const ch = first.get(num);
if (!ch) {
throw new Error(
`No chapter titled "Chapter ${num}" in the listing.` +
" Use --list-only and check spelling (decimals like 160.5 must be listed explicitly).",
);
}
return {
url: ch.url,
title: ch.title,
pickLabel: `Ch.${num}`,
chapterNumber: num,
};
});
}
/**
* Prints numbered chapters for manual `--select` choice.
* @param {import("./mangareadFetchChapters.js").ChapterLink[]} chapters - Chapter rows.
* @returns {void}
*/
function printChapterMenu(chapters) {
const digits = String(chapters.length).length;
for (let i = 0; i < chapters.length; i += 1) {
const n = i + 1;
const pad = String(n).padStart(digits, " ");
const title = chapters[i].title.replace(/\s+/g, " ").trim();
console.log(`${pad}. ${title}`);
console.log(` ${chapters[i].url}`);
}
}
/**
* @param {number[]} indices1 - Sorted unique 1-based indices.
* @param {import("./mangareadFetchChapters.js").ChapterLink[]} chapters - Full list.
* @returns {Array<{ url: string, title: string, index: number, pickLabel: string }>} One item per requested index.
*/
function chaptersByIndices(indices1, chapters) {
return indices1.map((idx) => ({
index: idx,
title: chapters[idx - 1].title,
url: chapters[idx - 1].url,
pickLabel: `#${String(idx)}`,
}));
}
/**
* Fetches chapter list, queues selected chapters, downloads one `.cbz` at a time.
* @returns {Promise<void>}
*/
async function main() {
let parsed;
try {
parsed = parseCli(process.argv);
} catch (err) {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
return;
}
if (parsed.help) {
console.log(`Usage:
node src/downloadChaptersQueue.js <listing-page-url> [--list-only|-l]
node src/downloadChaptersQueue.js <listing-page-url> --numbers|-n <spec> [--out|-o <dir>]
node src/downloadChaptersQueue.js <listing-page-url> --select|-s <spec> [--out|-o <dir>]
node src/downloadChaptersQueue.js <listing-page-url> --all [--out|-o <dir>]
Any chapter URL for that manga usually works as <listing-page-url> (same HTML includes the chapter list).
--numbers spec: chapter numbers from link titles ("Chapter 22"), e.g. 22,23,24 or 22-24
Decimals (e.g. 160.5) must be listed explicitly; integer ranges expand to whole numbers only.
--select spec: 1-based row numbers on the **printed list** (oldest first; row 1 is Chapter 1),
e.g. 1,3,5-10. This is the menu index, not a substring of the title (use --numbers for that).
Chapters are processed in list order for --select/--all, or in the order given for --numbers.
Default --out: data/comics/<comic title>/ (under current working directory).`);
return;
}
if (!parsed.listingUrl) {
console.error(
"Usage: node src/downloadChaptersQueue.js <listing-page-url> [options]\nTry --help for details.",
);
process.exitCode = 1;
return;
}
let chapters;
let comicTitle;
try {
({ chapters, comicTitle } = await fetchWpMangaChapterLinks(parsed.listingUrl));
chapters = chaptersInReadingOrder(chapters);
} catch (err) {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
return;
}
console.log(`Found ${String(chapters.length)} chapter link(s) on listing page.`);
printChapterMenu(chapters);
if (parsed.listOnly) {
return;
}
const wantsAll = parsed.allChapters;
const wantsSelect = Boolean(parsed.selectSpec);
const wantsNumbers = Boolean(parsed.numbersSpec);
const pickModes = [wantsAll, wantsSelect, wantsNumbers].filter(Boolean).length;
if (pickModes === 0) {
console.error(
"Choose: --numbers <spec>, --select <spec>, or --all (or use --list-only to exit after listing).",
);
process.exitCode = 1;
return;
}
if (pickModes > 1) {
console.error("Use only one of --all, --select, or --numbers.");
process.exitCode = 1;
return;
}
/** @type {Array<{ url: string, title: string, pickLabel: string }>} */
let queue;
try {
if (wantsAll) {
const indices = Array.from({ length: chapters.length }, (_, i) => i + 1);
queue = chaptersByIndices(indices, chapters);
} else if (wantsSelect) {
const indices = parseChapterSelection(
/** @type {string} */ (parsed.selectSpec),
chapters.length,
);
queue = chaptersByIndices(indices, chapters);
} else {
const orderedNums = parseMangaChapterNumbersSpec(
/** @type {string} */ (parsed.numbersSpec),
);
queue = queueItemsByChapterNumbers(orderedNums, chapters);
}
} catch (err) {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
return;
}
const outDir = path.join(parsed.outDir, sanitizeComicDirName(comicTitle));
console.log(`Comic: ${comicTitle}`);
console.log(`Output directory: ${outDir}`);
let failedCount = 0;
let done = 0;
for (const item of queue) {
done += 1;
const stem = stemForCbzFilename(item.url);
const targetPath = path.join(outDir, `${stem}.cbz`);
console.log(
`\n[${String(done)}/${String(queue.length)}] ${item.pickLabel} ${item.title}`,
);
console.log(`${item.url}`);
try {
const { imageCount, outputPath } = await chapterUrlToCbz(item.url, targetPath);
console.log(`${String(imageCount)} page(s) → ${outputPath}`);
} catch (err) {
console.error(`${err instanceof Error ? err.message : err}`);
failedCount += 1;
}
}
if (failedCount > 0) {
console.error(
`\n${String(failedCount)} chapter(s) failed (${String(queue.length - failedCount)} succeeded).`,
);
process.exitCode = 1;
}
}
module.exports = {
chapterNumberFromWpTitle,
chaptersByIndices,
chaptersInReadingOrder,
parseChapterSelection,
parseCli,
parseMangaChapterNumbersSpec,
printChapterMenu,
queueItemsByChapterNumbers,
};
if (require.main === module) {
main().catch((err) => {
console.error(err);
process.exitCode = 1;
});
}
+112
View File
@@ -0,0 +1,112 @@
const axios = require("axios");
/**
* Turn upstream scrape/HTTP failures into a short UI-friendly message.
* @param {unknown} err - Caught error (often Axios).
* @param {string} [siteLabel] - e.g. "ManhwaSusu".
* @returns {string}
*/
function formatUpstreamError(err, siteLabel) {
const label = siteLabel || "Source site";
const status = err && typeof err === "object" && "response" in err
? /** @type {{ response?: { status?: number } }} */ (err).response?.status
: undefined;
const code =
err && typeof err === "object" && "code" in err
? /** @type {{ code?: string }} */ (err).code
: undefined;
const raw = err instanceof Error ? err.message : String(err);
if (status === 403) {
return (
`${label} blocked the request (HTTP 403), usually Cloudflare. ` +
`Set FLARESOLVERR_URL (e.g. http://flaresolverr:8191/v1) and retry.`
);
}
if (status === 404) {
return `${label} returned 404 — check the series URL.`;
}
if (status === 429) {
return `${label} rate-limited the request (HTTP 429). Try again later.`;
}
if (status && status >= 500) {
return `${label} is unavailable (HTTP ${String(status)}).`;
}
if (code === "ECONNABORTED" || /timeout/iu.test(raw)) {
return `${label} request timed out.`;
}
if (code === "ENOTFOUND" || code === "ECONNREFUSED") {
return `Could not reach ${label}: ${raw}`;
}
if (/status code 403/iu.test(raw)) {
return (
`${label} blocked the request (HTTP 403), usually Cloudflare. ` +
`Set FLARESOLVERR_URL (e.g. http://flaresolverr:8191/v1) and retry.`
);
}
return raw;
}
/**
* Fetch HTML, optionally via FlareSolverr when `FLARESOLVERR_URL` is set.
* @param {string} pageUrl - Absolute page URL.
* @param {import("axios").AxiosRequestConfig} [directConfig] - Used for direct axios GET.
* @returns {Promise<string>} HTML body.
*/
async function fetchHtmlWithOptionalFlareSolverr(pageUrl, directConfig = {}) {
const flareBase = (process.env.FLARESOLVERR_URL || "").trim().replace(/\/+$/u, "");
if (flareBase) {
const endpoint = flareBase.endsWith("/v1") ? flareBase : `${flareBase}/v1`;
const { data } = await axios.post(
endpoint,
{
cmd: "request.get",
url: pageUrl,
maxTimeout: 60_000,
},
{
timeout: 90_000,
headers: { "Content-Type": "application/json" },
validateStatus: (s) => s >= 200 && s < 300,
},
);
if (!data || data.status !== "ok" || !data.solution) {
const msg =
data && typeof data.message === "string"
? data.message
: "FlareSolverr request failed";
throw new Error(msg);
}
const status = Number(data.solution.status) || 0;
if (status < 200 || status >= 300) {
const err = new Error(`Request failed with status code ${String(status)}`);
/** @type {any} */ (err).response = { status };
throw err;
}
return String(data.solution.response || "");
}
const { data, status } = await axios.get(pageUrl, {
responseType: "text",
timeout: 30_000,
validateStatus: (s) => s >= 200 && s < 300,
...directConfig,
headers: {
...(directConfig.headers || {}),
},
});
if (status !== 200) {
throw new Error(`Unexpected HTTP status ${String(status)}`);
}
return typeof data === "string" ? data : String(data);
}
module.exports = {
fetchHtmlWithOptionalFlareSolverr,
formatUpstreamError,
};
+19
View File
@@ -0,0 +1,19 @@
const axios = require("axios");
const { parse } = require("node-html-parser");
/**
* Fetches HTML from a URL and returns the parsed document root.
* @async
* @param {string} url - Absolute URL to fetch.
* @returns {Promise<object>} Parsed HTML root (node-html-parser tree).
*/
async function fetchAndParseHtml(url) {
const { data } = await axios.get(url, {
responseType: "text",
validateStatus: (status) => status >= 200 && status < 300,
timeout: 15_000,
});
return parse(data);
}
module.exports = { fetchAndParseHtml };
+426
View File
@@ -0,0 +1,426 @@
const fs = require("fs/promises");
const path = require("path");
const axios = require("axios");
const { parse } = require("node-html-parser");
const { fetchHtmlWithOptionalFlareSolverr } = require("./flaresolverr.js");
const PAGE_URL =
"https://www.mangaread.org/manga/the-chaebeols-youngest-son/chapter-560/";
const OUTPUT_HTML = path.join(__dirname, "..", "data", "mangaread-chapter-560.html");
/** @type {import("axios").AxiosRequestConfig} */
const DEFAULT_GET_HTML = {
responseType: "text",
timeout: 30_000,
headers: {
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
},
validateStatus: (s) => s >= 200 && s < 300,
};
/**
* Madara / WPManga often serves desktop HTML as manga info (`manga-page`) and the reader
* (`reading-manga` + `img.wp-manga-chapter-img`) only when the client looks like mobile.
* @type {import("axios").AxiosRequestConfig}
*/
const DEFAULT_GET_HTML_READER_MOBILE = {
...DEFAULT_GET_HTML,
headers: {
...DEFAULT_GET_HTML.headers,
"User-Agent":
"Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 " +
"(KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",
},
};
/**
* @typedef {object} ChapterLink
* @property {string} url - Absolute or relative chapter URL from the anchor `href`.
* @property {string} title - Visible chapter label (trimmed).
*/
/**
* Returns every `a` inside `li.wp-manga-chapter` with URL and title.
* @param {string} html - Raw HTML document.
* @returns {ChapterLink[]} Extracted links in document order.
*/
function extractWpMangaChapters(html) {
const root = parse(html);
const items = root.querySelectorAll("li.wp-manga-chapter");
/** @type {ChapterLink[]} */
const chapters = [];
for (const li of items) {
const a = li.querySelector("a");
if (!a) {
continue;
}
const href = a.getAttribute("href");
const title = (a.textContent || "").trim().replace(/\s+/g, " ");
if (href) {
chapters.push({ url: href.trim(), title });
}
}
return chapters;
}
/**
* @param {string} html - Listing or chapter page HTML.
* @returns {string|null} Series title when found in markup.
*/
function extractWpMangaComicTitle(html) {
const root = parse(html);
const og = root.querySelector('meta[property="og:title"]');
if (og) {
const t = (og.getAttribute("content") || "").trim();
if (t.length > 0) {
return t.replace(/\s+/g, " ");
}
}
const h1 = root.querySelector(".profile-manga .post-title h1");
if (h1) {
const t = (h1.textContent || "").trim().replace(/\s+/g, " ");
if (t.length > 0) {
return t;
}
}
const titleEl = root.querySelector("title");
if (titleEl) {
const m = /^Read\s+(.+?)\s+-\s+manga/iu.exec((titleEl.textContent || "").trim());
if (m) {
return m[1].trim().replace(/\s+/g, " ");
}
}
return null;
}
/**
* @param {string} pageUrl - Manga or chapter URL.
* @returns {string} Slug segment from `/manga/{slug}/`, or `"comic"`.
*/
function mangaSlugFromUrl(pageUrl) {
try {
const { pathname } = new URL(pageUrl);
const parts = pathname.split("/").filter(Boolean);
const mi = parts.indexOf("manga");
if (mi !== -1 && parts[mi + 1] !== undefined) {
return parts[mi + 1];
}
} catch {
/* invalid URL */
}
return "comic";
}
/**
* Derives `https://host/manga/{slug}/` from a listing or chapter URL.
* @param {string} pageUrl - Any URL under a manga (chapter or series page).
* @returns {string} Normalized manga base URL with trailing slash.
*/
function mangaBaseUrlFromListingUrl(pageUrl) {
const u = new URL(pageUrl);
const parts = u.pathname.split("/").filter(Boolean);
const mi = parts.indexOf("manga");
if (mi === -1 || parts[mi + 1] === undefined) {
throw new Error(
'Could not find /manga/{slug}/ in URL; use a manga or chapter link from this site.',
);
}
const slug = parts[mi + 1];
return new URL(`/manga/${slug}/`, u.origin).href;
}
/**
* WPManga / Madara fills `#manga-chapters-holder` via POST to `ajax/chapters/` when
* the initial GET does not include chapter rows.
* @param {string} mangaBaseUrl - Output of {@link mangaBaseUrlFromListingUrl}.
* @param {string} refererUrl - Original page URL (for Referer).
* @returns {Promise<string>} HTML fragment containing `li.wp-manga-chapter` items.
*/
async function fetchMangaChapterListAjaxHtml(mangaBaseUrl, refererUrl) {
const base = mangaBaseUrl.endsWith("/") ? mangaBaseUrl : `${mangaBaseUrl}/`;
const ajaxUrl = new URL("ajax/chapters/", base).href;
const origin = new URL(refererUrl).origin;
const { data, status } = await axios.post(ajaxUrl, "", {
...DEFAULT_GET_HTML,
timeout: 45_000,
headers: {
...DEFAULT_GET_HTML.headers,
Accept: "*/*",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
Referer: refererUrl,
Origin: origin,
"X-Requested-With": "XMLHttpRequest",
},
validateStatus: (s) => s >= 200 && s < 300,
});
if (status !== 200) {
throw new Error(`ajax/chapters/ returned HTTP ${String(status)}`);
}
return typeof data === "string" ? data : String(data);
}
/**
* Fetches HTML for {@link pageUrl}.
* @param {string} pageUrl - URL to load.
* @param {object} [options] - Request shape.
* @param {"desktop"|"reader"} [options.variant] - `reader` requests mobile reader markup when the theme separates info vs reader.
* @returns {Promise<string>} Response body as text.
*/
async function fetchPageHtml(pageUrl, options = {}) {
/** @type {import("axios").AxiosRequestConfig} */
const cfg =
options.variant === "reader" ? DEFAULT_GET_HTML_READER_MOBILE : DEFAULT_GET_HTML;
return fetchHtmlWithOptionalFlareSolverr(pageUrl, cfg);
}
/**
* Loads a page and extracts each chapter anchor URL and title under `li.wp-manga-chapter`.
* If the first response has no such rows (common on series-only URLs), POSTs to `ajax/chapters/`.
* @param {string} pageUrl - Any page containing that markup (e.g. Mangaread chapter URL).
* @returns {Promise<{ chapters: ChapterLink[], comicTitle: string }>} Chapter rows and series title.
*/
async function fetchWpMangaChapterLinks(pageUrl) {
const html = await fetchPageHtml(pageUrl);
let chapters = extractWpMangaChapters(html);
if (chapters.length === 0) {
const base = mangaBaseUrlFromListingUrl(pageUrl);
const ajaxHtml = await fetchMangaChapterListAjaxHtml(base, pageUrl);
chapters = extractWpMangaChapters(ajaxHtml);
}
const comicTitle =
extractWpMangaComicTitle(html) || mangaSlugFromUrl(pageUrl);
return { chapters, comicTitle };
}
/**
* @typedef {object} SavedChapterImage
* @property {string} imageUrl - Absolute URL that was downloaded.
* @property {string} filePath - Absolute path of the saved file.
*/
/**
* @param {import("node-html-parser").HTMLElement} img - Image element.
* @returns {string|null} First usable image URL attribute, or null.
*/
function pickChapterImageSrc(img) {
/** @type {(string|null)[]} */
const attrs = [
img.getAttribute("data-src"),
img.getAttribute("data-lazy-src"),
img.getAttribute("data-original"),
img.getAttribute("data-full-url"),
img.getAttribute("src"),
];
for (const raw of attrs) {
if (raw && typeof raw === "string") {
const t = raw.trim();
if (t.length > 0 && !t.startsWith("data:")) {
return t;
}
}
}
return null;
}
/**
* @param {string} pageUrl - Chapter or site URL (for folder name).
* @returns {string} Filesystem-safe single path segment.
*/
function slugFromUrlForDir(pageUrl) {
try {
const { pathname } = new URL(pageUrl);
const parts = pathname.split("/").filter(Boolean);
const last = parts[parts.length - 1] || "chapter";
const safe = last.replace(/[^a-z0-9._-]+/gi, "_").slice(0, 96);
return safe.length > 0 ? safe : "chapter";
} catch {
return "chapter";
}
}
/**
* @param {string} contentType - `Content-Type` header value.
* @returns {string} File extension including dot, or empty string.
*/
function extensionFromContentType(contentType) {
if (!contentType || typeof contentType !== "string") {
return "";
}
const ct = contentType.split(";")[0].trim().toLowerCase();
/** @type {Record<string, string>} */
const map = {
"image/jpeg": ".jpg",
"image/jpg": ".jpg",
"image/pjpeg": ".jpg",
"image/png": ".png",
"image/webp": ".webp",
"image/gif": ".gif",
"image/avif": ".avif",
"image/svg+xml": ".svg",
};
return map[ct] || "";
}
/**
* @param {string} imageUrl - Absolute image URL.
* @returns {string} Lowercase extension with dot, or empty string.
*/
function extensionFromImageUrl(imageUrl) {
try {
const p = new URL(imageUrl).pathname;
const ext = path.extname(p).toLowerCase();
if (ext.length > 0 && ext.length <= 8 && /^\.[a-z0-9]+$/u.test(ext)) {
return ext;
}
} catch {
/* invalid URL */
}
return "";
}
/**
* Collects image URLs from `img.wp-manga-chapter-img` in document order.
* @param {string} html - Document HTML.
* @param {string} baseHref - URL used to resolve relative `src` values.
* @returns {string[]} Absolute image URLs (deduped sequential).
*/
function collectWpMangaChapterImageUrls(html, baseHref) {
const root = parse(html);
const imgs = root.querySelectorAll("img.wp-manga-chapter-img");
const baseUrl = new URL(baseHref);
/** @type {string[]} */
const urls = [];
for (const img of imgs) {
const raw = pickChapterImageSrc(img);
if (!raw) {
continue;
}
try {
urls.push(new URL(raw, baseUrl).href);
} catch {
/* skip bad href */
}
}
return urls;
}
/**
* Fetches a chapter page, selects `img.wp-manga-chapter-img`, downloads each image to disk.
* Uses a mobile UA first (Madara reader HTML), then falls back to desktop if no images appear.
*
* Pure client-side lazy sites may still return an empty list.
* @param {string} chapterPageUrl - Reader/chapter page URL.
* @param {string} [outDir] - Directory for saved files (default: `data/chapter-images/<slug>`).
* @returns {Promise<SavedChapterImage[]>} One record per image, in DOM order.
*/
async function downloadWpMangaChapterImages(chapterPageUrl, outDir) {
let urls = collectWpMangaChapterImageUrls(
await fetchPageHtml(chapterPageUrl, { variant: "reader" }),
chapterPageUrl,
);
if (urls.length === 0) {
urls = collectWpMangaChapterImageUrls(await fetchPageHtml(chapterPageUrl), chapterPageUrl);
}
const targetDir =
typeof outDir === "string" && outDir.trim().length > 0
? path.resolve(outDir.trim())
: path.join(__dirname, "..", "data", "chapter-images", slugFromUrlForDir(chapterPageUrl));
await fs.mkdir(targetDir, { recursive: true });
/** @type {import("axios").AxiosRequestConfig} */
const imageRequest = {
responseType: "arraybuffer",
timeout: 120_000,
maxContentLength: 50 * 1024 * 1024,
maxBodyLength: 50 * 1024 * 1024,
headers: {
...DEFAULT_GET_HTML.headers,
Accept: "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
Referer: chapterPageUrl,
},
validateStatus: (s) => s >= 200 && s < 300,
};
/** @type {SavedChapterImage[]} */
const saved = [];
let index = 0;
for (const imageUrl of urls) {
index += 1;
const { data, headers } = await axios.get(imageUrl, imageRequest);
const ct = headers["content-type"];
let ext = extensionFromImageUrl(imageUrl);
if (ext.length === 0) {
ext = extensionFromContentType(typeof ct === "string" ? ct : "") || ".bin";
}
const filename = `page-${String(index).padStart(4, "0")}${ext}`;
const filePath = path.join(targetDir, filename);
await fs.writeFile(filePath, Buffer.from(data));
saved.push({ imageUrl, filePath });
}
return saved;
}
/**
* Fetches the page, writes HTML to disk, parses chapter rows.
* @returns {Promise<void>}
*/
async function main() {
const html = await fetchPageHtml(PAGE_URL);
await fs.mkdir(path.dirname(OUTPUT_HTML), { recursive: true });
await fs.writeFile(OUTPUT_HTML, html, "utf8");
const chapters = extractWpMangaChapters(html);
console.log(`Saved HTML: ${OUTPUT_HTML}`);
console.log(`Found ${String(chapters.length)} li.wp-manga-chapter link(s):`);
console.log(JSON.stringify(chapters, null, 2));
}
module.exports = {
collectWpMangaChapterImageUrls,
downloadWpMangaChapterImages,
extractWpMangaChapters,
extractWpMangaComicTitle,
fetchMangaChapterListAjaxHtml,
fetchPageHtml,
fetchWpMangaChapterLinks,
mangaBaseUrlFromListingUrl,
mangaSlugFromUrl,
};
if (require.main === module) {
main().catch((err) => {
console.error(err);
process.exitCode = 1;
});
}
+180
View File
@@ -0,0 +1,180 @@
const { parse } = require("node-html-parser");
const { CHAPTER_PATHNAME_PATTERN, MANHWASUSU_HOSTS } = require("./constants.js");
const { fetchManhwaSusuHtml } = require("./io.js");
/**
* @typedef {object} ChapterLink
* @property {string} url - Absolute chapter URL.
* @property {string} title - Chapter label (trimmed).
*/
/**
* @param {string} pathname - URL pathname.
* @returns {boolean} Whether this path is a ManhwaSusu chapter reader URL.
*/
function isChapterPathname(pathname) {
const p = pathname.replace(/\/+$/u, "");
return CHAPTER_PATHNAME_PATTERN.test(p);
}
/**
* @param {string} url - Any http(s) URL.
* @returns {boolean} True when hostname matches known ManhwaSusu hosts (hint only).
*/
function isLikelyManhwaSusuSite(url) {
try {
const { hostname } = new URL(url);
return MANHWASUSU_HOSTS.some((h) => h === hostname.toLowerCase());
} catch {
return false;
}
}
/**
* Skip duplicate “First / Last chapter” buttons; the full list still contains the same targets.
* @param {import("node-html-parser").HTMLElement} anchor - `<a>`.
* @returns {boolean} True when the link text is only “First chapter” or “Last chapter”.
*/
function isFirstLastNavOnlyLink(anchor) {
const t = (anchor.textContent || "").trim().replace(/\s+/gu, " ");
return /^first chapter$/iu.test(t) || /^last chapter$/iu.test(t);
}
/**
* Parses chapter anchors from a series / listing HTML document.
* @param {string} html - Raw HTML.
* @param {string} pageUrl - Base URL for resolving relative `href`s.
* @returns {ChapterLink[]} Unique links in **document order** (site lists newest first).
*/
function extractChapterLinksFromListingHtml(html, pageUrl) {
const root = parse(html);
const base = new URL(pageUrl);
/** @type {Map<string, ChapterLink>} */
const byUrl = new Map();
for (const a of root.querySelectorAll("a[href]")) {
const href = (a.getAttribute("href") || "").trim();
if (href.length === 0 || href.startsWith("#")) {
continue;
}
let abs;
try {
abs = new URL(href, base).href;
} catch {
continue;
}
const { pathname } = new URL(abs);
if (!isChapterPathname(pathname)) {
continue;
}
if (isFirstLastNavOnlyLink(a)) {
continue;
}
if (byUrl.has(abs)) {
continue;
}
let title = "";
const pEl = a.querySelector("p");
if (pEl) {
title = (pEl.textContent || "").trim().replace(/\s+/gu, " ");
}
if (title.length === 0) {
title = (a.textContent || "").trim().replace(/\s+/gu, " ");
}
title = title
.replace(/\s+\d+\s*(min|hour|day|week|mth|month|year)s?\s+ago$/iu, "")
.trim();
const fallback = pathname.split("/").filter(Boolean).pop() || "chapter";
byUrl.set(abs, { url: abs, title: title.length > 0 ? title : fallback });
}
return [...byUrl.values()];
}
/**
* @param {string} html - Series listing HTML.
* @returns {string|null} Comic title when found in markup.
*/
function extractComicTitleFromListingHtml(html) {
const root = parse(html);
const stripSiteSuffix = (raw) =>
raw.replace(/\s*-\s*ManhwaSusu\s*$/iu, "").trim().replace(/\s+/gu, " ");
const og = root.querySelector('meta[property="og:title"]');
if (og) {
const t = stripSiteSuffix(og.getAttribute("content") || "");
if (t.length > 0) {
return t;
}
}
const titleEl = root.querySelector("title");
if (titleEl) {
const t = stripSiteSuffix(titleEl.textContent || "");
if (t.length > 0) {
return t;
}
}
for (const h1 of root.querySelectorAll("h1")) {
const t = (h1.textContent || "").trim().replace(/\s+/gu, " ");
if (
t.length > 0 &&
!/^ManhwaSusu$/iu.test(t) &&
!/chapter\s*list/iu.test(t)
) {
return t;
}
}
return null;
}
/**
* @param {string} pageUrl - Series or chapter URL under `/read/{slug}/`.
* @returns {string} Series slug segment, or `"comic"`.
*/
function seriesSlugFromListingUrl(pageUrl) {
try {
const { pathname } = new URL(pageUrl);
const parts = pathname.split("/").filter(Boolean);
const readIdx = parts.indexOf("read");
if (readIdx !== -1 && parts[readIdx + 1] !== undefined) {
return parts[readIdx + 1];
}
} catch {
/* invalid URL */
}
return "comic";
}
/**
* @param {string} listingPageUrl - e.g. `https://manhwasusu.com/read/{slug}/`
* @returns {Promise<{ chapters: ChapterLink[], comicTitle: string }>} Chapter rows and series title.
*/
async function fetchChapterLinks(listingPageUrl) {
const html = await fetchManhwaSusuHtml(listingPageUrl);
const chapters = extractChapterLinksFromListingHtml(html, listingPageUrl);
const comicTitle =
extractComicTitleFromListingHtml(html) || seriesSlugFromListingUrl(listingPageUrl);
return { chapters, comicTitle };
}
module.exports = {
extractChapterLinksFromListingHtml,
extractComicTitleFromListingHtml,
fetchChapterLinks,
isChapterPathname,
isFirstLastNavOnlyLink,
isLikelyManhwaSusuSite,
seriesSlugFromListingUrl,
};
+55
View File
@@ -0,0 +1,55 @@
/**
* ManhwaSusu-specific URLs and DOM selectors. Adjust here if routes or markup change.
* @see https://manhwasusu.com/read/{slug}/ — series + chapter index
*/
/** Hostnames that this adapter is intended for (for friendly checks only). */
const MANHWASUSU_HOSTS = /** @type {const} */ (["manhwasusu.com", "www.manhwasusu.com"]);
/** Canonical site origin used as Referer for CDN panel downloads. */
const MANHWASUSU_ORIGIN = "https://manhwasusu.com/";
/** Default sample listing used by the fetch CLI when no URL is overridden. */
const DEFAULT_LISTING_URL = "https://manhwasusu.com/read/switch-on-season-2/";
/** Path shape: `/read/{seriesSlug}/chapter-{id}/` (SSR uses trailing slash optional). */
const CHAPTER_PATHNAME_PATTERN = /^\/read\/[^/]+\/chapter-[^/]+$/iu;
/**
* Reader lazily fills `src` from `data-src`; class `imgku` wraps panel images on chapter pages.
* @type {readonly [string, ...string[]]}
*/
const READER_IMG_SELECTORS = ["img.imgku"];
/**
* Browser-like headers for CDN image GETs (captured from manhwasusu.com reader → manhwature.com).
* Referer must be the site origin; chapter URLs are often rejected by the CDN.
* @type {Readonly<Record<string, string>>}
*/
const CDN_IMAGE_HEADERS = {
Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9,id;q=0.8",
"Cache-Control": "no-cache",
Pragma: "no-cache",
"Sec-CH-UA":
'"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"',
"Sec-CH-UA-Mobile": "?0",
"Sec-CH-UA-Platform": '"Windows"',
"Sec-Fetch-Dest": "image",
"Sec-Fetch-Mode": "no-cors",
"Sec-Fetch-Site": "cross-site",
"Sec-Fetch-Storage-Access": "active",
Referer: MANHWASUSU_ORIGIN,
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
"(KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0",
};
module.exports = {
CDN_IMAGE_HEADERS,
CHAPTER_PATHNAME_PATTERN,
DEFAULT_LISTING_URL,
MANHWASUSU_HOSTS,
MANHWASUSU_ORIGIN,
READER_IMG_SELECTORS,
};
+169
View File
@@ -0,0 +1,169 @@
const fs = require("fs/promises");
const path = require("path");
const os = require("os");
const AdmZip = require("adm-zip");
const { downloadChapterImages } = require("./images.js");
const { extractComicTitleFromListingHtml, seriesSlugFromListingUrl } = require("./chapters.js");
const { fetchManhwaSusuHtml } = require("./io.js");
/**
* `/read/my-series/chapter-12/` → `my-series-chapter-12`.
* @param {string} chapterPageUrl - Full chapter URL.
* @returns {string} Suggested `.cbz` stem.
*/
function stemForManhwaSusuChapterCbz(chapterPageUrl) {
try {
const { pathname } = new URL(chapterPageUrl);
const segs = pathname.replace(/\/+$/u, "").split("/").filter(Boolean);
const readIdx = segs.indexOf("read");
if (readIdx !== -1 && segs.length >= readIdx + 3) {
const slug = segs[readIdx + 1];
const chapSeg = segs[readIdx + 2];
return `${slug}-${chapSeg}`;
}
if (segs.length >= 2) {
return segs.slice(-2).join("-");
}
return segs.at(-1) || "chapter";
} catch {
return "chapter";
}
}
/**
* @param {string} stem - File base segment.
* @returns {string} Sanitized stem.
*/
function sanitizeFsStem(stem) {
const cleaned = stem.replace(/[^a-z0-9._-]+/giu, "_").replace(/^_+|_+$/gu, "");
return cleaned.length > 0 ? cleaned.slice(0, 180) : "chapter";
}
/**
* @param {string} name - Human-readable comic title.
* @returns {string} Filesystem-safe directory name.
*/
function sanitizeComicDirName(name) {
const collapsed = name.trim().replace(/\s+/gu, " ");
const cleaned = [...collapsed]
.filter((ch) => {
const code = ch.charCodeAt(0);
if (code < 32 || code === 127) {
return false;
}
return !/[<>:"/\\|?*]/.test(ch);
})
.join("")
.replace(/\.+$/g, "")
.trim();
return cleaned.length > 0 ? cleaned.slice(0, 180) : "comic";
}
/**
* @param {string|undefined} cliOut - Positional output path/dir.
* @param {string} defaultStem - When `cliOut` is omitted or directory-like.
* @param {string} [comicTitle] - When set, default output goes under this subdirectory.
* @returns {string} Absolute `.cbz` path.
*/
function resolveCliOutputPath(cliOut, defaultStem, comicTitle) {
const stem = sanitizeFsStem(defaultStem);
const baseDir =
typeof comicTitle === "string" && comicTitle.trim().length > 0
? path.join(
process.cwd(),
"data",
"manhwasusu",
"comics",
sanitizeComicDirName(comicTitle),
)
: path.join(process.cwd(), "data", "manhwasusu", "comics");
if (cliOut === undefined || cliOut.trim() === "") {
return path.join(baseDir, `${stem}.cbz`);
}
const trimmed = cliOut.trim();
const abs = path.resolve(trimmed);
if (abs.toLowerCase().endsWith(".cbz")) {
return abs;
}
return path.join(abs, `${stem}.cbz`);
}
/**
* Downloads ManhwaSusu chapter panels to temp storage, packs `.cbz`, removes temp folder.
* @param {string} chapterUrl - Chapter reader URL.
* @param {string} outputCbzPath - Destination `.cbz`.
* @returns {Promise<{ imageCount: number, outputPath: string }>} Page count and written archive path.
*/
async function chapterUrlToCbz(chapterUrl, outputCbzPath) {
const absOut = outputCbzPath.toLowerCase().endsWith(".cbz")
? path.resolve(outputCbzPath)
: `${path.resolve(outputCbzPath)}.cbz`;
const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "cxyz-manhwasusu-chapter-"));
try {
const saved = await downloadChapterImages(chapterUrl, tmpRoot);
if (saved.length === 0) {
throw new Error("No chapter images were found for this ManhwaSusu URL.");
}
await fs.mkdir(path.dirname(absOut), { recursive: true });
const zip = new AdmZip();
for (const { filePath } of saved) {
zip.addLocalFile(filePath, "", path.basename(filePath));
}
zip.writeZip(absOut);
return { imageCount: saved.length, outputPath: absOut };
} finally {
await fs.rm(tmpRoot, { recursive: true, force: true }).catch(() => {});
}
}
/**
* CLI: `node src/manhwasusu/downloadChapterCbz.js <chapterUrl> [.cbz|dir]`
* @returns {Promise<void>}
*/
async function main() {
const chapterUrl = process.argv[2];
if (!chapterUrl || chapterUrl.startsWith("-")) {
console.error(
"Usage: node src/manhwasusu/downloadChapterCbz.js <chapterUrl> [output.cbz|outputDirectory]",
);
process.exitCode = 1;
return;
}
const stem = stemForManhwaSusuChapterCbz(chapterUrl);
let comicTitle = seriesSlugFromListingUrl(chapterUrl);
try {
const html = await fetchManhwaSusuHtml(chapterUrl);
comicTitle = extractComicTitleFromListingHtml(html) || comicTitle;
} catch {
/* keep slug fallback */
}
const target = resolveCliOutputPath(process.argv[3], stem, comicTitle);
try {
const { imageCount, outputPath } = await chapterUrlToCbz(chapterUrl, target);
console.log(`Wrote ${String(imageCount)} page(s) → ${outputPath}`);
} catch (err) {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
}
}
module.exports = {
chapterUrlToCbz,
resolveCliOutputPath,
sanitizeComicDirName,
sanitizeFsStem,
stemForManhwaSusuChapterCbz,
};
if (require.main === module) {
main();
}
+426
View File
@@ -0,0 +1,426 @@
const path = require("path");
const { fetchChapterLinks, isLikelyManhwaSusuSite } = require("./chapters.js");
const { stemForManhwaSusuChapterCbz, chapterUrlToCbz, sanitizeComicDirName } =
require("./downloadChapterCbz.js");
/**
* @typedef {object} ParsedCli
* @property {boolean} [help] - Print usage and exit when true.
* @property {string|undefined} listingUrl - Series listing URL (required unless help).
* @property {boolean} listOnly - Only print the chapter menu.
* @property {boolean} allChapters - Queue every chapter after reversal to reading order.
* @property {string|undefined} selectSpec - `--select` string (1-based menu indices).
* @property {string|undefined} numbersSpec - `--numbers` string (chapter numbers from titles).
* @property {string} outDir - Output directory for `.cbz` files.
*/
/**
* @param {string[]} argv - `process.argv`.
* @returns {ParsedCli} Parsed flags and positional URL.
*/
function parseCli(argv) {
/** @type {ParsedCli} */
const result = {
help: false,
listingUrl: undefined,
listOnly: false,
allChapters: false,
selectSpec: undefined,
numbersSpec: undefined,
outDir: path.join(process.cwd(), "data", "manhwasusu", "comics"),
};
for (let i = 2; i < argv.length; i += 1) {
const a = argv[i];
if (a === "--help" || a === "-h") {
result.help = true;
} else if (a === "--list-only" || a === "-l") {
result.listOnly = true;
} else if (a === "--all") {
result.allChapters = true;
} else if (a === "--select" || a === "-s") {
const next = argv[i + 1];
if (next === undefined || next.startsWith("-")) {
throw new Error("Missing value for --select");
}
result.selectSpec = next;
i += 1;
} else if (a === "--numbers" || a === "-n") {
const next = argv[i + 1];
if (next === undefined || next.startsWith("-")) {
throw new Error("Missing value for --numbers");
}
result.numbersSpec = next;
i += 1;
} else if (a === "--out" || a === "-o") {
const next = argv[i + 1];
if (next === undefined || next.startsWith("-")) {
throw new Error("Missing value for --out");
}
result.outDir = path.resolve(next);
i += 1;
} else if (!a.startsWith("-")) {
if (result.listingUrl) {
throw new Error(`Unexpected extra argument: ${a}`);
}
result.listingUrl = a;
} else {
throw new Error(`Unknown option: ${a}`);
}
}
return result;
}
/**
* Listing HTML is newest-first; reverse so `--select` line 1 is Chapter 1.
* @param {import("./chapters.js").ChapterLink[]} chapters - Site order.
* @returns {import("./chapters.js").ChapterLink[]} New array, oldest chapter first.
*/
function chaptersInReadingOrder(chapters) {
return [...chapters].reverse();
}
/**
* @param {string} spec - `--select` value.
* @param {number} count - Chapter count.
* @returns {number[]} Sorted unique 1-based row indices into the reversed list.
*/
function parseChapterSelection(spec, count) {
if (count <= 0) {
throw new Error("Chapter list is empty; cannot select indices.");
}
const trimmed = spec.trim();
if (trimmed.length === 0) {
throw new Error("--select value is empty.");
}
/** @type {Set<number>} */
const seen = new Set();
for (const rawPart of trimmed.split(",")) {
const part = rawPart.trim();
if (part.length === 0) {
continue;
}
if (/^\d+$/u.test(part)) {
const n = Number.parseInt(part, 10);
if (n < 1 || n > count) {
throw new Error(
`Index ${String(n)} is out of range (valid: 1${String(count)}).`,
);
}
seen.add(n);
continue;
}
const rangeMatch = /^(\d+)\s*-\s*(\d+)$/u.exec(part);
if (rangeMatch) {
const lo = Number.parseInt(rangeMatch[1], 10);
const hi = Number.parseInt(rangeMatch[2], 10);
const a = Math.min(lo, hi);
const b = Math.max(lo, hi);
for (let j = a; j <= b; j += 1) {
if (j < 1 || j > count) {
throw new Error(
`Range ${part} includes invalid index ${String(j)} (valid: 1${String(count)}).`,
);
}
seen.add(j);
}
continue;
}
throw new Error(`Invalid --select segment: "${part}"`);
}
if (seen.size === 0) {
throw new Error("No indices parsed from --select.");
}
return [...seen].sort((x, y) => x - y);
}
/**
* @param {string} title - e.g. `Chapter 22`.
* @returns {string|null} Chapter number string from title, or null.
*/
function chapterNumberFromListingTitle(title) {
const m = /^Chapter\s+([\d.]+)\s*$/iu.exec(title.trim().replace(/\s+/gu, " "));
return m ? m[1] : null;
}
/**
* @param {string} spec - `--numbers` argument.
* @returns {string[]} Ordered unique chapter number strings as entered.
*/
function parseChapterNumbersSpec(spec) {
const trimmed = spec.trim();
if (trimmed.length === 0) {
throw new Error("--numbers value is empty.");
}
/** @type {string[]} */
const ordered = [];
/** @type {Set<string>} */
const seen = new Set();
for (const rawPart of trimmed.split(",")) {
const part = rawPart.trim();
if (part.length === 0) {
continue;
}
const rangeMatch = /^(\d+)\s*-\s*(\d+)$/u.exec(part);
if (rangeMatch) {
const lo = Number.parseInt(rangeMatch[1], 10);
const hi = Number.parseInt(rangeMatch[2], 10);
const a = Math.min(lo, hi);
const b = Math.max(lo, hi);
for (let n = a; n <= b; n += 1) {
const k = String(n);
if (!seen.has(k)) {
seen.add(k);
ordered.push(k);
}
}
continue;
}
if (/^\d+(\.\d+)?$/u.test(part)) {
if (!seen.has(part)) {
seen.add(part);
ordered.push(part);
}
continue;
}
throw new Error(`Invalid --numbers segment: "${part}"`);
}
if (ordered.length === 0) {
throw new Error("No chapter numbers parsed from --numbers.");
}
return ordered;
}
/**
* @param {string[]} orderedNums - Chapter numbers requested.
* @param {import("./chapters.js").ChapterLink[]} chapters - Ordered oldest→newest after reverse.
* @returns {Array<{ url: string, title: string, pickLabel: string, chapterNumber: string }>}
* Rows to download.
*/
function queueItemsByChapterNumbers(orderedNums, chapters) {
/** @type {Map<string, import("./chapters.js").ChapterLink>} */
const first = new Map();
for (const ch of chapters) {
const num = chapterNumberFromListingTitle(ch.title);
if (num !== null && !first.has(num)) {
first.set(num, ch);
}
}
return orderedNums.map((num) => {
const ch = first.get(num);
if (!ch) {
throw new Error(
`No chapter titled "Chapter ${num}" in the ManhwaSusu listing. ` +
"Use --list-only and check titles.",
);
}
return {
url: ch.url,
title: ch.title,
pickLabel: `Ch.${num}`,
chapterNumber: num,
};
});
}
/**
* @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first list.
* @returns {void}
*/
function printChapterMenu(chapters) {
const digits = String(chapters.length).length;
for (let i = 0; i < chapters.length; i += 1) {
const n = i + 1;
const pad = String(n).padStart(digits, " ");
const title = chapters[i].title.replace(/\s+/gu, " ").trim();
console.log(`${pad}. ${title}`);
console.log(` ${chapters[i].url}`);
}
}
/**
* @param {number[]} indices1 - Sorted 1-based indices.
* @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first.
*/
/**
* @param {number[]} indices1 - Sorted 1-based indices.
* @param {import("./chapters.js").ChapterLink[]} chapters - Oldest-first chapter list.
* @returns {Array<{ index: number, title: string, url: string, pickLabel: string }>} Queue rows.
*/
function chaptersByIndices(indices1, chapters) {
return indices1.map((idx) => ({
index: idx,
title: chapters[idx - 1].title,
url: chapters[idx - 1].url,
pickLabel: `#${String(idx)}`,
}));
}
/**
* CLI entry: fetch listing, optionally build `.cbz` archives.
* @returns {Promise<void>}
*/
async function main() {
let parsed;
try {
parsed = parseCli(process.argv);
} catch (err) {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
return;
}
if (parsed.help) {
console.log(`ManhwaSusu chapter queue (dedicated adapter; URLs like /read/{slug}/):
Usage:
node src/manhwasusu/downloadChaptersQueue.js <listing-url> [--list-only|-l]
node src/manhwasusu/downloadChaptersQueue.js <listing-url> --numbers|-n <spec> [--out|-o <dir>]
node src/manhwasusu/downloadChaptersQueue.js <listing-url> --select|-s <spec> [--out|-o <dir>]
node src/manhwasusu/downloadChaptersQueue.js <listing-url> --all [--out|-o <dir>]
Listing URL: series page ending in /read/{slug}/ (or chapter URL — site embeds chapter list markup).
Default --out: data/manhwasusu/comics/<comic title>/
--numbers: titles like "Chapter 22", e.g. 22,23,24 or 22-24
--select: 1-based rows on printed list after reversal (line 1 = Chapter 1)
`);
return;
}
if (!parsed.listingUrl) {
console.error(
"Usage: node src/manhwasusu/downloadChaptersQueue.js <listing-url> [options]",
);
process.exitCode = 1;
return;
}
if (!isLikelyManhwaSusuSite(parsed.listingUrl)) {
console.warn(
"Warning: URL hostname does not look like ManhwaSusu; parsers may fail if structure differs.",
);
}
let chapters;
let comicTitle;
try {
({ chapters, comicTitle } = await fetchChapterLinks(parsed.listingUrl));
chapters = chaptersInReadingOrder(chapters);
} catch (err) {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
return;
}
console.log(`Found ${String(chapters.length)} ManhwaSusu chapter link(s).`);
printChapterMenu(chapters);
if (parsed.listOnly) {
return;
}
const wantsAll = parsed.allChapters;
const wantsSelect = Boolean(parsed.selectSpec);
const wantsNumbers = Boolean(parsed.numbersSpec);
const pickModes = [wantsAll, wantsSelect, wantsNumbers].filter(Boolean).length;
if (pickModes === 0) {
console.error("Choose --numbers, --select, or --all (or --list-only).");
process.exitCode = 1;
return;
}
if (pickModes > 1) {
console.error("Use only one of --all, --select, or --numbers.");
process.exitCode = 1;
return;
}
/** @type {Array<{ url: string, title: string, pickLabel: string }>} */
let queue;
try {
if (wantsAll) {
const indices = Array.from({ length: chapters.length }, (_, i) => i + 1);
queue = chaptersByIndices(indices, chapters);
} else if (wantsSelect) {
const indices = parseChapterSelection(
/** @type {string} */ (parsed.selectSpec),
chapters.length,
);
queue = chaptersByIndices(indices, chapters);
} else {
const orderedNums = parseChapterNumbersSpec(
/** @type {string} */ (parsed.numbersSpec),
);
queue = queueItemsByChapterNumbers(orderedNums, chapters);
}
} catch (err) {
console.error(err instanceof Error ? err.message : err);
process.exitCode = 1;
return;
}
const outDir = path.join(parsed.outDir, sanitizeComicDirName(comicTitle));
console.log(`Comic: ${comicTitle}`);
console.log(`Output directory: ${outDir}`);
let failedCount = 0;
let done = 0;
for (const item of queue) {
done += 1;
const stem = stemForManhwaSusuChapterCbz(item.url);
const targetPath = path.join(outDir, `${stem}.cbz`);
console.log(`\n[${String(done)}/${String(queue.length)}] ${item.pickLabel} ${item.title}`);
console.log(`${item.url}`);
try {
const { imageCount, outputPath } = await chapterUrlToCbz(item.url, targetPath);
console.log(`${String(imageCount)} page(s) → ${outputPath}`);
} catch (err) {
console.error(`${err instanceof Error ? err.message : err}`);
failedCount += 1;
}
}
if (failedCount > 0) {
console.error(
`\n${String(failedCount)} chapter(s) failed (${String(queue.length - failedCount)} succeeded).`,
);
process.exitCode = 1;
}
}
module.exports = {
chapterNumberFromListingTitle,
chaptersByIndices,
chaptersInReadingOrder,
parseChapterNumbersSpec,
parseChapterSelection,
parseCli,
printChapterMenu,
queueItemsByChapterNumbers,
};
if (require.main === module) {
main().catch((err) => {
console.error(err);
process.exitCode = 1;
});
}
+45
View File
@@ -0,0 +1,45 @@
const fs = require("fs/promises");
const path = require("path");
const { DEFAULT_LISTING_URL } = require("./constants.js");
const { extractChapterLinksFromListingHtml } = require("./chapters.js");
const { fetchManhwaSusuHtml } = require("./io.js");
const OUTPUT_HTML = path.join(
__dirname,
"..",
"..",
"data",
"manhwasusu",
"debug-listing.html",
);
/**
* CLI: snapshot listing HTML + print chapter JSON. Optional `MANHWASUSU_LISTING_URL` env overrides default.
* @returns {Promise<void>}
*/
async function main() {
const pageUrl = process.env.MANHWASUSU_LISTING_URL || DEFAULT_LISTING_URL;
const html = await fetchManhwaSusuHtml(pageUrl);
await fs.mkdir(path.dirname(OUTPUT_HTML), { recursive: true });
await fs.writeFile(OUTPUT_HTML, html, "utf8");
const chapters = extractChapterLinksFromListingHtml(html, pageUrl);
console.log(`Source: ${pageUrl}`);
console.log(`Saved HTML: ${OUTPUT_HTML}`);
console.log(
`Found ${String(chapters.length)} chapter link(s) (newest-first on page; reverse for reading order):`,
);
console.log(JSON.stringify(chapters, null, 2));
}
if (require.main === module) {
main().catch((err) => {
console.error(err);
process.exitCode = 1;
});
}
module.exports = { main };
+100
View File
@@ -0,0 +1,100 @@
const fs = require("fs/promises");
const path = require("path");
const axios = require("axios");
const { parse } = require("node-html-parser");
const { READER_IMG_SELECTORS } = require("./constants.js");
const {
extensionFromContentType,
extensionFromImageUrl,
fetchManhwaSusuHtml,
imageGetConfig,
pickReaderImgSrc,
slugFromChapterUrl,
} = require("./io.js");
/**
* Collects panel URLs from chapter HTML (lazy `data-src` on reader images).
* @param {string} html - Chapter document.
* @param {string} baseHref - Used to resolve relative attributes.
* @returns {string[]} Absolute image URLs in DOM order.
*/
function collectChapterImageUrls(html, baseHref) {
const root = parse(html);
const baseUrl = new URL(baseHref);
const selector = READER_IMG_SELECTORS.join(", ");
/** @type {string[]} */
const urls = [];
for (const img of root.querySelectorAll(selector)) {
const raw = pickReaderImgSrc(img);
if (!raw) {
continue;
}
try {
urls.push(new URL(raw, baseUrl).href);
} catch {
/* skip bad href */
}
}
return urls;
}
/**
* @typedef {object} SavedChapterImage
* @property {string} imageUrl - Remote URL that was downloaded.
* @property {string} filePath - Path of the saved file.
*/
/**
* Downloads reader panels into `data/manhwasusu/chapters/{slug}/` by default.
* @param {string} chapterPageUrl - Full `/read/{slug}/chapter-N/` URL.
* @param {string} [outDir] - Override output directory.
* @returns {Promise<SavedChapterImage[]>} One record per saved panel file.
*/
async function downloadChapterImages(chapterPageUrl, outDir) {
const html = await fetchManhwaSusuHtml(chapterPageUrl);
const urls = collectChapterImageUrls(html, chapterPageUrl);
const targetDir =
typeof outDir === "string" && outDir.trim().length > 0
? path.resolve(outDir.trim())
: path.join(
path.join(__dirname, "..", ".."),
"data",
"manhwasusu",
"chapters",
slugFromChapterUrl(chapterPageUrl),
);
await fs.mkdir(targetDir, { recursive: true });
const cfg = imageGetConfig(chapterPageUrl);
/** @type {SavedChapterImage[]} */
const saved = [];
let index = 0;
for (const imageUrl of urls) {
index += 1;
const { data, headers } = await axios.get(imageUrl, cfg);
const ct = headers["content-type"];
let ext = extensionFromImageUrl(imageUrl);
if (ext.length === 0) {
ext = extensionFromContentType(typeof ct === "string" ? ct : "") || ".bin";
}
const filename = `page-${String(index).padStart(4, "0")}${ext}`;
const filePath = path.join(targetDir, filename);
await fs.writeFile(filePath, Buffer.from(data));
saved.push({ imageUrl, filePath });
}
return saved;
}
module.exports = {
collectChapterImageUrls,
downloadChapterImages,
};
+88
View File
@@ -0,0 +1,88 @@
/**
* ManhwaSusu adapter: HTTP, listing/chapter parsing, image download, and CBZ CLIs live under this folder.
* Tweak {@link ./constants.js} when routes or reader markup change.
*/
const {
CDN_IMAGE_HEADERS,
CHAPTER_PATHNAME_PATTERN,
DEFAULT_LISTING_URL,
MANHWASUSU_HOSTS,
MANHWASUSU_ORIGIN,
READER_IMG_SELECTORS,
} = require("./constants.js");
const {
extractChapterLinksFromListingHtml,
extractComicTitleFromListingHtml,
fetchChapterLinks,
isChapterPathname,
isFirstLastNavOnlyLink,
isLikelyManhwaSusuSite,
seriesSlugFromListingUrl,
} = require("./chapters.js");
const {
collectChapterImageUrls,
downloadChapterImages,
} = require("./images.js");
const {
DEFAULT_GET_HTML,
extensionFromContentType,
extensionFromImageUrl,
fetchManhwaSusuHtml,
imageGetConfig,
pickReaderImgSrc,
slugFromChapterUrl,
} = require("./io.js");
const {
chapterUrlToCbz,
resolveCliOutputPath,
sanitizeComicDirName,
sanitizeFsStem,
stemForManhwaSusuChapterCbz,
} = require("./downloadChapterCbz.js");
module.exports = {
CDN_IMAGE_HEADERS,
CHAPTER_PATHNAME_PATTERN,
DEFAULT_GET_HTML,
DEFAULT_LISTING_URL,
MANHWASUSU_HOSTS,
MANHWASUSU_ORIGIN,
READER_IMG_SELECTORS,
chapterUrlToCbz,
collectChapterImageUrls,
/** @deprecated Use {@link collectChapterImageUrls} */
collectManhwaSusuChapterImageUrls: collectChapterImageUrls,
downloadChapterImages,
/** @deprecated Use {@link downloadChapterImages} */
downloadManhwaSusuChapterImages: downloadChapterImages,
extensionFromContentType,
extensionFromImageUrl,
extractChapterLinksFromListingHtml,
extractComicTitleFromListingHtml,
/** @deprecated Use {@link extractChapterLinksFromListingHtml} */
extractManhwaSusuChapters: extractChapterLinksFromListingHtml,
fetchChapterLinks,
fetchManhwaSusuHtml,
/** @deprecated Use {@link fetchChapterLinks} */
fetchManhwaSusuChapterLinks: fetchChapterLinks,
imageGetConfig,
isChapterPathname,
isFirstLastNavOnlyLink,
/** @deprecated Use {@link isFirstLastNavOnlyLink} */
isFirstLastChapterNavLink: isFirstLastNavOnlyLink,
isLikelyManhwaSusuSite,
/** @deprecated Use {@link isChapterPathname} */
isManhwaSusuChapterPath: isChapterPathname,
pickReaderImgSrc,
resolveCliOutputPath,
sanitizeComicDirName,
sanitizeFsStem,
seriesSlugFromListingUrl,
slugFromChapterUrl,
stemForManhwaSusuChapterCbz,
};
+141
View File
@@ -0,0 +1,141 @@
const path = require("path");
const axios = require("axios");
const { fetchHtmlWithOptionalFlareSolverr } = require("../flaresolverr.js");
const { CDN_IMAGE_HEADERS, MANHWASUSU_ORIGIN } = require("./constants.js");
/** @type {import("axios").AxiosRequestConfig} */
const DEFAULT_GET_HTML = {
responseType: "text",
timeout: 30_000,
headers: {
Accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9,id;q=0.8",
Referer: MANHWASUSU_ORIGIN,
"User-Agent": CDN_IMAGE_HEADERS["User-Agent"],
},
validateStatus: (s) => s >= 200 && s < 300,
};
/**
* @param {string} pageUrl - URL to load.
* @returns {Promise<string>} Response body text.
*/
async function fetchManhwaSusuHtml(pageUrl) {
return fetchHtmlWithOptionalFlareSolverr(pageUrl, DEFAULT_GET_HTML);
}
/**
* Chooses a usable Lazy-load / CDN attribute from a reader `<img>` (same order as many WPstyle lazy themes).
* @param {import("node-html-parser").HTMLElement} img - Image element.
* @returns {string|null} First non-placeholder image URL, or null.
*/
function pickReaderImgSrc(img) {
/** @type {(string|null)[]} */
const attrs = [
img.getAttribute("data-src"),
img.getAttribute("data-lazy-src"),
img.getAttribute("data-original"),
img.getAttribute("data-full-url"),
img.getAttribute("src"),
];
for (const raw of attrs) {
if (raw && typeof raw === "string") {
const t = raw.trim();
if (
t.length > 0 &&
!t.startsWith("data:") &&
t !== "/readerarea.svg" &&
!t.endsWith("readerarea.svg")
) {
return t;
}
}
}
return null;
}
/**
* @param {string} contentType - `Content-Type` header value.
* @returns {string} File extension including dot, or empty string.
*/
function extensionFromContentType(contentType) {
if (!contentType || typeof contentType !== "string") {
return "";
}
const ct = contentType.split(";")[0].trim().toLowerCase();
/** @type {Record<string, string>} */
const map = {
"image/jpeg": ".jpg",
"image/jpg": ".jpg",
"image/pjpeg": ".jpg",
"image/png": ".png",
"image/webp": ".webp",
"image/gif": ".gif",
"image/avif": ".avif",
"image/svg+xml": ".svg",
};
return map[ct] || "";
}
/**
* @param {string} imageUrl - Absolute image URL.
* @returns {string} Lowercase extension with dot, or empty string.
*/
function extensionFromImageUrl(imageUrl) {
try {
const p = new URL(imageUrl).pathname;
const ext = path.extname(p).toLowerCase();
if (ext.length > 0 && ext.length <= 8 && /^\.[a-z0-9]+$/u.test(ext)) {
return ext;
}
} catch {
/* invalid URL */
}
return "";
}
/**
* @param {string} pageUrl - Chapter URL (directory name fragment).
* @returns {string} Single filesystem-safe segment.
*/
function slugFromChapterUrl(pageUrl) {
try {
const { pathname } = new URL(pageUrl);
const parts = pathname.split("/").filter(Boolean);
const last = parts[parts.length - 1] || "chapter";
const safe = last.replace(/[^a-z0-9._-]+/giu, "_").slice(0, 96);
return safe.length > 0 ? safe : "chapter";
} catch {
return "chapter";
}
}
/**
* Axios config for CDN panel bytes (browser-like headers from manhwasusu reader).
* @param {string} [_referrerChapterUrl] - Kept for call-site compatibility; CDN expects site-origin Referer.
* @returns {import("axios").AxiosRequestConfig} GET config for binary image responses.
*/
function imageGetConfig(_referrerChapterUrl) {
return {
responseType: "arraybuffer",
timeout: 120_000,
maxContentLength: 50 * 1024 * 1024,
maxBodyLength: 50 * 1024 * 1024,
headers: { ...CDN_IMAGE_HEADERS },
validateStatus: (s) => s >= 200 && s < 300,
};
}
module.exports = {
DEFAULT_GET_HTML,
extensionFromContentType,
extensionFromImageUrl,
fetchManhwaSusuHtml,
imageGetConfig,
pickReaderImgSrc,
slugFromChapterUrl,
};
+5
View File
@@ -0,0 +1,5 @@
/**
* Re-export for older imports. Prefer `require("./manhwasusu")` or `require("./manhwasusu/index.js")`.
* @see {@link ./manhwasusu/index.js}
*/
module.exports = require("./manhwasusu/index.js");
+473
View File
@@ -0,0 +1,473 @@
const fs = require("fs");
const path = require("path");
const Database = require("better-sqlite3");
/**
* @param {string} dataDir - Absolute data directory (SQLite + CBZ roots live under this).
* @returns {import("better-sqlite3").Database} Opened database with schema applied.
*/
function openDatabase(dataDir) {
fs.mkdirSync(dataDir, { recursive: true });
const dbPath = path.join(dataDir, "app.db");
const db = new Database(dbPath);
db.pragma("journal_mode = WAL");
db.pragma("foreign_keys = ON");
migrate(db);
return db;
}
/**
* @param {import("better-sqlite3").Database} db - Database handle.
*/
function migrate(db) {
db.exec(`
CREATE TABLE IF NOT EXISTS titles (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL CHECK (source IN ('mangaread', 'manhwasusu')),
url TEXT NOT NULL UNIQUE,
slug TEXT NOT NULL,
title TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
last_checked_at TEXT
);
CREATE TABLE IF NOT EXISTS chapters (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title_id INTEGER NOT NULL REFERENCES titles(id) ON DELETE CASCADE,
chapter_key TEXT NOT NULL,
chapter_number TEXT,
label TEXT NOT NULL,
url TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending'
CHECK (status IN ('pending', 'downloading', 'downloaded', 'failed')),
cbz_path TEXT,
downloaded_at TEXT,
error TEXT,
UNIQUE (title_id, chapter_key)
);
CREATE TABLE IF NOT EXISTS jobs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
type TEXT NOT NULL CHECK (type IN ('watch_sync', 'redownload', 'download')),
title_id INTEGER REFERENCES titles(id) ON DELETE SET NULL,
chapter_id INTEGER REFERENCES chapters(id) ON DELETE SET NULL,
status TEXT NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued', 'running', 'done', 'failed')),
message TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
finished_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_chapters_title_id ON chapters(title_id);
CREATE INDEX IF NOT EXISTS idx_chapters_status ON chapters(status);
CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status);
`);
// Drop active jobs left behind after a title was removed (ON DELETE SET NULL).
db.prepare(
`
DELETE FROM jobs
WHERE status IN ('queued', 'running')
AND (title_id IS NULL OR title_id NOT IN (SELECT id FROM titles))
`,
).run();
}
/**
* @param {import("better-sqlite3").Database} db
* @returns {object[]}
*/
function listTitlesWithCounts(db) {
return db
.prepare(
`
SELECT
t.*,
COUNT(c.id) AS total,
SUM(CASE WHEN c.status = 'downloaded' THEN 1 ELSE 0 END) AS downloaded,
SUM(CASE WHEN c.status = 'pending' THEN 1 ELSE 0 END) AS pending,
SUM(CASE WHEN c.status = 'downloading' THEN 1 ELSE 0 END) AS downloading,
SUM(CASE WHEN c.status = 'failed' THEN 1 ELSE 0 END) AS failed
FROM titles t
LEFT JOIN chapters c ON c.title_id = t.id
GROUP BY t.id
ORDER BY t.id DESC
`,
)
.all()
.map(normalizeTitleRow);
}
/**
* @param {object} row
* @returns {object}
*/
function normalizeTitleRow(row) {
return {
...row,
enabled: Boolean(row.enabled),
total: Number(row.total) || 0,
downloaded: Number(row.downloaded) || 0,
pending: Number(row.pending) || 0,
downloading: Number(row.downloading) || 0,
failed: Number(row.failed) || 0,
};
}
/**
* @param {import("better-sqlite3").Database} db
* @param {number} id
* @returns {object|undefined}
*/
function getTitle(db, id) {
const row = db
.prepare(
`
SELECT
t.*,
COUNT(c.id) AS total,
SUM(CASE WHEN c.status = 'downloaded' THEN 1 ELSE 0 END) AS downloaded,
SUM(CASE WHEN c.status = 'pending' THEN 1 ELSE 0 END) AS pending,
SUM(CASE WHEN c.status = 'downloading' THEN 1 ELSE 0 END) AS downloading,
SUM(CASE WHEN c.status = 'failed' THEN 1 ELSE 0 END) AS failed
FROM titles t
LEFT JOIN chapters c ON c.title_id = t.id
WHERE t.id = ?
GROUP BY t.id
`,
)
.get(id);
return row ? normalizeTitleRow(row) : undefined;
}
/**
* @param {import("better-sqlite3").Database} db
* @param {number} titleId
* @returns {object[]}
*/
function listChapters(db, titleId) {
return db
.prepare(
`
SELECT * FROM chapters
WHERE title_id = ?
ORDER BY
CASE WHEN chapter_number GLOB '[0-9]*' THEN CAST(chapter_number AS REAL) ELSE 1e18 END,
id
`,
)
.all(titleId);
}
/**
* @param {import("better-sqlite3").Database} db
* @param {{ source: string, url: string, slug: string, title: string }} data
* @returns {object}
*/
function insertTitle(db, data) {
const info = db
.prepare(
`
INSERT INTO titles (source, url, slug, title, enabled)
VALUES (@source, @url, @slug, @title, 1)
`,
)
.run(data);
return getTitle(db, Number(info.lastInsertRowid));
}
/**
* @param {import("better-sqlite3").Database} db
* @param {number} id
* @param {{ enabled?: boolean, title?: string }} patch
* @returns {object|undefined}
*/
function updateTitle(db, id, patch) {
const current = db.prepare("SELECT * FROM titles WHERE id = ?").get(id);
if (!current) {
return undefined;
}
const enabled =
patch.enabled === undefined ? current.enabled : patch.enabled ? 1 : 0;
const title = patch.title === undefined ? current.title : patch.title;
db.prepare("UPDATE titles SET enabled = ?, title = ? WHERE id = ?").run(
enabled,
title,
id,
);
return getTitle(db, id);
}
/**
* @param {import("better-sqlite3").Database} db
* @param {number} id
* @returns {boolean}
*/
function deleteTitle(db, id) {
const info = db.prepare("DELETE FROM titles WHERE id = ?").run(id);
return info.changes > 0;
}
/**
* Upsert a chapter row; does not overwrite downloading/downloaded unless forcePending.
* @param {import("better-sqlite3").Database} db
* @param {object} chapter
* @returns {{ id: number, inserted: boolean }}
*/
function upsertChapter(db, chapter) {
const existing = db
.prepare(
"SELECT id, status, cbz_path FROM chapters WHERE title_id = ? AND chapter_key = ?",
)
.get(chapter.title_id, chapter.chapter_key);
if (existing) {
db.prepare(
`
UPDATE chapters
SET label = @label, url = @url, chapter_number = @chapter_number,
cbz_path = COALESCE(@cbz_path, cbz_path)
WHERE id = @id
`,
).run({
id: existing.id,
label: chapter.label,
url: chapter.url,
chapter_number: chapter.chapter_number,
cbz_path: chapter.cbz_path ?? null,
});
return { id: existing.id, inserted: false };
}
const info = db
.prepare(
`
INSERT INTO chapters (
title_id, chapter_key, chapter_number, label, url, status, cbz_path
) VALUES (
@title_id, @chapter_key, @chapter_number, @label, @url, @status, @cbz_path
)
`,
)
.run(chapter);
return { id: Number(info.lastInsertRowid), inserted: true };
}
/**
* @param {import("better-sqlite3").Database} db
* @param {number} id
* @returns {object|undefined}
*/
function getChapter(db, id) {
return db.prepare("SELECT * FROM chapters WHERE id = ?").get(id);
}
/**
* @param {import("better-sqlite3").Database} db
* @param {number} id
* @param {object} fields
*/
function updateChapter(db, id, fields) {
const allowed = [
"status",
"cbz_path",
"downloaded_at",
"error",
"label",
"url",
];
const sets = [];
const values = {};
for (const key of allowed) {
if (Object.prototype.hasOwnProperty.call(fields, key)) {
sets.push(`${key} = @${key}`);
values[key] = fields[key];
}
}
if (sets.length === 0) {
return;
}
values.id = id;
db.prepare(`UPDATE chapters SET ${sets.join(", ")} WHERE id = @id`).run(values);
}
/**
* @param {import("better-sqlite3").Database} db
* @param {{ type: string, title_id?: number|null, chapter_id?: number|null, message?: string|null }} data
* @returns {number} Job id
*/
function createJob(db, data) {
const info = db
.prepare(
`
INSERT INTO jobs (type, title_id, chapter_id, status, message)
VALUES (@type, @title_id, @chapter_id, 'queued', @message)
`,
)
.run({
type: data.type,
title_id: data.title_id ?? null,
chapter_id: data.chapter_id ?? null,
message: data.message ?? null,
});
return Number(info.lastInsertRowid);
}
/**
* @param {import("better-sqlite3").Database} db
* @param {number} id
* @param {object} fields
*/
function updateJob(db, id, fields) {
const allowed = ["status", "message", "finished_at"];
const sets = [];
const values = {};
for (const key of allowed) {
if (Object.prototype.hasOwnProperty.call(fields, key)) {
sets.push(`${key} = @${key}`);
values[key] = fields[key];
}
}
if (sets.length === 0) {
return;
}
values.id = id;
db.prepare(`UPDATE jobs SET ${sets.join(", ")} WHERE id = @id`).run(values);
}
/**
* @param {import("better-sqlite3").Database} db
* @param {{ active?: boolean, limit?: number }} opts
* @returns {object[]}
*/
function listJobs(db, opts = {}) {
const limit = opts.limit ?? 50;
if (opts.active) {
return db
.prepare(
`
SELECT j.*, t.title AS title_name, c.label AS chapter_label
FROM jobs j
INNER JOIN titles t ON t.id = j.title_id
LEFT JOIN chapters c ON c.id = j.chapter_id
WHERE j.status IN ('queued', 'running')
ORDER BY
CASE j.status WHEN 'running' THEN 0 ELSE 1 END,
j.id ASC
LIMIT ?
`,
)
.all(limit);
}
return db
.prepare(
`
SELECT j.*, t.title AS title_name, c.label AS chapter_label
FROM jobs j
LEFT JOIN titles t ON t.id = j.title_id
LEFT JOIN chapters c ON c.id = j.chapter_id
ORDER BY j.id DESC
LIMIT ?
`,
)
.all(limit);
}
/**
* @param {import("better-sqlite3").Database} db
* @returns {object[]}
*/
function listEnabledTitles(db) {
return db
.prepare("SELECT * FROM titles WHERE enabled = 1 ORDER BY id ASC")
.all();
}
/**
* @param {import("better-sqlite3").Database} db
* @param {number} titleId
*/
function touchTitleChecked(db, titleId) {
db.prepare(
"UPDATE titles SET last_checked_at = datetime('now') WHERE id = ?",
).run(titleId);
}
/**
* Chapters that need a download (pending/failed, or downloaded but file missing).
* @param {import("better-sqlite3").Database} db
* @param {number} titleId
* @returns {object[]}
*/
function listChaptersNeedingDownload(db, titleId) {
return db
.prepare(
`
SELECT * FROM chapters
WHERE title_id = ?
AND status IN ('pending', 'failed')
ORDER BY id ASC
`,
)
.all(titleId);
}
/**
* @param {import("better-sqlite3").Database} db - Database handle.
* @param {string} url - Exact listing URL.
* @returns {object|undefined} Title row when found.
*/
function findTitleByUrl(db, url) {
return db.prepare("SELECT * FROM titles WHERE url = ?").get(url);
}
/**
* Pick the next enabled title due for a watch sync (never-checked first, then oldest).
* @param {import("better-sqlite3").Database} db - Database handle.
* @param {number} minAgeHours - Hours since last check before a title is due again.
* @returns {object|undefined} Title row, or undefined when nothing is due.
*/
function pickNextTitleDue(db, minAgeHours) {
const hours = Math.max(1, Math.floor(Number(minAgeHours) || 24));
const modifier = `-${hours} hours`;
return db
.prepare(
`
SELECT *
FROM titles
WHERE enabled = 1
AND (
last_checked_at IS NULL
OR last_checked_at <= datetime('now', '${modifier}')
)
ORDER BY
CASE WHEN last_checked_at IS NULL THEN 0 ELSE 1 END,
last_checked_at ASC,
id ASC
LIMIT 1
`,
)
.get();
}
module.exports = {
openDatabase,
listTitlesWithCounts,
getTitle,
listChapters,
insertTitle,
updateTitle,
deleteTitle,
upsertChapter,
getChapter,
updateChapter,
createJob,
updateJob,
listJobs,
listEnabledTitles,
touchTitleChecked,
listChaptersNeedingDownload,
findTitleByUrl,
pickNextTitleDue,
};
+92
View File
@@ -0,0 +1,92 @@
const path = require("path");
const fs = require("fs");
const Fastify = require("fastify");
const fastifyStatic = require("@fastify/static");
const { openDatabase } = require("./db.js");
const { DownloadQueue } = require("./queue.js");
const { startWatcher } = require("./watcher.js");
const routesPlugin = require("./routes.js");
const PORT = Number(process.env.PORT) || 3000;
const DATA_DIR = path.resolve(process.env.DATA_DIR || path.join(process.cwd(), "data"));
/** How often the watcher wakes (default: every hour). */
const WATCH_CRON = process.env.WATCH_CRON || "0 * * * *";
/** Only sync a title if it has not been checked for this many hours (default: daily). */
const WATCH_TITLE_INTERVAL_HOURS = Number(process.env.WATCH_TITLE_INTERVAL_HOURS) || 24;
const PUBLIC_DIR = path.join(process.cwd(), "public");
/**
* @returns {Promise<void>}
*/
async function main() {
fs.mkdirSync(DATA_DIR, { recursive: true });
fs.mkdirSync(PUBLIC_DIR, { recursive: true });
const db = openDatabase(DATA_DIR);
const queue = new DownloadQueue(db);
const app = Fastify({ logger: true });
// Allow POST/PATCH with Content-Type: application/json and an empty body
app.addContentTypeParser(
"application/json",
{ parseAs: "string" },
(request, body, done) => {
if (body === "" || body == null) {
done(null, {});
return;
}
try {
done(null, JSON.parse(body));
} catch (err) {
done(err instanceof Error ? err : new Error(String(err)), undefined);
}
},
);
await app.register(routesPlugin, { db, queue, dataDir: DATA_DIR });
await app.register(fastifyStatic, {
root: PUBLIC_DIR,
prefix: "/",
wildcard: false,
});
// SPA fallback for non-API routes
app.setNotFoundHandler((request, reply) => {
if (request.url.startsWith("/api/")) {
return reply.code(404).send({ error: "Not found" });
}
const indexPath = path.join(PUBLIC_DIR, "index.html");
if (fs.existsSync(indexPath)) {
return reply.type("text/html").send(fs.readFileSync(indexPath));
}
return reply
.code(503)
.type("text/plain")
.send("Frontend not built. Run: npm run build:web");
});
const watcher = startWatcher(db, queue, DATA_DIR, WATCH_CRON, {
titleIntervalHours: WATCH_TITLE_INTERVAL_HOURS,
});
const shutdown = async () => {
watcher.stop();
await app.close();
db.close();
process.exit(0);
};
process.on("SIGINT", () => void shutdown());
process.on("SIGTERM", () => void shutdown());
await app.listen({ port: PORT, host: "0.0.0.0" });
app.log.info(`Data dir: ${DATA_DIR}`);
app.log.info(`Watch cron: ${WATCH_CRON}`);
app.log.info(`Watch title interval: ${String(WATCH_TITLE_INTERVAL_HOURS)}h (one title per tick)`);
}
main().catch((err) => {
console.error(err);
process.exit(1);
});
+359
View File
@@ -0,0 +1,359 @@
const dbApi = require("./db.js");
const {
downloadChapter,
cbzExists,
removeCbzIfExists,
} = require("./sources.js");
/**
* Serial in-process download queue.
*/
class DownloadQueue {
/**
* @param {import("better-sqlite3").Database} db
* @param {{ onChange?: () => void }} [opts]
*/
constructor(db, opts = {}) {
this.db = db;
this.onChange = opts.onChange;
/** @type {number[]} */
this.pendingJobIds = [];
this.running = false;
/** @type {Set<number>} Chapters cancelled while a download may still be in flight. */
this.cancelledChapterIds = new Set();
}
/**
* Enqueue a chapter download job.
* @param {number} chapterId
* @param {{ type?: 'download'|'redownload', force?: boolean }} [opts]
* @returns {number|null} Job id, or null if skipped (already queued/running for same chapter)
*/
enqueueChapter(chapterId, opts = {}) {
const type = opts.type || (opts.force ? "redownload" : "download");
const chapter = dbApi.getChapter(this.db, chapterId);
if (!chapter) {
throw new Error(`Chapter ${String(chapterId)} not found.`);
}
const active = dbApi
.listJobs(this.db, { active: true, limit: 500 })
.find((j) => j.chapter_id === chapterId);
if (active) {
return active.id;
}
if (opts.force) {
removeCbzIfExists(chapter.cbz_path);
dbApi.updateChapter(this.db, chapterId, {
status: "pending",
error: null,
downloaded_at: null,
});
}
this.cancelledChapterIds.delete(chapterId);
const jobId = dbApi.createJob(this.db, {
type,
title_id: chapter.title_id,
chapter_id: chapterId,
message: `Queued ${chapter.label}`,
});
this.pendingJobIds.push(jobId);
this.#notify();
void this.#pump();
return jobId;
}
/**
* Enqueue all chapters that need downloading for a title.
* @param {number} titleId
* @param {{ alsoMissingFiles?: boolean }} [opts]
* @returns {number} Number of jobs enqueued
*/
enqueueMissingForTitle(titleId, opts = {}) {
const chapters = dbApi.listChapters(this.db, titleId);
let count = 0;
for (const ch of chapters) {
const missingFile =
Boolean(opts.alsoMissingFiles) &&
ch.status === "downloaded" &&
!cbzExists(ch.cbz_path);
const needs =
ch.status === "pending" ||
ch.status === "failed" ||
missingFile;
if (!needs) {
continue;
}
if (missingFile) {
dbApi.updateChapter(this.db, ch.id, { status: "pending", error: "CBZ missing on disk" });
}
this.enqueueChapter(ch.id, { type: "download" });
count += 1;
}
return count;
}
/**
* Drop all waiting download jobs. The in-flight job (if any) is left to finish.
* @returns {{ cleared: number }}
*/
clear() {
this.pendingJobIds = [];
const info = this.db
.prepare(
`
UPDATE jobs
SET status = 'failed',
message = 'Cleared from queue',
finished_at = datetime('now')
WHERE status = 'queued'
`,
)
.run();
this.#notify();
return { cleared: Number(info.changes) || 0 };
}
/**
* Cancel a stuck/queued/downloading chapter: drop its jobs and reset status to pending.
* If a download is mid-flight, ignore its success when it finishes.
* @param {number} chapterId - Chapter row id.
* @returns {{ ok: boolean, chapter: object|undefined }}
*/
cancelChapter(chapterId) {
const chapter = dbApi.getChapter(this.db, chapterId);
if (!chapter) {
return { ok: false, chapter: undefined };
}
this.cancelledChapterIds.add(chapterId);
this.pendingJobIds = this.pendingJobIds.filter((jobId) => {
const job = this.db.prepare("SELECT chapter_id FROM jobs WHERE id = ?").get(jobId);
return !job || job.chapter_id !== chapterId;
});
this.db
.prepare(
`
UPDATE jobs
SET status = 'failed',
message = 'Cancelled by user',
finished_at = datetime('now')
WHERE chapter_id = ?
AND status IN ('queued', 'running')
`,
)
.run(chapterId);
dbApi.updateChapter(this.db, chapterId, {
status: "pending",
error: "Cancelled by user",
downloaded_at: null,
});
this.#notify();
return { ok: true, chapter: dbApi.getChapter(this.db, chapterId) };
}
/**
* Remove a watched title: cancel its jobs/queue entries, then delete the row.
* @param {number} titleId - Title id to remove.
* @returns {boolean} True when the title existed and was deleted.
*/
removeTitle(titleId) {
const title = this.db.prepare("SELECT id FROM titles WHERE id = ?").get(titleId);
if (!title) {
return false;
}
const chapters = dbApi.listChapters(this.db, titleId);
for (const ch of chapters) {
this.cancelledChapterIds.add(ch.id);
}
this.pendingJobIds = this.pendingJobIds.filter((jobId) => {
const job = this.db.prepare("SELECT title_id FROM jobs WHERE id = ?").get(jobId);
return !job || job.title_id !== titleId;
});
this.db
.prepare(
`
UPDATE jobs
SET status = 'failed',
message = 'Title removed',
finished_at = datetime('now')
WHERE title_id = ?
AND status IN ('queued', 'running')
`,
)
.run(titleId);
this.db.prepare("DELETE FROM jobs WHERE title_id = ?").run(titleId);
dbApi.deleteTitle(this.db, titleId);
this.#notify();
return true;
}
async #pump() {
if (this.running) {
return;
}
this.running = true;
try {
while (this.pendingJobIds.length > 0) {
const jobId = this.pendingJobIds.shift();
if (jobId === undefined) {
break;
}
await this.#runJob(jobId);
}
} finally {
this.running = false;
this.#notify();
}
}
/**
* @param {number} jobId
*/
async #runJob(jobId) {
const job = this.db.prepare("SELECT * FROM jobs WHERE id = ?").get(jobId);
if (!job || job.status !== "queued") {
return;
}
const chapter = job.chapter_id
? dbApi.getChapter(this.db, job.chapter_id)
: null;
if (!chapter) {
dbApi.updateJob(this.db, jobId, {
status: "failed",
message: "Chapter missing",
finished_at: new Date().toISOString(),
});
this.#notify();
return;
}
const title = dbApi.getTitle(this.db, chapter.title_id);
if (!title) {
dbApi.updateJob(this.db, jobId, {
status: "failed",
message: "Title missing",
finished_at: new Date().toISOString(),
});
this.#notify();
return;
}
// Skip if already present (unless redownload)
if (
job.type !== "redownload" &&
chapter.status === "downloaded" &&
cbzExists(chapter.cbz_path)
) {
dbApi.updateJob(this.db, jobId, {
status: "done",
message: "Already downloaded",
finished_at: new Date().toISOString(),
});
this.#notify();
return;
}
dbApi.updateJob(this.db, jobId, {
status: "running",
message: `Downloading ${chapter.label}`,
});
dbApi.updateChapter(this.db, chapter.id, {
status: "downloading",
error: null,
});
this.#notify();
const cbzPath = chapter.cbz_path;
if (!cbzPath) {
dbApi.updateChapter(this.db, chapter.id, {
status: "failed",
error: "No cbz_path set",
});
dbApi.updateJob(this.db, jobId, {
status: "failed",
message: "No cbz_path set",
finished_at: new Date().toISOString(),
});
this.#notify();
return;
}
try {
const { imageCount, outputPath } = await downloadChapter(
title.source,
chapter.url,
cbzPath,
);
if (this.cancelledChapterIds.has(chapter.id)) {
this.cancelledChapterIds.delete(chapter.id);
dbApi.updateJob(this.db, jobId, {
status: "failed",
message: "Cancelled by user",
finished_at: new Date().toISOString(),
});
// Keep chapter as pending if cancel already reset it; otherwise force pending.
const latest = dbApi.getChapter(this.db, chapter.id);
if (latest && latest.status === "downloading") {
dbApi.updateChapter(this.db, chapter.id, {
status: "pending",
error: "Cancelled by user",
});
}
this.#notify();
return;
}
dbApi.updateChapter(this.db, chapter.id, {
status: "downloaded",
cbz_path: outputPath,
downloaded_at: new Date().toISOString(),
error: null,
});
dbApi.updateJob(this.db, jobId, {
status: "done",
message: `${chapter.label}: ${String(imageCount)} page(s)`,
finished_at: new Date().toISOString(),
});
} catch (err) {
if (this.cancelledChapterIds.has(chapter.id)) {
this.cancelledChapterIds.delete(chapter.id);
}
const msg = err instanceof Error ? err.message : String(err);
const latest = dbApi.getChapter(this.db, chapter.id);
if (!latest || latest.status === "downloading") {
dbApi.updateChapter(this.db, chapter.id, {
status: "failed",
error: msg,
});
}
dbApi.updateJob(this.db, jobId, {
status: "failed",
message: msg,
finished_at: new Date().toISOString(),
});
}
this.#notify();
}
#notify() {
if (typeof this.onChange === "function") {
this.onChange();
}
}
}
module.exports = { DownloadQueue };
+226
View File
@@ -0,0 +1,226 @@
const dbApi = require("./db.js");
const {
detectSource,
normalizeListingUrl,
fetchSeriesChapters,
resolveCbzPath,
cbzExists,
} = require("./sources.js");
const { syncTitle } = require("./watcher.js");
const { formatUpstreamError } = require("../flaresolverr.js");
/**
* @param {import("fastify").FastifyInstance} fastify - Fastify app.
* @param {{ db: import("better-sqlite3").Database, queue: import("./queue.js").DownloadQueue, dataDir: string }} opts - Shared deps.
*/
async function routesPlugin(fastify, opts) {
const { db, queue, dataDir } = opts;
fastify.get("/api/titles", async () => {
return { titles: dbApi.listTitlesWithCounts(db) };
});
fastify.get("/api/titles/:id", async (request, reply) => {
const id = Number(request.params.id);
const title = dbApi.getTitle(db, id);
if (!title) {
return reply.code(404).send({ error: "Title not found" });
}
return { title };
});
fastify.post("/api/titles", async (request, reply) => {
const body = request.body || {};
const url = typeof body.url === "string" ? body.url.trim() : "";
if (!url) {
return reply.code(400).send({ error: "url is required" });
}
let source;
let listingUrl;
try {
source = detectSource(url);
listingUrl = normalizeListingUrl(url, source);
} catch (err) {
return reply
.code(400)
.send({ error: err instanceof Error ? err.message : String(err) });
}
const siteLabel = source === "manhwasusu" ? "ManhwaSusu" : "Mangaread";
const existing = dbApi.findTitleByUrl(db, listingUrl);
if (existing) {
try {
const result = await syncTitle(db, queue, existing.id, dataDir, {
enqueue: true,
});
return {
title: dbApi.getTitle(db, existing.id),
created: false,
...result,
};
} catch (err) {
return reply.code(502).send({
error: formatUpstreamError(err, siteLabel),
});
}
}
let series;
try {
series = await fetchSeriesChapters(listingUrl, source);
} catch (err) {
return reply.code(502).send({
error: formatUpstreamError(err, siteLabel),
});
}
const title = dbApi.insertTitle(db, {
source: series.source,
url: series.listingUrl,
slug: series.slug,
title: series.comicTitle,
});
for (const ch of series.chapters) {
const cbzPath = resolveCbzPath(
series.source,
series.comicTitle,
ch.url,
dataDir,
);
const exists = cbzExists(cbzPath);
dbApi.upsertChapter(db, {
title_id: title.id,
chapter_key: ch.chapter_key,
chapter_number: ch.chapter_number,
label: ch.title,
url: ch.url,
status: exists ? "downloaded" : "pending",
cbz_path: cbzPath,
});
if (exists) {
const row = db
.prepare(
"SELECT id FROM chapters WHERE title_id = ? AND chapter_key = ?",
)
.get(title.id, ch.chapter_key);
if (row) {
dbApi.updateChapter(db, row.id, {
status: "downloaded",
downloaded_at: new Date().toISOString(),
error: null,
});
}
}
}
dbApi.touchTitleChecked(db, title.id);
const enqueued = queue.enqueueMissingForTitle(title.id, {
alsoMissingFiles: true,
});
return {
title: dbApi.getTitle(db, title.id),
created: true,
added: series.chapters.length,
enqueued,
total: series.chapters.length,
};
});
fastify.patch("/api/titles/:id", async (request, reply) => {
const id = Number(request.params.id);
const body = request.body || {};
const patch = {};
if (typeof body.enabled === "boolean") {
patch.enabled = body.enabled;
}
if (typeof body.title === "string" && body.title.trim()) {
patch.title = body.title.trim();
}
const title = dbApi.updateTitle(db, id, patch);
if (!title) {
return reply.code(404).send({ error: "Title not found" });
}
return { title };
});
fastify.delete("/api/titles/:id", async (request, reply) => {
const id = Number(request.params.id);
const ok = queue.removeTitle(id);
if (!ok) {
return reply.code(404).send({ error: "Title not found" });
}
return {
ok: true,
jobs: dbApi.listJobs(db, { active: true, limit: 20 }),
};
});
fastify.get("/api/titles/:id/chapters", async (request, reply) => {
const id = Number(request.params.id);
const title = dbApi.getTitle(db, id);
if (!title) {
return reply.code(404).send({ error: "Title not found" });
}
return { chapters: dbApi.listChapters(db, id) };
});
fastify.post("/api/titles/:id/sync", async (request, reply) => {
const id = Number(request.params.id);
const title = dbApi.getTitle(db, id);
if (!title) {
return reply.code(404).send({ error: "Title not found" });
}
const siteLabel = title.source === "manhwasusu" ? "ManhwaSusu" : "Mangaread";
try {
const result = await syncTitle(db, queue, id, dataDir, { enqueue: true });
return { title: dbApi.getTitle(db, id), ...result };
} catch (err) {
return reply.code(502).send({
error: formatUpstreamError(err, siteLabel),
});
}
});
fastify.post("/api/chapters/:id/redownload", async (request, reply) => {
const id = Number(request.params.id);
const chapter = dbApi.getChapter(db, id);
if (!chapter) {
return reply.code(404).send({ error: "Chapter not found" });
}
const jobId = queue.enqueueChapter(id, { force: true, type: "redownload" });
return { ok: true, jobId, chapter: dbApi.getChapter(db, id) };
});
fastify.post("/api/chapters/:id/cancel", async (request, reply) => {
const id = Number(request.params.id);
const result = queue.cancelChapter(id);
if (!result.ok) {
return reply.code(404).send({ error: "Chapter not found" });
}
return { ok: true, chapter: result.chapter };
});
fastify.get("/api/jobs", async (request) => {
const active =
request.query.active === "1" ||
request.query.active === "true" ||
request.query.active === true;
const limit = request.query.limit ? Number(request.query.limit) : 50;
return { jobs: dbApi.listJobs(db, { active, limit }) };
});
fastify.post("/api/jobs/clear", async () => {
const result = queue.clear();
return {
ok: true,
cleared: result.cleared,
jobs: dbApi.listJobs(db, { active: true, limit: 20 }),
};
});
}
module.exports = routesPlugin;
+258
View File
@@ -0,0 +1,258 @@
const fs = require("fs");
const path = require("path");
const {
fetchWpMangaChapterLinks,
mangaSlugFromUrl,
} = require("../mangareadFetchChapters.js");
const {
chapterUrlToCbz: mangareadChapterUrlToCbz,
sanitizeComicDirName: mangareadSanitizeDir,
sanitizeFsStem: mangareadSanitizeStem,
stemForCbzFilename: mangareadStem,
} = require("../downloadChapterCbz.js");
const { chaptersInReadingOrder: mangareadReadingOrder } = require("../downloadChaptersQueue.js");
const { chapterNumberFromWpTitle } = require("../downloadChaptersQueue.js");
const manhwa = require("../manhwasusu/index.js");
const {
chaptersInReadingOrder: manhwaReadingOrder,
} = require("../manhwasusu/downloadChaptersQueue.js");
/**
* @typedef {'mangaread'|'manhwasusu'} SourceId
*/
/**
* Detect source from a series listing URL.
* @param {string} rawUrl
* @returns {SourceId}
*/
function detectSource(rawUrl) {
let parsed;
try {
parsed = new URL(rawUrl);
} catch {
throw new Error("Invalid URL.");
}
const host = parsed.hostname.toLowerCase();
const pathname = parsed.pathname.replace(/\/+$/u, "");
if (
host.includes("manhwasusu") ||
manhwa.isLikelyManhwaSusuSite(rawUrl) ||
/^\/read\/[^/]+/iu.test(pathname)
) {
if (manhwa.isLikelyManhwaSusuSite(rawUrl) || host.includes("manhwasusu")) {
return "manhwasusu";
}
}
if (
host.includes("mangaread") ||
/^\/manga\/[^/]+/iu.test(pathname)
) {
return "mangaread";
}
// Heuristic fallbacks by path shape
if (/^\/read\/[^/]+/iu.test(pathname)) {
return "manhwasusu";
}
if (/^\/manga\/[^/]+/iu.test(pathname)) {
return "mangaread";
}
throw new Error(
"Unsupported URL. Use a Mangaread `/manga/{slug}/` or ManhwaSusu `/read/{slug}/` series page.",
);
}
/**
* Normalize listing URL (strip chapter segment if pasted a chapter URL).
* @param {string} rawUrl
* @param {SourceId} source
* @returns {string}
*/
function normalizeListingUrl(rawUrl, source) {
const u = new URL(rawUrl);
const segs = u.pathname.replace(/\/+$/u, "").split("/").filter(Boolean);
if (source === "mangaread") {
const mangaIdx = segs.indexOf("manga");
if (mangaIdx !== -1 && segs[mangaIdx + 1]) {
u.pathname = `/manga/${segs[mangaIdx + 1]}/`;
u.search = "";
u.hash = "";
return u.toString();
}
}
if (source === "manhwasusu") {
const readIdx = segs.indexOf("read");
if (readIdx !== -1 && segs[readIdx + 1]) {
u.pathname = `/read/${segs[readIdx + 1]}/`;
u.search = "";
u.hash = "";
return u.toString();
}
}
if (!u.pathname.endsWith("/")) {
u.pathname = `${u.pathname}/`;
}
return u.toString();
}
/**
* @param {string} chapterUrl
* @returns {string} e.g. chapter-22
*/
function chapterKeyFromUrl(chapterUrl) {
try {
const segs = new URL(chapterUrl).pathname.replace(/\/+$/u, "").split("/").filter(Boolean);
return segs.at(-1) || "chapter";
} catch {
return "chapter";
}
}
/**
* @param {string} title
* @param {SourceId} source
* @returns {string|null}
*/
function chapterNumberFromTitle(title, source) {
if (source === "mangaread") {
return chapterNumberFromWpTitle(title);
}
const m = /^Chapter\s+([\d.]+)/iu.exec(title.trim().replace(/\s+/gu, " "));
return m ? m[1] : null;
}
/**
* @param {SourceId} source - Site adapter id.
* @param {string} dataDir - Absolute data root.
* @returns {string} Comics output directory for the source.
*/
function comicsRoot(source, dataDir) {
if (source === "manhwasusu") {
return path.join(dataDir, "manhwasusu", "comics");
}
return path.join(dataDir, "comics");
}
/**
* @param {SourceId} source
* @param {string} comicTitle
* @param {string} chapterUrl
* @param {string} dataDir
* @returns {string} Absolute CBZ path
*/
function resolveCbzPath(source, comicTitle, chapterUrl, dataDir) {
const dirName =
source === "manhwasusu"
? manhwa.sanitizeComicDirName(comicTitle)
: mangareadSanitizeDir(comicTitle);
const stem =
source === "manhwasusu"
? manhwa.sanitizeFsStem(manhwa.stemForManhwaSusuChapterCbz(chapterUrl))
: mangareadSanitizeStem(mangareadStem(chapterUrl));
return path.join(comicsRoot(source, dataDir), dirName, `${stem}.cbz`);
}
/**
* Fetch chapter list for a series listing URL.
* @param {string} listingUrl
* @param {SourceId} [sourceHint]
* @returns {Promise<{ source: SourceId, listingUrl: string, slug: string, comicTitle: string, chapters: Array<{ url: string, title: string, chapter_key: string, chapter_number: string|null }> }>}
*/
async function fetchSeriesChapters(listingUrl, sourceHint) {
const source = sourceHint || detectSource(listingUrl);
const normalized = normalizeListingUrl(listingUrl, source);
if (source === "manhwasusu") {
const { chapters, comicTitle } = await manhwa.fetchChapterLinks(normalized);
const ordered = manhwaReadingOrder(chapters);
const slug = manhwa.seriesSlugFromListingUrl(normalized);
return {
source,
listingUrl: normalized,
slug,
comicTitle,
chapters: ordered.map((ch) => ({
url: ch.url,
title: ch.title,
chapter_key: chapterKeyFromUrl(ch.url),
chapter_number: chapterNumberFromTitle(ch.title, source),
})),
};
}
const { chapters, comicTitle } = await fetchWpMangaChapterLinks(normalized);
const ordered = mangareadReadingOrder(chapters);
const slug = mangaSlugFromUrl(normalized);
return {
source,
listingUrl: normalized,
slug,
comicTitle,
chapters: ordered.map((ch) => ({
url: ch.url,
title: ch.title,
chapter_key: chapterKeyFromUrl(ch.url),
chapter_number: chapterNumberFromTitle(ch.title, source),
})),
};
}
/**
* Download one chapter to CBZ.
* @param {SourceId} source
* @param {string} chapterUrl
* @param {string} outputCbzPath
* @returns {Promise<{ imageCount: number, outputPath: string }>}
*/
async function downloadChapter(source, chapterUrl, outputCbzPath) {
if (source === "manhwasusu") {
return manhwa.chapterUrlToCbz(chapterUrl, outputCbzPath);
}
return mangareadChapterUrlToCbz(chapterUrl, outputCbzPath);
}
/**
* @param {string|null|undefined} cbzPath
* @returns {boolean}
*/
function cbzExists(cbzPath) {
if (!cbzPath) {
return false;
}
try {
return fs.existsSync(cbzPath) && fs.statSync(cbzPath).isFile();
} catch {
return false;
}
}
/**
* @param {string|null|undefined} cbzPath
*/
function removeCbzIfExists(cbzPath) {
if (cbzPath && fs.existsSync(cbzPath)) {
fs.unlinkSync(cbzPath);
}
}
module.exports = {
detectSource,
normalizeListingUrl,
chapterKeyFromUrl,
resolveCbzPath,
fetchSeriesChapters,
downloadChapter,
cbzExists,
removeCbzIfExists,
comicsRoot,
};
+172
View File
@@ -0,0 +1,172 @@
const cron = require("node-cron");
const dbApi = require("./db.js");
const {
fetchSeriesChapters,
resolveCbzPath,
cbzExists,
} = require("./sources.js");
/**
* Sync a title's chapter list from the source site and enqueue missing downloads.
* @param {import("better-sqlite3").Database} db - Database handle.
* @param {import("./queue.js").DownloadQueue} queue - Download queue.
* @param {number} titleId - Title to sync.
* @param {string} dataDir - Absolute data root for CBZ paths.
* @param {{ enqueue?: boolean }} [opts] - Options; `enqueue` defaults to true.
* @returns {Promise<{ added: number, enqueued: number, total: number }>} Sync summary.
*/
async function syncTitle(db, queue, titleId, dataDir, opts = {}) {
const title = dbApi.getTitle(db, titleId);
if (!title) {
throw new Error(`Title ${String(titleId)} not found.`);
}
const syncJobId = dbApi.createJob(db, {
type: "watch_sync",
title_id: titleId,
message: `Syncing ${title.title}`,
});
dbApi.updateJob(db, syncJobId, { status: "running" });
try {
const series = await fetchSeriesChapters(title.url, title.source);
let added = 0;
for (const ch of series.chapters) {
const cbzPath = resolveCbzPath(
title.source,
series.comicTitle || title.title,
ch.url,
dataDir,
);
const exists = cbzExists(cbzPath);
const { inserted } = dbApi.upsertChapter(db, {
title_id: titleId,
chapter_key: ch.chapter_key,
chapter_number: ch.chapter_number,
label: ch.title,
url: ch.url,
status: exists ? "downloaded" : "pending",
cbz_path: cbzPath,
});
if (inserted) {
added += 1;
}
// Heal statuses without interrupting an in-flight download
const row = db
.prepare(
"SELECT id, status, cbz_path FROM chapters WHERE title_id = ? AND chapter_key = ?",
)
.get(titleId, ch.chapter_key);
if (row && row.status === "downloading") {
continue;
}
if (row && row.status === "downloaded" && !cbzExists(row.cbz_path)) {
dbApi.updateChapter(db, row.id, {
status: "pending",
error: "CBZ missing on disk",
});
}
if (row && exists && row.status !== "downloaded") {
dbApi.updateChapter(db, row.id, {
status: "downloaded",
cbz_path: cbzPath,
downloaded_at: new Date().toISOString(),
error: null,
});
}
}
// Refresh comic title if site provides a better one
if (series.comicTitle && series.comicTitle !== title.title) {
dbApi.updateTitle(db, titleId, { title: series.comicTitle });
}
dbApi.touchTitleChecked(db, titleId);
let enqueued = 0;
if (opts.enqueue !== false) {
enqueued = queue.enqueueMissingForTitle(titleId, { alsoMissingFiles: true });
}
dbApi.updateJob(db, syncJobId, {
status: "done",
message: `Sync complete: +${String(added)} chapter(s), ${String(enqueued)} queued`,
finished_at: new Date().toISOString(),
});
return {
added,
enqueued,
total: series.chapters.length,
};
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
dbApi.updateJob(db, syncJobId, {
status: "failed",
message: msg,
finished_at: new Date().toISOString(),
});
throw err;
}
}
/**
* Watches titles on a schedule. Each tick syncs at most one due title
* (oldest `last_checked_at` first), so titles are staggered instead of
* all hitting the source sites at once. Default cadence is hourly wake +
* 24h per-title cooldown each title roughly once per day, unlimited count.
* @param {import("better-sqlite3").Database} db - Database handle.
* @param {import("./queue.js").DownloadQueue} queue - Download queue.
* @param {string} dataDir - Absolute data root.
* @param {string} cronExpr - node-cron expression.
* @param {{ titleIntervalHours?: number }} [opts] - Per-title cooldown in hours.
* @returns {{ stop: () => void }} Handle to stop the cron task.
*/
function startWatcher(db, queue, dataDir, cronExpr, opts = {}) {
if (!cron.validate(cronExpr)) {
throw new Error(`Invalid WATCH_CRON expression: ${cronExpr}`);
}
const titleIntervalHours =
Number(opts.titleIntervalHours) > 0 ? Number(opts.titleIntervalHours) : 24;
let busy = false;
const task = cron.schedule(cronExpr, () => {
if (busy) {
console.warn("[watcher] previous sync still running; skipping this tick");
return;
}
busy = true;
void (async () => {
try {
const title = dbApi.pickNextTitleDue(db, titleIntervalHours);
if (!title) {
console.info("[watcher] no titles due this tick");
return;
}
console.info(
`[watcher] syncing 1 title: ${title.title} (${title.source})`,
);
await syncTitle(db, queue, title.id, dataDir, { enqueue: true });
} catch (err) {
console.error(
"[watcher] sync failed:",
err instanceof Error ? err.message : err,
);
} finally {
busy = false;
}
})();
});
return {
stop() {
task.stop();
},
};
}
module.exports = { syncTitle, startWatcher };
+12
View File
@@ -0,0 +1,12 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Watch & Download</title>
</head>
<body>
<div id="app"></div>
<script type="module" src="/src/main.js"></script>
</body>
</html>
+378
View File
@@ -0,0 +1,378 @@
<script>
let titles = $state([]);
let jobs = $state([]);
let chaptersByTitle = $state({});
let url = $state("");
let error = $state("");
let busy = $state(false);
let loadingChapters = $state({});
let expanded = $state({});
async function api(path, options = {}) {
const method = (options.method || "GET").toUpperCase();
const headers = { ...(options.headers || {}) };
let body = options.body;
if (body !== undefined && body !== null) {
if (!headers["Content-Type"]) {
headers["Content-Type"] = "application/json";
}
} else if (method === "POST" || method === "PUT" || method === "PATCH") {
// Fastify rejects Content-Type: application/json with an empty body
headers["Content-Type"] = "application/json";
body = "{}";
}
const res = await fetch(path, {
...options,
method,
headers,
body,
});
const data = await res.json().catch(() => ({}));
if (!res.ok) {
throw new Error(data.error || data.message || res.statusText || "Request failed");
}
return data;
}
async function refresh() {
const [t, j] = await Promise.all([
api("/api/titles"),
api("/api/jobs?active=1&limit=20"),
]);
titles = t.titles || [];
jobs = j.jobs || [];
}
async function loadChapters(titleId) {
loadingChapters = { ...loadingChapters, [titleId]: true };
try {
const data = await api(`/api/titles/${titleId}/chapters`);
chaptersByTitle = { ...chaptersByTitle, [titleId]: data.chapters || [] };
} finally {
loadingChapters = { ...loadingChapters, [titleId]: false };
}
}
async function toggleExpand(titleId) {
const next = !expanded[titleId];
expanded = { ...expanded, [titleId]: next };
if (next && !chaptersByTitle[titleId]) {
await loadChapters(titleId);
}
}
async function addTitle(event) {
event.preventDefault();
error = "";
busy = true;
try {
await api("/api/titles", {
method: "POST",
body: JSON.stringify({ url: url.trim() }),
});
url = "";
await refresh();
} catch (err) {
error = err instanceof Error ? err.message : String(err);
} finally {
busy = false;
}
}
async function syncTitle(id) {
error = "";
busy = true;
try {
await api(`/api/titles/${id}/sync`, { method: "POST" });
await refresh();
if (expanded[id]) {
await loadChapters(id);
}
} catch (err) {
error = err instanceof Error ? err.message : String(err);
} finally {
busy = false;
}
}
async function toggleEnabled(title) {
error = "";
try {
await api(`/api/titles/${title.id}`, {
method: "PATCH",
body: JSON.stringify({ enabled: !title.enabled }),
});
await refresh();
} catch (err) {
error = err instanceof Error ? err.message : String(err);
}
}
async function removeTitle(id) {
if (!confirm("Remove this title from the watch list?")) {
return;
}
error = "";
try {
const data = await api(`/api/titles/${id}`, { method: "DELETE" });
const nextCh = { ...chaptersByTitle };
delete nextCh[id];
chaptersByTitle = nextCh;
if (data.jobs) {
jobs = data.jobs;
}
await refresh();
} catch (err) {
error = err instanceof Error ? err.message : String(err);
}
}
async function redownload(chapterId, titleId) {
error = "";
try {
await api(`/api/chapters/${chapterId}/redownload`, { method: "POST" });
await refresh();
await loadChapters(titleId);
} catch (err) {
error = err instanceof Error ? err.message : String(err);
}
}
async function cancelChapter(chapterId, titleId) {
error = "";
try {
await api(`/api/chapters/${chapterId}/cancel`, { method: "POST" });
await refresh();
if (titleId && expanded[titleId]) {
await loadChapters(titleId);
}
} catch (err) {
error = err instanceof Error ? err.message : String(err);
}
}
async function clearQueue() {
if (!confirm("Clear all waiting download jobs? The current download (if any) will finish.")) {
return;
}
error = "";
try {
const data = await api("/api/jobs/clear", { method: "POST" });
jobs = data.jobs || [];
await refresh();
} catch (err) {
error = err instanceof Error ? err.message : String(err);
}
}
$effect(() => {
void refresh();
const id = setInterval(() => {
void (async () => {
await refresh();
for (const [titleId, isOpen] of Object.entries(expanded)) {
if (isOpen) {
await loadChapters(Number(titleId));
}
}
})();
}, 2000);
return () => clearInterval(id);
});
</script>
<main class="app-shell">
<header>
<h1>Watch & Download</h1>
</header>
{#if error}
<p class="error-banner" role="alert">{error}</p>
{/if}
<form class="add-title" onsubmit={addTitle}>
<label>
Series URL
<input
type="url"
bind:value={url}
placeholder=""
required
disabled={busy}
/>
</label>
<button type="submit" disabled={busy || !url.trim()}>
{busy ? "Working…" : "Add & watch"}
</button>
</form>
<section class="jobs-strip" aria-live="polite">
<div class="section-heading">
<h2>Activity</h2>
<button
type="button"
class="outline contrast"
disabled={jobs.length === 0}
onclick={clearQueue}
>
Clear queue
</button>
</div>
{#if jobs.length === 0}
<p class="muted">No active jobs.</p>
{:else}
<ul class="job-list">
{#each jobs as job (job.id)}
<li class="job-row">
<div class="job-info">
<strong>{job.status}</strong>
{job.message || job.type}
{#if job.title_name}
<span class="muted">({job.title_name})</span>
{/if}
</div>
{#if job.chapter_id}
<button
type="button"
class="outline contrast job-cancel"
onclick={() => cancelChapter(job.chapter_id, job.title_id)}
>
Cancel
</button>
{/if}
</li>
{/each}
</ul>
{/if}
</section>
<section>
<h2>Titles</h2>
{#if titles.length === 0}
<p class="muted">No titles yet. Add a series URL above.</p>
{:else}
{#each titles as title (title.id)}
<article>
<header>
<div class="title-meta">
<strong>{title.title}</strong>
<span class="muted">{title.source}</span>
<span class="progress-text">
{title.downloaded} / {title.total} downloaded
{#if title.downloading > 0}
· {title.downloading} downloading
{/if}
{#if title.failed > 0}
· {title.failed} failed
{/if}
</span>
</div>
<p class="muted url-line">
<a href={title.url} target="_blank" rel="noreferrer">{title.url}</a>
{#if title.last_checked_at}
· last checked {title.last_checked_at}
{/if}
· watch {title.enabled ? "on" : "off"}
</p>
<div class="row-actions">
<button type="button" class="outline" onclick={() => toggleExpand(title.id)}>
{expanded[title.id] ? "Hide chapters" : "Show chapters"}
</button>
<button type="button" class="outline" disabled={busy} onclick={() => syncTitle(title.id)}>
Sync now
</button>
<button type="button" class="outline secondary" onclick={() => toggleEnabled(title)}>
{title.enabled ? "Disable watch" : "Enable watch"}
</button>
<button type="button" class="outline contrast" onclick={() => removeTitle(title.id)}>
Remove
</button>
</div>
</header>
{#if expanded[title.id]}
{#if loadingChapters[title.id] && !chaptersByTitle[title.id]}
<p class="muted">Loading chapters…</p>
{:else}
<ul class="chapter-list">
{#each chaptersByTitle[title.id] || [] as ch (ch.id)}
<li class="chapter-card">
<div class="chapter-label">{ch.label}</div>
<span class={`status status-${ch.status}`}>{ch.status}</span>
{#if ch.error}
<div class="muted chapter-error">{ch.error}</div>
{/if}
<div class="chapter-actions">
{#if ch.status === "downloading"}
<button
type="button"
class="outline contrast"
onclick={() => cancelChapter(ch.id, title.id)}
>
Cancel
</button>
{/if}
<button
type="button"
class="outline"
onclick={() => redownload(ch.id, title.id)}
>
Redownload
</button>
</div>
</li>
{/each}
</ul>
<div class="table-wrap">
<table>
<thead>
<tr>
<th>Chapter</th>
<th>Status</th>
<th></th>
</tr>
</thead>
<tbody>
{#each chaptersByTitle[title.id] || [] as ch (ch.id)}
<tr>
<td>{ch.label}</td>
<td>
<span class={`status status-${ch.status}`}>{ch.status}</span>
{#if ch.error}
<div class="muted">{ch.error}</div>
{/if}
</td>
<td>
<div class="chapter-actions">
{#if ch.status === "downloading"}
<button
type="button"
class="outline contrast"
onclick={() => cancelChapter(ch.id, title.id)}
>
Cancel
</button>
{/if}
<button
type="button"
class="outline"
onclick={() => redownload(ch.id, title.id)}
>
Redownload
</button>
</div>
</td>
</tr>
{/each}
</tbody>
</table>
</div>
{/if}
{/if}
</article>
{/each}
{/if}
</section>
</main>
+332
View File
@@ -0,0 +1,332 @@
:root {
--pico-font-family: "Segoe UI", system-ui, sans-serif;
}
*,
*::before,
*::after {
box-sizing: border-box;
}
html {
overflow-x: clip;
}
body {
margin: 0;
overflow-x: clip;
width: 100%;
}
#app {
width: 100%;
min-width: 0;
max-width: 100%;
}
/* Fluid layout — avoid Pico .container stepped max-widths (510/700/950px) */
.app-shell {
display: block;
width: 100%;
max-width: min(960px, 100%);
margin-inline: auto;
padding-inline: max(1rem, env(safe-area-inset-left, 0px), env(safe-area-inset-right, 0px));
padding-block: 1.25rem;
min-width: 0;
}
h1 {
font-size: clamp(1.5rem, 5vw, 2rem);
line-height: 1.2;
overflow-wrap: anywhere;
}
.muted {
color: var(--pico-muted-color);
font-size: 0.9rem;
}
.url-line,
.url-line a {
overflow-wrap: anywhere;
word-break: break-word;
}
.row-actions {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 0.5rem;
margin-top: 0.75rem;
}
.row-actions button {
width: 100%;
margin-bottom: 0;
white-space: normal;
line-height: 1.2;
padding-inline: 0.5rem;
}
.status {
text-transform: uppercase;
font-size: 0.75rem;
letter-spacing: 0.04em;
font-weight: 600;
}
.status-pending {
color: #a67c00;
}
.status-downloading {
color: #0b6bcb;
}
.status-downloaded {
color: #1b7f3a;
}
.status-failed {
color: #c62828;
}
.progress-text {
font-variant-numeric: tabular-nums;
}
.error-banner {
color: #c62828;
overflow-wrap: anywhere;
}
.jobs-strip {
margin-bottom: 1.5rem;
}
.section-heading {
display: flex;
flex-wrap: wrap;
align-items: center;
justify-content: space-between;
gap: 0.75rem;
margin-bottom: 0.75rem;
}
.section-heading h2 {
margin-bottom: 0;
}
.section-heading button {
width: auto;
margin-bottom: 0;
}
.jobs-strip ul {
margin-bottom: 0;
padding-inline-start: 0;
list-style: none;
}
.jobs-strip li {
overflow-wrap: anywhere;
word-break: break-word;
}
.job-list {
display: grid;
gap: 0.5rem;
}
.job-row {
display: flex;
flex-wrap: wrap;
align-items: center;
justify-content: space-between;
gap: 0.5rem 0.75rem;
padding: 0.5rem 0.65rem;
border: 1px solid var(--pico-muted-border-color);
border-radius: var(--pico-border-radius);
}
.job-info {
flex: 1 1 12rem;
min-width: 0;
}
.job-cancel {
width: auto;
margin-bottom: 0;
flex: 0 0 auto;
}
@media (max-width: 639px) {
.job-cancel {
width: 100%;
}
}
.title-meta {
display: flex;
flex-direction: column;
gap: 0.25rem;
min-width: 0;
}
.title-meta strong {
overflow-wrap: anywhere;
word-break: break-word;
}
article {
min-width: 0;
max-width: 100%;
overflow: hidden;
}
article header {
min-width: 0;
}
section {
min-width: 0;
max-width: 100%;
}
form.add-title {
display: grid;
gap: 0.75rem;
margin-bottom: 2rem;
min-width: 0;
max-width: 100%;
}
form.add-title label {
min-width: 0;
}
form.add-title input {
max-width: 100%;
min-width: 0;
}
form.add-title button {
width: 100%;
margin-bottom: 0;
}
.table-wrap {
width: 100%;
max-width: 100%;
overflow-x: auto;
-webkit-overflow-scrolling: touch;
margin-top: 0.75rem;
}
.table-wrap table {
margin-bottom: 0;
min-width: 28rem;
}
.table-wrap th,
.table-wrap td {
white-space: nowrap;
}
.table-wrap td:first-child {
white-space: normal;
overflow-wrap: anywhere;
max-width: 12rem;
}
.chapter-list {
display: grid;
gap: 0.75rem;
margin-top: 0.75rem;
list-style: none;
padding: 0;
}
.chapter-card {
border: 1px solid var(--pico-muted-border-color);
border-radius: var(--pico-border-radius);
padding: 0.75rem;
min-width: 0;
}
.chapter-card .chapter-label {
font-weight: 600;
overflow-wrap: anywhere;
margin-bottom: 0.35rem;
}
.chapter-card .chapter-error {
margin-top: 0.25rem;
overflow-wrap: anywhere;
}
.chapter-card button {
width: 100%;
margin-top: 0.5rem;
margin-bottom: 0;
}
.chapter-actions {
display: flex;
flex-wrap: wrap;
gap: 0.5rem;
margin-top: 0.5rem;
}
.chapter-actions button {
margin-top: 0;
margin-bottom: 0;
}
@media (max-width: 639px) {
.chapter-actions button {
width: 100%;
}
}
@media (min-width: 640px) {
.app-shell {
padding-inline: max(1.5rem, env(safe-area-inset-left, 0px), env(safe-area-inset-right, 0px));
padding-block: 1.5rem;
}
form.add-title {
grid-template-columns: minmax(0, 1fr) auto;
align-items: end;
}
form.add-title button {
width: auto;
}
.title-meta {
flex-direction: row;
flex-wrap: wrap;
gap: 0.75rem 1.25rem;
align-items: baseline;
}
.row-actions {
display: flex;
flex-wrap: wrap;
grid-template-columns: unset;
}
.row-actions button {
width: auto;
}
.chapter-list {
display: none;
}
}
@media (max-width: 639px) {
.table-wrap {
display: none;
}
}
+9
View File
@@ -0,0 +1,9 @@
import "@picocss/pico/css/pico.min.css";
import "./app.css";
import { mount } from "svelte";
import App from "./App.svelte";
const target = document.getElementById("app");
if (target) {
mount(App, { target });
}
+8
View File
@@ -0,0 +1,8 @@
import { vitePreprocess } from "@sveltejs/vite-plugin-svelte";
/** @type {import('@sveltejs/vite-plugin-svelte').SvelteConfig} */
const config = {
preprocess: vitePreprocess(),
};
export default config;
+20
View File
@@ -0,0 +1,20 @@
import { defineConfig } from "vite";
import { svelte } from "@sveltejs/vite-plugin-svelte";
import path from "node:path";
import { fileURLToPath } from "node:url";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
export default defineConfig({
root: __dirname,
plugins: [svelte()],
build: {
outDir: path.resolve(__dirname, "../public"),
emptyOutDir: true,
},
server: {
proxy: {
"/api": "http://127.0.0.1:3000",
},
},
});