const dns = require("dns"); const fs = require("fs"); const https = require("https"); const net = require("net"); const path = require("path"); const axios = require("axios"); const USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"; /** * @typedef {object} ParsedCli * @property {boolean} help - Print usage and exit when true. * @property {string|undefined} url - Absolute page URL. * @property {string|undefined} outPath - Optional file to write the body to. * @property {number} timeoutMs - Axios timeout in milliseconds. * @property {string|null} domain - Optional Host header / logical hostname. */ /** * Parse CLI args: `node scripts/fetch-page.js [--domain ] [--out ] [--timeout ]`. * @param {string[]} argv - `process.argv.slice(2)`. * @returns {ParsedCli} Parsed flags and positional URL. */ function parseCli(argv) { /** @type {ParsedCli} */ const result = { help: false, url: undefined, outPath: undefined, timeoutMs: 15_000, domain: null, }; for (let i = 0; i < argv.length; i += 1) { const arg = argv[i]; if (arg === "-h" || arg === "--help") { result.help = true; continue; } if (arg === "--domain" || arg === "-d") { result.domain = argv[i + 1]; i += 1; continue; } if (arg === "--out" || arg === "-o") { result.outPath = argv[i + 1]; i += 1; continue; } if (arg.startsWith("--out=")) { result.outPath = arg.slice("--out=".length); continue; } if (arg === "--timeout") { result.timeoutMs = Number(argv[i + 1]); i += 1; continue; } if (arg.startsWith("--timeout=")) { result.timeoutMs = Number(arg.slice("--timeout=".length)); continue; } if (!arg.startsWith("-") && result.url === undefined) { result.url = arg; continue; } throw new Error(`Unknown argument: ${arg}`); } return result; } /** * Write a diagnostic line to stderr so stdout stays the page body. * @param {string} line - Message. * @returns {void} */ function log(line) { console.error(line); } /** * Flatten Axios or Node headers into a plain object. * @param {unknown} headers - AxiosHeaders, IncomingHttpHeaders, or plain object. * @returns {Record} Header names mapped to string values. */ function plainHeaders(headers) { if (!headers || typeof headers !== "object") { return {}; } const raw = typeof /** @type {{ toJSON?: () => unknown }} */ (headers).toJSON === "function" ? /** @type {{ toJSON: () => unknown }} */ (headers).toJSON() : headers; /** @type {Record} */ const out = {}; for (const [key, value] of Object.entries(raw || {})) { if (value == null) { continue; } out[key] = Array.isArray(value) ? value.join(", ") : String(value); } return out; } /** * Log HTTP headers, one per line. * @param {unknown} headers - Response or request headers. * @returns {void} */ function logHeaders(headers) { const entries = Object.entries(plainHeaders(headers)); if (entries.length === 0) { log(" (no headers)"); return; } for (const [key, value] of entries) { log(` ${key}: ${value}`); } } /** * Resolve A/AAAA records and OS lookup for a hostname; log the query. * @param {string} hostname - Name or IP to resolve. * @param {string} [label] - Extra context, e.g. "connect" or "Host header". * @returns {Promise} */ async function logDnsQuery(hostname, label) { const tag = label ? ` (${label})` : ""; if (net.isIP(hostname)) { log(`dns query${tag}: ${hostname}`); log(` skipped — host is already an IP (IPv${String(net.isIP(hostname))})`); return; } const servers = dns.getServers(); log(`dns query${tag}: ${hostname}`); log(` servers: ${servers.join(", ")}`); const started = Date.now(); const [a, aaaa, lookup] = await Promise.all([ dns.promises.resolve4(hostname, { ttl: true }).catch((err) => err), dns.promises.resolve6(hostname, { ttl: true }).catch((err) => err), dns.promises.lookup(hostname, { all: true }).catch((err) => err), ]); const elapsed = Date.now() - started; if (a instanceof Error) { log(` A: ${a.message}`); } else { for (const rec of a) { log(` A: ${rec.address} ttl=${String(rec.ttl)}`); } if (a.length === 0) { log(" A: (empty)"); } } if (aaaa instanceof Error) { log(` AAAA: ${aaaa.message}`); } else { for (const rec of aaaa) { log(` AAAA: ${rec.address} ttl=${String(rec.ttl)}`); } if (aaaa.length === 0) { log(" AAAA: (empty)"); } } if (lookup instanceof Error) { log(` lookup (getaddrinfo): ${lookup.message}`); } else { for (const rec of lookup) { log(` lookup (getaddrinfo): ${rec.address} IPv${String(rec.family)}`); } if (lookup.length === 0) { log(" lookup (getaddrinfo): (empty)"); } } log(` elapsed: ${String(elapsed)}ms`); } /** * Agent `lookup` that logs the hostname Node actually resolves for the socket. * @param {string} hostname - Name passed by http/https. * @param {import("dns").LookupOptions} options - dns.lookup options. * @param {(...args: unknown[]) => void} callback - Node lookup callback. * @returns {void} */ function loggedLookup(hostname, options, callback) { const started = Date.now(); dns.lookup(hostname, options, (err, address, family) => { const elapsed = Date.now() - started; if (err) { log( `dns lookup (socket): ${hostname} failed in ${String(elapsed)}ms: ${err.message}`, ); } else { const addrs = Array.isArray(address) ? address .map((row) => `${row.address} IPv${String(row.family)}`) .join(", ") : `${String(address)} IPv${String(family)}`; log( `dns lookup (socket): ${hostname} -> ${addrs} (${String(elapsed)}ms)`, ); } callback(err, address, family); }); } /** * Log axios failure details (TLS handshake, no response, or HTTP error). * @param {unknown} err - Caught error. * @returns {void} */ function logAxiosError(err) { const ax = /** @type {import("axios").AxiosError} */ (err); log("response: (failed)"); if (ax.code) { log(` code: ${ax.code}`); } log(` message: ${ax instanceof Error ? ax.message : String(ax)}`); const syscall = ax && typeof ax === "object" && "syscall" in ax ? /** @type {{ syscall?: string }} */ (ax).syscall : undefined; const address = ax && typeof ax === "object" && "address" in ax ? /** @type {{ address?: string }} */ (ax).address : undefined; const port = ax && typeof ax === "object" && "port" in ax ? /** @type {{ port?: number }} */ (ax).port : undefined; if (syscall) { log(` syscall: ${syscall}`); } if (address != null) { log(` peer: ${address}${port != null ? `:${String(port)}` : ""}`); } if (ax.response) { log( ` http status: ${String(ax.response.status)} ${ax.response.statusText || ""}`.trim(), ); log(" http headers:"); logHeaders(ax.response.headers); const body = ax.response.data; if (typeof body === "string" && body.length > 0) { const preview = body.length > 500 ? `${body.slice(0, 500)}…` : body; log(` http body preview: ${preview}`); } return; } log(" (no HTTP response — connection or TLS failed before headers)"); if (ax.code === "EPROTO" || /ssl|tls|handshake/iu.test(String(ax.message))) { log( " hint: HTTPS to an IP uses that IP as TLS SNI unless servername is set; CDNs often send alert 40", ); } } /** * GET a page URL and return the response body as text. * @param {string} url - Absolute page URL. * @param {number} timeoutMs - Axios timeout. * @param {string|null} domain - Optional Host header / logical hostname. * @returns {Promise<{ status: number, contentType: string, body: string }>} HTTP status, content-type, and body text. */ async function fetchPage(url, timeoutMs, domain) { const parsedUrl = new URL(url); const connectHost = parsedUrl.hostname; const hostHeader = domain || parsedUrl.host; const sni = domain && net.isIP(connectHost) ? domain : connectHost; await logDnsQuery(connectHost, "connect"); if (domain && domain !== connectHost) { await logDnsQuery(domain, "Host / --domain"); } log("request:"); log(` GET ${url}`); log(` connect host: ${connectHost}`); log(` Host header: ${hostHeader}`); log(` tls servername (SNI): ${sni}`); const httpsAgent = new https.Agent({ lookup: loggedLookup, servername: sni, }); let data; let status; let headers; try { ({ data, status, headers } = await axios.get(url, { responseType: "text", timeout: timeoutMs, httpsAgent, validateStatus: (s) => s >= 200 && s < 300, headers: { Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9", "User-Agent": USER_AGENT, Host: hostHeader, }, })); } catch (err) { logAxiosError(err); throw err; } log(`response: ${String(status)}`); logHeaders(headers); return { status, contentType: String(headers["content-type"] || ""), body: typeof data === "string" ? data : String(data ?? ""), }; } /** * Fetch a website page with axios and print the body (or write it to a file). * * Usage: * node scripts/fetch-page.js [--domain ] [--out ] [--timeout ] * @returns {Promise} */ async function main() { let parsed; try { parsed = parseCli(process.argv.slice(2)); } catch (err) { console.error(err instanceof Error ? err.message : err); process.exitCode = 1; return; } if (parsed.help || !parsed.url) { console.error( "Usage: node scripts/fetch-page.js [--domain ] [--out ] [--timeout ]", ); if (!parsed.help) { process.exitCode = 1; } return; } if (!Number.isFinite(parsed.timeoutMs) || parsed.timeoutMs <= 0) { console.error("--timeout must be a positive number of milliseconds"); process.exitCode = 1; return; } let page; try { page = await fetchPage(parsed.url, parsed.timeoutMs, parsed.domain); } catch (err) { console.error(err instanceof Error ? err.message : err); process.exitCode = 1; return; } log(`bytes: ${String(Buffer.byteLength(page.body, "utf8"))}`); if (parsed.outPath) { const abs = path.resolve(parsed.outPath); fs.writeFileSync(abs, page.body, "utf8"); log(`wrote: ${abs}`); return; } process.stdout.write(page.body); if (!page.body.endsWith("\n")) { process.stdout.write("\n"); } } if (require.main === module) { main().catch((err) => { console.error(err); process.exitCode = 1; }); } module.exports = { fetchPage, main, parseCli };