cxyz/scripts/fetch-page.js

const dns = require("dns");
const fs = require("fs");
const https = require("https");
const net = require("net");
const path = require("path");
const axios = require("axios");

const USER_AGENT =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " +
  "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";

/**
 * @typedef {object} ParsedCli
 * @property {boolean} help - Print usage and exit when true.
 * @property {string|undefined} url - Absolute page URL.
 * @property {string|undefined} outPath - Optional file to write the body to.
 * @property {number} timeoutMs - Axios timeout in milliseconds.
 * @property {string|null} domain - Optional Host header / logical hostname.
 */

/**
 * Parse CLI args: `node scripts/fetch-page.js <url> [--domain <host>] [--out <file>] [--timeout <ms>]`.
 * @param {string[]} argv - `process.argv.slice(2)`.
 * @returns {ParsedCli} Parsed flags and positional URL.
 */
function parseCli(argv) {
  /** @type {ParsedCli} */
  const result = {
    help: false,
    url: undefined,
    outPath: undefined,
    timeoutMs: 15_000,
    domain: null,
  };

  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];
    if (arg === "-h" || arg === "--help") {
      result.help = true;
      continue;
    }

    if (arg === "--domain" || arg === "-d") {
      result.domain = argv[i + 1];
      i += 1;
      continue;
    }

    if (arg === "--out" || arg === "-o") {
      result.outPath = argv[i + 1];
      i += 1;
      continue;
    }
    if (arg.startsWith("--out=")) {
      result.outPath = arg.slice("--out=".length);
      continue;
    }
    if (arg === "--timeout") {
      result.timeoutMs = Number(argv[i + 1]);
      i += 1;
      continue;
    }
    if (arg.startsWith("--timeout=")) {
      result.timeoutMs = Number(arg.slice("--timeout=".length));
      continue;
    }
    if (!arg.startsWith("-") && result.url === undefined) {
      result.url = arg;
      continue;
    }
    throw new Error(`Unknown argument: ${arg}`);
  }

  return result;
}

/**
 * Write a diagnostic line to stderr so stdout stays the page body.
 * @param {string} line - Message.
 * @returns {void}
 */
function log(line) {
  console.error(line);
}

/**
 * Flatten Axios or Node headers into a plain object.
 * @param {unknown} headers - AxiosHeaders, IncomingHttpHeaders, or plain object.
 * @returns {Record<string, string>} Header names mapped to string values.
 */
function plainHeaders(headers) {
  if (!headers || typeof headers !== "object") {
    return {};
  }
  const raw =
    typeof /** @type {{ toJSON?: () => unknown }} */ (headers).toJSON ===
    "function"
      ? /** @type {{ toJSON: () => unknown }} */ (headers).toJSON()
      : headers;
  /** @type {Record<string, string>} */
  const out = {};
  for (const [key, value] of Object.entries(raw || {})) {
    if (value == null) {
      continue;
    }
    out[key] = Array.isArray(value) ? value.join(", ") : String(value);
  }
  return out;
}

/**
 * Log HTTP headers, one per line.
 * @param {unknown} headers - Response or request headers.
 * @returns {void}
 */
function logHeaders(headers) {
  const entries = Object.entries(plainHeaders(headers));
  if (entries.length === 0) {
    log("  (no headers)");
    return;
  }
  for (const [key, value] of entries) {
    log(`  ${key}: ${value}`);
  }
}

/**
 * Resolve A/AAAA records and OS lookup for a hostname; log the query.
 * @param {string} hostname - Name or IP to resolve.
 * @param {string} [label] - Extra context, e.g. "connect" or "Host header".
 * @returns {Promise<void>}
 */
async function logDnsQuery(hostname, label) {
  const tag = label ? ` (${label})` : "";
  if (net.isIP(hostname)) {
    log(`dns query${tag}: ${hostname}`);
    log(`  skipped — host is already an IP (IPv${String(net.isIP(hostname))})`);
    return;
  }

  const servers = dns.getServers();
  log(`dns query${tag}: ${hostname}`);
  log(`  servers: ${servers.join(", ")}`);

  const started = Date.now();
  const [a, aaaa, lookup] = await Promise.all([
    dns.promises.resolve4(hostname, { ttl: true }).catch((err) => err),
    dns.promises.resolve6(hostname, { ttl: true }).catch((err) => err),
    dns.promises.lookup(hostname, { all: true }).catch((err) => err),
  ]);
  const elapsed = Date.now() - started;

  if (a instanceof Error) {
    log(`  A:    ${a.message}`);
  } else {
    for (const rec of a) {
      log(`  A:    ${rec.address}  ttl=${String(rec.ttl)}`);
    }
    if (a.length === 0) {
      log("  A:    (empty)");
    }
  }

  if (aaaa instanceof Error) {
    log(`  AAAA: ${aaaa.message}`);
  } else {
    for (const rec of aaaa) {
      log(`  AAAA: ${rec.address}  ttl=${String(rec.ttl)}`);
    }
    if (aaaa.length === 0) {
      log("  AAAA: (empty)");
    }
  }

  if (lookup instanceof Error) {
    log(`  lookup (getaddrinfo): ${lookup.message}`);
  } else {
    for (const rec of lookup) {
      log(`  lookup (getaddrinfo): ${rec.address}  IPv${String(rec.family)}`);
    }
    if (lookup.length === 0) {
      log("  lookup (getaddrinfo): (empty)");
    }
  }

  log(`  elapsed: ${String(elapsed)}ms`);
}

/**
 * Agent `lookup` that logs the hostname Node actually resolves for the socket.
 * @param {string} hostname - Name passed by http/https.
 * @param {import("dns").LookupOptions} options - dns.lookup options.
 * @param {(...args: unknown[]) => void} callback - Node lookup callback.
 * @returns {void}
 */
function loggedLookup(hostname, options, callback) {
  const started = Date.now();
  dns.lookup(hostname, options, (err, address, family) => {
    const elapsed = Date.now() - started;
    if (err) {
      log(
        `dns lookup (socket): ${hostname} failed in ${String(elapsed)}ms: ${err.message}`,
      );
    } else {
      const addrs = Array.isArray(address)
        ? address
            .map((row) => `${row.address} IPv${String(row.family)}`)
            .join(", ")
        : `${String(address)} IPv${String(family)}`;
      log(
        `dns lookup (socket): ${hostname} -> ${addrs}  (${String(elapsed)}ms)`,
      );
    }
    callback(err, address, family);
  });
}

/**
 * Log axios failure details (TLS handshake, no response, or HTTP error).
 * @param {unknown} err - Caught error.
 * @returns {void}
 */
function logAxiosError(err) {
  const ax = /** @type {import("axios").AxiosError} */ (err);
  log("response: (failed)");
  if (ax.code) {
    log(`  code: ${ax.code}`);
  }
  log(`  message: ${ax instanceof Error ? ax.message : String(ax)}`);

  const syscall =
    ax && typeof ax === "object" && "syscall" in ax
      ? /** @type {{ syscall?: string }} */ (ax).syscall
      : undefined;
  const address =
    ax && typeof ax === "object" && "address" in ax
      ? /** @type {{ address?: string }} */ (ax).address
      : undefined;
  const port =
    ax && typeof ax === "object" && "port" in ax
      ? /** @type {{ port?: number }} */ (ax).port
      : undefined;
  if (syscall) {
    log(`  syscall: ${syscall}`);
  }
  if (address != null) {
    log(`  peer: ${address}${port != null ? `:${String(port)}` : ""}`);
  }

  if (ax.response) {
    log(
      `  http status: ${String(ax.response.status)} ${ax.response.statusText || ""}`.trim(),
    );
    log("  http headers:");
    logHeaders(ax.response.headers);
    const body = ax.response.data;
    if (typeof body === "string" && body.length > 0) {
      const preview = body.length > 500 ? `${body.slice(0, 500)}…` : body;
      log(`  http body preview: ${preview}`);
    }
    return;
  }

  log("  (no HTTP response — connection or TLS failed before headers)");
  if (ax.code === "EPROTO" || /ssl|tls|handshake/iu.test(String(ax.message))) {
    log(
      "  hint: HTTPS to an IP uses that IP as TLS SNI unless servername is set; CDNs often send alert 40",
    );
  }
}

/**
 * GET a page URL and return the response body as text.
 * @param {string} url - Absolute page URL.
 * @param {number} timeoutMs - Axios timeout.
 * @param {string|null} domain - Optional Host header / logical hostname.
 * @returns {Promise<{ status: number, contentType: string, body: string }>} HTTP status, content-type, and body text.
 */
async function fetchPage(url, timeoutMs, domain) {
  const parsedUrl = new URL(url);
  const connectHost = parsedUrl.hostname;
  const hostHeader = domain || parsedUrl.host;
  const sni = domain && net.isIP(connectHost) ? domain : connectHost;

  await logDnsQuery(connectHost, "connect");
  if (domain && domain !== connectHost) {
    await logDnsQuery(domain, "Host / --domain");
  }

  log("request:");
  log(`  GET ${url}`);
  log(`  connect host: ${connectHost}`);
  log(`  Host header: ${hostHeader}`);
  log(`  tls servername (SNI): ${sni}`);

  const httpsAgent = new https.Agent({
    lookup: loggedLookup,
    servername: sni,
  });

  let data;
  let status;
  let headers;
  try {
    ({ data, status, headers } = await axios.get(url, {
      responseType: "text",
      timeout: timeoutMs,
      httpsAgent,
      validateStatus: (s) => s >= 200 && s < 300,
      headers: {
        Accept:
          "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.9",
        "User-Agent": USER_AGENT,
        Host: hostHeader,
      },
    }));
  } catch (err) {
    logAxiosError(err);
    throw err;
  }

  log(`response: ${String(status)}`);
  logHeaders(headers);

  return {
    status,
    contentType: String(headers["content-type"] || ""),
    body: typeof data === "string" ? data : String(data ?? ""),
  };
}

/**
 * Fetch a website page with axios and print the body (or write it to a file).
 *
 * Usage:
 *   node scripts/fetch-page.js <url> [--domain <host>] [--out <file>] [--timeout <ms>]
 * @returns {Promise<void>}
 */
async function main() {
  let parsed;
  try {
    parsed = parseCli(process.argv.slice(2));
  } catch (err) {
    console.error(err instanceof Error ? err.message : err);
    process.exitCode = 1;
    return;
  }

  if (parsed.help || !parsed.url) {
    console.error(
      "Usage: node scripts/fetch-page.js <url> [--domain <host>] [--out <file>] [--timeout <ms>]",
    );
    if (!parsed.help) {
      process.exitCode = 1;
    }
    return;
  }

  if (!Number.isFinite(parsed.timeoutMs) || parsed.timeoutMs <= 0) {
    console.error("--timeout must be a positive number of milliseconds");
    process.exitCode = 1;
    return;
  }

  let page;
  try {
    page = await fetchPage(parsed.url, parsed.timeoutMs, parsed.domain);
  } catch (err) {
    console.error(err instanceof Error ? err.message : err);
    process.exitCode = 1;
    return;
  }

  log(`bytes: ${String(Buffer.byteLength(page.body, "utf8"))}`);

  if (parsed.outPath) {
    const abs = path.resolve(parsed.outPath);
    fs.writeFileSync(abs, page.body, "utf8");
    log(`wrote: ${abs}`);
    return;
  }

  process.stdout.write(page.body);
  if (!page.body.endsWith("\n")) {
    process.stdout.write("\n");
  }
}

if (require.main === module) {
  main().catch((err) => {
    console.error(err);
    process.exitCode = 1;
  });
}

module.exports = { fetchPage, main, parseCli };