feat: integrate node-html-parser for HTML fetching and parsing

- Added node-html-parser as a dependency for parsing HTML content. - Implemented fetch-html script to fetch and parse HTML from a specified URL. - Updated script sandbox to allow the use of node-html-parser. - Created a new workflow for fetching data from dev.to, including configuration for URL and selectors. - Updated package.json to include node-html-parser in server package dependencies.
2026-08-14 15:02:27 +07:00
parent 023cf83cfa
commit 389f94f950
6 changed files with 184 additions and 2 deletions
@@ -0,0 +1,85 @@
+import { parse } from "node-html-parser";
+import jsonata from "jsonata";
+
+function ensureDataObject(ctx) {
+  if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) {
+    ctx.data = {};
+  }
+}
+
+function serializeElement(el) {
+  const attrs = el.attributes ?? {};
+  const classNames = attrs.class ? attrs.class.split(/\s+/).filter(Boolean) : [];
+  return {
+    tagName: el.tagName?.toLowerCase() ?? null,
+    id: attrs.id ?? null,
+    classNames,
+    attributes: { ...attrs },
+    text: el.text?.trim() ?? "",
+    innerHTML: el.innerHTML ?? "",
+    outerHTML: el.outerHTML ?? "",
+  };
+}
+
+function previewValue(value) {
+  const json = JSON.stringify(value);
+  if (json.length <= 500) {
+    return value;
+  }
+  return { preview: `${json.slice(0, 500)}...`, truncated: true };
+}
+
+export default async function fetchHtml(ctx) {
+  const url = ctx.config?.url;
+  if (typeof url !== "string" || url.length === 0) {
+    throw new Error("fetch-html: ctx.config.url is required");
+  }
+
+  const selector = ctx.config?.selector;
+  const outputVar = ctx.config?.outputVar;
+  const jsonataExpr = ctx.config?.jsonata;
+
+  if ((selector || jsonataExpr) && (typeof outputVar !== "string" || outputVar.length === 0)) {
+    throw new Error("fetch-html: ctx.config.outputVar is required when selector or jsonata is set");
+  }
+
+  ensureDataObject(ctx);
+
+  log.info({ url }, "fetch-html: fetching page");
+  const response = await $axios.get(url);
+  log.info(
+    { status: response.status, length: String(response.data ?? "").length },
+    "fetch-html: fetch complete",
+  );
+
+  ctx.data.httpResponse = String(response.data ?? "");
+  log.info(
+    { length: ctx.data.httpResponse.length },
+    "fetch-html: saved httpResponse",
+  );
+
+  if (selector) {
+    log.info({ selector, outputVar }, "fetch-html: selecting elements");
+    const root = parse(ctx.data.httpResponse);
+    const elements = root.querySelectorAll(selector);
+    ctx.data[outputVar] = elements.map(serializeElement);
+    log.info(
+      { outputVar, count: ctx.data[outputVar].length },
+      "fetch-html: saved selector matches",
+    );
+  }
+
+  if (jsonataExpr) {
+    log.info({ outputVar, jsonata: jsonataExpr }, "fetch-html: evaluating jsonata");
+    const expression = jsonata(jsonataExpr);
+    const input = ctx.data[outputVar];
+    const result = await expression.evaluate(input);
+    ctx.data[outputVar] = result;
+    log.info(
+      { outputVar, result: previewValue(result) },
+      "fetch-html: saved jsonata result",
+    );
+  }
+
+  return ctx;
+}