feat: integrate node-html-parser for HTML fetching and parsing

- Added node-html-parser as a dependency for parsing HTML content.
- Implemented fetch-html script to fetch and parse HTML from a specified URL.
- Updated script sandbox to allow the use of node-html-parser.
- Created a new workflow for fetching data from dev.to, including configuration for URL and selectors.
- Updated package.json to include node-html-parser in server package dependencies.

packages/server/package.json

@@ -21,6 +21,7 @@
     "jsonata": "^2.2.2",
     "knex": "^3.3.0",
     "node-cron": "^4.6.0",
+    "node-html-parser": "^9.0.1",
     "pino": "^10.3.1",
     "pino-roll": "^4.0.0",
     "yaml": "^2.9.0"

packages/server/script-sandbox.js

@@ -7,7 +7,7 @@ import { SCRIPTS_DIR } from "./paths.js";
 
 const hostRequire = createRequire(import.meta.url);
 
-const ALLOWED_MODULES = new Set(["axios", "jsonata"]);
+const ALLOWED_MODULES = new Set(["axios", "jsonata", "node-html-parser"]);
 
 const BUILTIN_NAMES = [
   "Infinity",

packages/server/scripts/fetch-html.js

@@ -0,0 +1,85 @@
+import { parse } from "node-html-parser";
+import jsonata from "jsonata";
+
+function ensureDataObject(ctx) {
+  if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) {
+    ctx.data = {};
+  }
+}
+
+function serializeElement(el) {
+  const attrs = el.attributes ?? {};
+  const classNames = attrs.class ? attrs.class.split(/\s+/).filter(Boolean) : [];
+  return {
+    tagName: el.tagName?.toLowerCase() ?? null,
+    id: attrs.id ?? null,
+    classNames,
+    attributes: { ...attrs },
+    text: el.text?.trim() ?? "",
+    innerHTML: el.innerHTML ?? "",
+    outerHTML: el.outerHTML ?? "",
+  };
+}
+
+function previewValue(value) {
+  const json = JSON.stringify(value);
+  if (json.length <= 500) {
+    return value;
+  }
+  return { preview: `${json.slice(0, 500)}...`, truncated: true };
+}
+
+export default async function fetchHtml(ctx) {
+  const url = ctx.config?.url;
+  if (typeof url !== "string" || url.length === 0) {
+    throw new Error("fetch-html: ctx.config.url is required");
+  }
+
+  const selector = ctx.config?.selector;
+  const outputVar = ctx.config?.outputVar;
+  const jsonataExpr = ctx.config?.jsonata;
+
+  if ((selector || jsonataExpr) && (typeof outputVar !== "string" || outputVar.length === 0)) {
+    throw new Error("fetch-html: ctx.config.outputVar is required when selector or jsonata is set");
+  }
+
+  ensureDataObject(ctx);
+
+  log.info({ url }, "fetch-html: fetching page");
+  const response = await $axios.get(url);
+  log.info(
+    { status: response.status, length: String(response.data ?? "").length },
+    "fetch-html: fetch complete",
+  );
+
+  ctx.data.httpResponse = String(response.data ?? "");
+  log.info(
+    { length: ctx.data.httpResponse.length },
+    "fetch-html: saved httpResponse",
+  );
+
+  if (selector) {
+    log.info({ selector, outputVar }, "fetch-html: selecting elements");
+    const root = parse(ctx.data.httpResponse);
+    const elements = root.querySelectorAll(selector);
+    ctx.data[outputVar] = elements.map(serializeElement);
+    log.info(
+      { outputVar, count: ctx.data[outputVar].length },
+      "fetch-html: saved selector matches",
+    );
+  }
+
+  if (jsonataExpr) {
+    log.info({ outputVar, jsonata: jsonataExpr }, "fetch-html: evaluating jsonata");
+    const expression = jsonata(jsonataExpr);
+    const input = ctx.data[outputVar];
+    const result = await expression.evaluate(input);
+    ctx.data[outputVar] = result;
+    log.info(
+      { outputVar, result: previewValue(result) },
+      "fetch-html: saved jsonata result",
+    );
+  }
+
+  return ctx;
+}

packages/server/workflows/default/fetch-devto.yaml

@@ -0,0 +1,12 @@
+name: fetch-devto
+scripts:
+  - script: fetch-html.js
+    config:
+      url: https://dev.to/t/productivity/top/week
+      selector: "#substories h2"
+      outputVar: titles
+      jsonata: "$[].text"
+triggers:
+  - type: HTTP
+    method: POST
+    path: /fetch-dev-to

packages/server/workflows/default/registers.yaml

@@ -2,4 +2,5 @@ scripts:
   - time-to-ntfy-example.yaml
   - test.yaml
   - manual-trigger.yaml
-  - cron-example.yaml
+  - cron-example.yaml
+  - fetch-devto.yaml