feat: integrate node-html-parser for HTML fetching and parsing

- Added node-html-parser as a dependency for parsing HTML content. - Implemented fetch-html script to fetch and parse HTML from a specified URL. - Updated script sandbox to allow the use of node-html-parser. - Created a new workflow for fetching data from dev.to, including configuration for URL and selectors. - Updated package.json to include node-html-parser in server package dependencies.
2026-08-14 15:02:27 +07:00
parent 023cf83cfa
commit 389f94f950
6 changed files with 184 additions and 2 deletions
@@ -21,6 +21,7 @@
    "jsonata": "^2.2.2",
    "knex": "^3.3.0",
    "node-cron": "^4.6.0",
+    "node-html-parser": "^9.0.1",
    "pino": "^10.3.1",
    "pino-roll": "^4.0.0",
    "yaml": "^2.9.0"
@@ -7,7 +7,7 @@ import { SCRIPTS_DIR } from "./paths.js";

 const hostRequire = createRequire(import.meta.url);

-const ALLOWED_MODULES = new Set(["axios", "jsonata"]);
+const ALLOWED_MODULES = new Set(["axios", "jsonata", "node-html-parser"]);

 const BUILTIN_NAMES = [
  "Infinity",
@@ -0,0 +1,85 @@
+import { parse } from "node-html-parser";
+import jsonata from "jsonata";
+
+function ensureDataObject(ctx) {
+  if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) {
+    ctx.data = {};
+  }
+}
+
+function serializeElement(el) {
+  const attrs = el.attributes ?? {};
+  const classNames = attrs.class ? attrs.class.split(/\s+/).filter(Boolean) : [];
+  return {
+    tagName: el.tagName?.toLowerCase() ?? null,
+    id: attrs.id ?? null,
+    classNames,
+    attributes: { ...attrs },
+    text: el.text?.trim() ?? "",
+    innerHTML: el.innerHTML ?? "",
+    outerHTML: el.outerHTML ?? "",
+  };
+}
+
+function previewValue(value) {
+  const json = JSON.stringify(value);
+  if (json.length <= 500) {
+    return value;
+  }
+  return { preview: `${json.slice(0, 500)}...`, truncated: true };
+}
+
+export default async function fetchHtml(ctx) {
+  const url = ctx.config?.url;
+  if (typeof url !== "string" || url.length === 0) {
+    throw new Error("fetch-html: ctx.config.url is required");
+  }
+
+  const selector = ctx.config?.selector;
+  const outputVar = ctx.config?.outputVar;
+  const jsonataExpr = ctx.config?.jsonata;
+
+  if ((selector || jsonataExpr) && (typeof outputVar !== "string" || outputVar.length === 0)) {
+    throw new Error("fetch-html: ctx.config.outputVar is required when selector or jsonata is set");
+  }
+
+  ensureDataObject(ctx);
+
+  log.info({ url }, "fetch-html: fetching page");
+  const response = await $axios.get(url);
+  log.info(
+    { status: response.status, length: String(response.data ?? "").length },
+    "fetch-html: fetch complete",
+  );
+
+  ctx.data.httpResponse = String(response.data ?? "");
+  log.info(
+    { length: ctx.data.httpResponse.length },
+    "fetch-html: saved httpResponse",
+  );
+
+  if (selector) {
+    log.info({ selector, outputVar }, "fetch-html: selecting elements");
+    const root = parse(ctx.data.httpResponse);
+    const elements = root.querySelectorAll(selector);
+    ctx.data[outputVar] = elements.map(serializeElement);
+    log.info(
+      { outputVar, count: ctx.data[outputVar].length },
+      "fetch-html: saved selector matches",
+    );
+  }
+
+  if (jsonataExpr) {
+    log.info({ outputVar, jsonata: jsonataExpr }, "fetch-html: evaluating jsonata");
+    const expression = jsonata(jsonataExpr);
+    const input = ctx.data[outputVar];
+    const result = await expression.evaluate(input);
+    ctx.data[outputVar] = result;
+    log.info(
+      { outputVar, result: previewValue(result) },
+      "fetch-html: saved jsonata result",
+    );
+  }
+
+  return ctx;
+}
@@ -0,0 +1,12 @@
+name: fetch-devto
+scripts:
+  - script: fetch-html.js
+    config:
+      url: https://dev.to/t/productivity/top/week
+      selector: "#substories h2"
+      outputVar: titles
+      jsonata: "$[].text"
+triggers:
+  - type: HTTP
+    method: POST
+    path: /fetch-dev-to
@@ -3,3 +3,4 @@ scripts:
  - test.yaml
  - manual-trigger.yaml
  - cron-example.yaml
+  - fetch-devto.yaml
@@ -43,6 +43,9 @@ importers:
      node-cron:
        specifier: ^4.6.0
        version: 4.6.0
+      node-html-parser:
+        specifier: ^9.0.1
+        version: 9.0.1
      pino:
        specifier: ^10.3.1
        version: 10.3.1
@@ -840,6 +843,9 @@ packages:
  bn.js@4.12.5:
    resolution: {integrity: sha512-3aRg6/JxfffFD+OlOjOFR3Vo79l39ooBTFucxx+MT3dhCtzn3EmiUPQo+6/OZuI2jbXi3YKgmiTFBgChQMwIRQ==}

+  boolbase@1.0.0:
+    resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
+
  brace-expansion@5.0.9:
    resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==}
    engines: {node: 20 || >=22}
@@ -900,6 +906,13 @@ packages:
    resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
    engines: {node: '>= 8'}

+  css-select@5.2.2:
+    resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==}
+
+  css-what@6.2.2:
+    resolution: {integrity: sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==}
+    engines: {node: '>= 6'}
+
  cytoscape-cose-bilkent@4.1.0:
    resolution: {integrity: sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==}
    peerDependencies:
@@ -1102,12 +1115,25 @@ packages:
    resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==}
    engines: {node: '>=8'}

+  dom-serializer@2.0.0:
+    resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==}
+
+  domelementtype@2.3.0:
+    resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==}
+
+  domhandler@5.0.3:
+    resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==}
+    engines: {node: '>= 4'}
+
  dompurify@3.4.13:
    resolution: {integrity: sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==}

  dompurify@3.4.8:
    resolution: {integrity: sha512-yb1cEmaOum7wFvOCSQxyfgVlv5D47Rc30iZWoMpbDIWTnJ6grDDQyu2KFJzB2k7u0pMuJcQ1zphH//fFnw2tjQ==}

+  domutils@3.2.2:
+    resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==}
+
  dunder-proto@1.0.1:
    resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
    engines: {node: '>= 0.4'}
@@ -1122,6 +1148,14 @@ packages:
    resolution: {integrity: sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==}
    engines: {node: '>=10.13.0'}

+  entities@4.5.0:
+    resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==}
+    engines: {node: '>=0.12'}
+
+  entities@8.0.0:
+    resolution: {integrity: sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==}
+    engines: {node: '>=20.19.0'}
+
  es-define-property@1.0.1:
    resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==}
    engines: {node: '>= 0.4'}
@@ -1559,10 +1593,16 @@ packages:
    resolution: {integrity: sha512-Si/bzYiKRHOB8/a99T2+SDGN582ONDMSTlJr5oCkT6GtnqPjZ2s10eoQRYkW9ZHwjVxONL+W8Fb+qR0AHMQsdg==}
    engines: {node: '>=20'}

+  node-html-parser@9.0.1:
+    resolution: {integrity: sha512-QrdiYYm1NnLRXsMXThUgVcF/syWfWIgHFmy8hylWMGFbHFtnRuXLBxxGQvIm3xaIJMUDJ0ayOmT/FGJYT3pZIw==}
+
  node-releases@2.0.53:
    resolution: {integrity: sha512-D9UOmYG3UH1V+ENW56t5QXBwJw1YEY18ruVeus89Rw+SyIgjPkCO84bRzO3uNIYosJbNwiabWVn48o3uJLjxFQ==}
    engines: {node: '>=18'}

+  nth-check@2.1.1:
+    resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
+
  obliterator@2.0.5:
    resolution: {integrity: sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==}

@@ -2584,6 +2624,8 @@ snapshots:

  bn.js@4.12.5: {}

+  boolbase@1.0.0: {}
+
  brace-expansion@5.0.9:
    dependencies:
      balanced-match: 4.0.4
@@ -2639,6 +2681,16 @@ snapshots:
      shebang-command: 2.0.0
      which: 2.0.2

+  css-select@5.2.2:
+    dependencies:
+      boolbase: 1.0.0
+      css-what: 6.2.2
+      domhandler: 5.0.3
+      domutils: 3.2.2
+      nth-check: 2.1.1
+
+  css-what@6.2.2: {}
+
  cytoscape-cose-bilkent@4.1.0(cytoscape@3.34.1):
    dependencies:
      cose-base: 1.0.3
@@ -2849,6 +2901,18 @@ snapshots:

  detect-libc@2.1.2: {}

+  dom-serializer@2.0.0:
+    dependencies:
+      domelementtype: 2.3.0
+      domhandler: 5.0.3
+      entities: 4.5.0
+
+  domelementtype@2.3.0: {}
+
+  domhandler@5.0.3:
+    dependencies:
+      domelementtype: 2.3.0
+
  dompurify@3.4.13:
    optionalDependencies:
      '@types/trusted-types': 2.0.7
@@ -2857,6 +2921,12 @@ snapshots:
    optionalDependencies:
      '@types/trusted-types': 2.0.7

+  domutils@3.2.2:
+    dependencies:
+      dom-serializer: 2.0.0
+      domelementtype: 2.3.0
+      domhandler: 5.0.3
+
  dunder-proto@1.0.1:
    dependencies:
      call-bind-apply-helpers: 1.0.2
@@ -2874,6 +2944,10 @@ snapshots:
      graceful-fs: 4.2.11
      tapable: 2.3.3

+  entities@4.5.0: {}
+
+  entities@8.0.0: {}
+
  es-define-property@1.0.1: {}

  es-errors@1.3.0: {}
@@ -3295,8 +3369,17 @@ snapshots:

  node-cron@4.6.0: {}

+  node-html-parser@9.0.1:
+    dependencies:
+      css-select: 5.2.2
+      entities: 8.0.0
+
  node-releases@2.0.53: {}

+  nth-check@2.1.1:
+    dependencies:
+      boolbase: 1.0.0
+
  obliterator@2.0.5: {}

  on-exit-leak-free@2.1.2: {}