From 389f94f950dc86d38b3557ea649fbe9e153c8449 Mon Sep 17 00:00:00 2001 From: Nasyarobby Putra Date: Fri, 14 Aug 2026 15:02:27 +0700 Subject: [PATCH] feat: integrate node-html-parser for HTML fetching and parsing - Added node-html-parser as a dependency for parsing HTML content. - Implemented fetch-html script to fetch and parse HTML from a specified URL. - Updated script sandbox to allow the use of node-html-parser. - Created a new workflow for fetching data from dev.to, including configuration for URL and selectors. - Updated package.json to include node-html-parser in server package dependencies. --- packages/server/package.json | 1 + packages/server/script-sandbox.js | 2 +- packages/server/scripts/fetch-html.js | 85 +++++++++++++++++++ .../server/workflows/default/fetch-devto.yaml | 12 +++ .../server/workflows/default/registers.yaml | 3 +- pnpm-lock.yaml | 83 ++++++++++++++++++ 6 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 packages/server/scripts/fetch-html.js create mode 100644 packages/server/workflows/default/fetch-devto.yaml diff --git a/packages/server/package.json b/packages/server/package.json index 1ac21fa..a6a2657 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -21,6 +21,7 @@ "jsonata": "^2.2.2", "knex": "^3.3.0", "node-cron": "^4.6.0", + "node-html-parser": "^9.0.1", "pino": "^10.3.1", "pino-roll": "^4.0.0", "yaml": "^2.9.0" diff --git a/packages/server/script-sandbox.js b/packages/server/script-sandbox.js index 9573bbc..d03b28a 100644 --- a/packages/server/script-sandbox.js +++ b/packages/server/script-sandbox.js @@ -7,7 +7,7 @@ import { SCRIPTS_DIR } from "./paths.js"; const hostRequire = createRequire(import.meta.url); -const ALLOWED_MODULES = new Set(["axios", "jsonata"]); +const ALLOWED_MODULES = new Set(["axios", "jsonata", "node-html-parser"]); const BUILTIN_NAMES = [ "Infinity", diff --git a/packages/server/scripts/fetch-html.js b/packages/server/scripts/fetch-html.js new file mode 100644 index 0000000..4acf8b8 --- /dev/null +++ b/packages/server/scripts/fetch-html.js @@ -0,0 +1,85 @@ +import { parse } from "node-html-parser"; +import jsonata from "jsonata"; + +function ensureDataObject(ctx) { + if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) { + ctx.data = {}; + } +} + +function serializeElement(el) { + const attrs = el.attributes ?? {}; + const classNames = attrs.class ? attrs.class.split(/\s+/).filter(Boolean) : []; + return { + tagName: el.tagName?.toLowerCase() ?? null, + id: attrs.id ?? null, + classNames, + attributes: { ...attrs }, + text: el.text?.trim() ?? "", + innerHTML: el.innerHTML ?? "", + outerHTML: el.outerHTML ?? "", + }; +} + +function previewValue(value) { + const json = JSON.stringify(value); + if (json.length <= 500) { + return value; + } + return { preview: `${json.slice(0, 500)}...`, truncated: true }; +} + +export default async function fetchHtml(ctx) { + const url = ctx.config?.url; + if (typeof url !== "string" || url.length === 0) { + throw new Error("fetch-html: ctx.config.url is required"); + } + + const selector = ctx.config?.selector; + const outputVar = ctx.config?.outputVar; + const jsonataExpr = ctx.config?.jsonata; + + if ((selector || jsonataExpr) && (typeof outputVar !== "string" || outputVar.length === 0)) { + throw new Error("fetch-html: ctx.config.outputVar is required when selector or jsonata is set"); + } + + ensureDataObject(ctx); + + log.info({ url }, "fetch-html: fetching page"); + const response = await $axios.get(url); + log.info( + { status: response.status, length: String(response.data ?? "").length }, + "fetch-html: fetch complete", + ); + + ctx.data.httpResponse = String(response.data ?? ""); + log.info( + { length: ctx.data.httpResponse.length }, + "fetch-html: saved httpResponse", + ); + + if (selector) { + log.info({ selector, outputVar }, "fetch-html: selecting elements"); + const root = parse(ctx.data.httpResponse); + const elements = root.querySelectorAll(selector); + ctx.data[outputVar] = elements.map(serializeElement); + log.info( + { outputVar, count: ctx.data[outputVar].length }, + "fetch-html: saved selector matches", + ); + } + + if (jsonataExpr) { + log.info({ outputVar, jsonata: jsonataExpr }, "fetch-html: evaluating jsonata"); + const expression = jsonata(jsonataExpr); + const input = ctx.data[outputVar]; + const result = await expression.evaluate(input); + ctx.data[outputVar] = result; + log.info( + { outputVar, result: previewValue(result) }, + "fetch-html: saved jsonata result", + ); + } + + return ctx; +} diff --git a/packages/server/workflows/default/fetch-devto.yaml b/packages/server/workflows/default/fetch-devto.yaml new file mode 100644 index 0000000..cc67579 --- /dev/null +++ b/packages/server/workflows/default/fetch-devto.yaml @@ -0,0 +1,12 @@ +name: fetch-devto +scripts: + - script: fetch-html.js + config: + url: https://dev.to/t/productivity/top/week + selector: "#substories h2" + outputVar: titles + jsonata: "$[].text" +triggers: + - type: HTTP + method: POST + path: /fetch-dev-to diff --git a/packages/server/workflows/default/registers.yaml b/packages/server/workflows/default/registers.yaml index f1ed72a..192c92e 100644 --- a/packages/server/workflows/default/registers.yaml +++ b/packages/server/workflows/default/registers.yaml @@ -2,4 +2,5 @@ scripts: - time-to-ntfy-example.yaml - test.yaml - manual-trigger.yaml - - cron-example.yaml \ No newline at end of file + - cron-example.yaml + - fetch-devto.yaml diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1564770..2d17605 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,6 +43,9 @@ importers: node-cron: specifier: ^4.6.0 version: 4.6.0 + node-html-parser: + specifier: ^9.0.1 + version: 9.0.1 pino: specifier: ^10.3.1 version: 10.3.1 @@ -840,6 +843,9 @@ packages: bn.js@4.12.5: resolution: {integrity: sha512-3aRg6/JxfffFD+OlOjOFR3Vo79l39ooBTFucxx+MT3dhCtzn3EmiUPQo+6/OZuI2jbXi3YKgmiTFBgChQMwIRQ==} + boolbase@1.0.0: + resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + brace-expansion@5.0.9: resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==} engines: {node: 20 || >=22} @@ -900,6 +906,13 @@ packages: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} + css-select@5.2.2: + resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==} + + css-what@6.2.2: + resolution: {integrity: sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==} + engines: {node: '>= 6'} + cytoscape-cose-bilkent@4.1.0: resolution: {integrity: sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==} peerDependencies: @@ -1102,12 +1115,25 @@ packages: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + + domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + + domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + dompurify@3.4.13: resolution: {integrity: sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==} dompurify@3.4.8: resolution: {integrity: sha512-yb1cEmaOum7wFvOCSQxyfgVlv5D47Rc30iZWoMpbDIWTnJ6grDDQyu2KFJzB2k7u0pMuJcQ1zphH//fFnw2tjQ==} + domutils@3.2.2: + resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==} + dunder-proto@1.0.1: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} @@ -1122,6 +1148,14 @@ packages: resolution: {integrity: sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==} engines: {node: '>=10.13.0'} + entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + + entities@8.0.0: + resolution: {integrity: sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==} + engines: {node: '>=20.19.0'} + es-define-property@1.0.1: resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} engines: {node: '>= 0.4'} @@ -1559,10 +1593,16 @@ packages: resolution: {integrity: sha512-Si/bzYiKRHOB8/a99T2+SDGN582ONDMSTlJr5oCkT6GtnqPjZ2s10eoQRYkW9ZHwjVxONL+W8Fb+qR0AHMQsdg==} engines: {node: '>=20'} + node-html-parser@9.0.1: + resolution: {integrity: sha512-QrdiYYm1NnLRXsMXThUgVcF/syWfWIgHFmy8hylWMGFbHFtnRuXLBxxGQvIm3xaIJMUDJ0ayOmT/FGJYT3pZIw==} + node-releases@2.0.53: resolution: {integrity: sha512-D9UOmYG3UH1V+ENW56t5QXBwJw1YEY18ruVeus89Rw+SyIgjPkCO84bRzO3uNIYosJbNwiabWVn48o3uJLjxFQ==} engines: {node: '>=18'} + nth-check@2.1.1: + resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + obliterator@2.0.5: resolution: {integrity: sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==} @@ -2584,6 +2624,8 @@ snapshots: bn.js@4.12.5: {} + boolbase@1.0.0: {} + brace-expansion@5.0.9: dependencies: balanced-match: 4.0.4 @@ -2639,6 +2681,16 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + css-select@5.2.2: + dependencies: + boolbase: 1.0.0 + css-what: 6.2.2 + domhandler: 5.0.3 + domutils: 3.2.2 + nth-check: 2.1.1 + + css-what@6.2.2: {} + cytoscape-cose-bilkent@4.1.0(cytoscape@3.34.1): dependencies: cose-base: 1.0.3 @@ -2849,6 +2901,18 @@ snapshots: detect-libc@2.1.2: {} + dom-serializer@2.0.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + + domelementtype@2.3.0: {} + + domhandler@5.0.3: + dependencies: + domelementtype: 2.3.0 + dompurify@3.4.13: optionalDependencies: '@types/trusted-types': 2.0.7 @@ -2857,6 +2921,12 @@ snapshots: optionalDependencies: '@types/trusted-types': 2.0.7 + domutils@3.2.2: + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + dunder-proto@1.0.1: dependencies: call-bind-apply-helpers: 1.0.2 @@ -2874,6 +2944,10 @@ snapshots: graceful-fs: 4.2.11 tapable: 2.3.3 + entities@4.5.0: {} + + entities@8.0.0: {} + es-define-property@1.0.1: {} es-errors@1.3.0: {} @@ -3295,8 +3369,17 @@ snapshots: node-cron@4.6.0: {} + node-html-parser@9.0.1: + dependencies: + css-select: 5.2.2 + entities: 8.0.0 + node-releases@2.0.53: {} + nth-check@2.1.1: + dependencies: + boolbase: 1.0.0 + obliterator@2.0.5: {} on-exit-leak-free@2.1.2: {}