From 9100b1d78be32f037ed95b48f79f3fe628c2cbc6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 15 Aug 2026 00:37:52 +0000 Subject: [PATCH] feat(scripts): add detect-url-changes change-detection step Co-authored-by: Nasyarobby Putra --- packages/server/scripts/detect-url-changes.js | 218 ++++++++++++++++++ .../server/test/detect-url-changes-smoke.js | 161 +++++++++++++ .../default/detect-example-changes.yaml | 18 ++ 3 files changed, 397 insertions(+) create mode 100644 packages/server/scripts/detect-url-changes.js create mode 100644 packages/server/test/detect-url-changes-smoke.js create mode 100644 packages/server/workflows/default/detect-example-changes.yaml diff --git a/packages/server/scripts/detect-url-changes.js b/packages/server/scripts/detect-url-changes.js new file mode 100644 index 0000000..a6e2059 --- /dev/null +++ b/packages/server/scripts/detect-url-changes.js @@ -0,0 +1,218 @@ +import jsonata from "jsonata"; + +function ensureDataObject(ctx) { + if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) { + ctx.data = {}; + } +} + +const ALLOWED_METHODS = new Set([ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE", + "HEAD", + "OPTIONS", +]); + +function resolveMethod(raw) { + if (raw == null || raw === "") return "GET"; + if (typeof raw !== "string") { + throw new Error("detect-url-changes: ctx.config.method must be a string"); + } + const method = raw.trim().toUpperCase(); + if (!ALLOWED_METHODS.has(method)) { + throw new Error(`detect-url-changes: unsupported method "${raw}"`); + } + return method; +} + +function previewValue(value) { + const json = JSON.stringify(value); + if (json == null) return value; + if (json.length <= 500) return value; + return { preview: `${json.slice(0, 500)}...`, truncated: true }; +} + +async function detectUrlChanges(ctx) { + const url = ctx.config?.url; + if (typeof url !== "string" || url.length === 0) { + throw new Error("detect-url-changes: ctx.config.url is required"); + } + + const method = resolveMethod(ctx.config?.method); + const headers = ctx.config?.headers; + const body = ctx.config?.body; + const key = + typeof ctx.config?.key === "string" && ctx.config.key.length > 0 + ? ctx.config.key + : url; + const fingerprintExpr = ctx.config?.fingerprint; + const outputVar = ctx.config?.outputVar; + const transformExpr = ctx.config?.transform; + const skipRemainingWhenUnchanged = ctx.config?.skipRemaining === true; + + if ( + typeof transformExpr === "string" && + transformExpr.length > 0 && + (typeof outputVar !== "string" || outputVar.length === 0) + ) { + throw new Error( + "detect-url-changes: ctx.config.outputVar is required when transform is set", + ); + } + + ensureDataObject(ctx); + + log.info({ url, method, key }, "detect-url-changes: fetching url"); + const response = await $axios.request({ + method, + url, + ...(headers && typeof headers === "object" ? { headers } : {}), + ...(body !== undefined ? { data: body } : {}), + }); + log.info( + { status: response.status }, + "detect-url-changes: fetch complete", + ); + + ctx.data.httpResponse = response.data; + + let fingerprintSource = ctx.data.httpResponse; + if (typeof fingerprintExpr === "string" && fingerprintExpr.length > 0) { + log.info( + { jsonata: fingerprintExpr }, + "detect-url-changes: evaluating fingerprint jsonata", + ); + fingerprintSource = await jsonata(fingerprintExpr).evaluate(ctx); + } + + const result = await $fingerprint.claim(key, fingerprintSource, { + maxAge: ctx.config?.maxAge, + }); + + ctx.data.hasChanges = result.changed; + ctx.data.fingerprint = result.hash; + ctx.data.fingerprintChanged = result.changed; + ctx.data.fingerprintPrevious = result.previous; + ctx.data.fingerprintAt = result.changed ? result.at : result.previousAt; + ctx.data.fingerprintAge = result.ageMs; + ctx.data.fingerprintExpired = result.expired; + + log.info( + { + key, + hasChanges: result.changed, + expired: result.expired, + ageMs: result.ageMs, + }, + "detect-url-changes: result", + ); + + if (typeof outputVar === "string" && outputVar.length > 0) { + if (typeof transformExpr === "string" && transformExpr.length > 0) { + log.info( + { outputVar, jsonata: transformExpr }, + "detect-url-changes: evaluating transform jsonata", + ); + const transformed = await jsonata(transformExpr).evaluate(ctx); + ctx.data[outputVar] = transformed; + log.info( + { outputVar, value: previewValue(transformed) }, + "detect-url-changes: saved transform result", + ); + } else { + ctx.data[outputVar] = ctx.data.httpResponse; + log.info({ outputVar }, "detect-url-changes: saved raw response to outputVar"); + } + } + + if (skipRemainingWhenUnchanged && !result.changed) { + ctx.skipRemaining = true; + } + + return ctx; +} + +detectUrlChanges.meta = { + description: + "Fetch a URL, fingerprint the response (or a JSONata-derived value), and report whether it changed since the last run", + config: { + url: { type: "string", required: true, description: "URL to fetch" }, + method: { + type: "string", + default: "GET", + description: "HTTP method (GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS)", + }, + headers: { + type: "object", + required: false, + description: "Optional request headers", + }, + body: { + type: "any", + required: false, + description: "Optional request body for non-GET methods", + }, + key: { + type: "string", + required: false, + description: "KV key for the stored fingerprint. Defaults to the URL", + }, + fingerprint: { + type: "string", + required: false, + description: + "JSONata against ctx used as the fingerprint source. Omitted fingerprints the raw response", + }, + outputVar: { + type: "string", + required: false, + description: "ctx.data key to store output. Required when transform is set", + }, + transform: { + type: "string", + required: false, + description: + "JSONata against ctx stored at outputVar. Omitted stores the raw response", + }, + maxAge: { + type: "string", + required: false, + description: + "Optional age limit (e.g. 24h, 7d, or milliseconds). Older matching fingerprints report a change again", + }, + skipRemaining: { + type: "boolean", + default: false, + description: "Skip later steps when the URL has not changed", + }, + }, + input: {}, + output: { + hasChanges: { + type: "boolean", + description: "True when the URL changed since the last run", + }, + httpResponse: { type: "any", description: "Raw response body" }, + fingerprint: { type: "string", description: "SHA-256 hex of the source value" }, + fingerprintChanged: { type: "boolean" }, + fingerprintPrevious: { type: "string", required: false }, + fingerprintAt: { type: "string", required: false, description: "ISO timestamp of the stored record" }, + fingerprintAge: { type: "number", required: false, description: "Age in milliseconds" }, + fingerprintExpired: { type: "boolean" }, + }, + example: { + data: {}, + config: { + url: "https://example.com/", + fingerprint: "data.httpResponse", + outputVar: "message", + transform: '"example.com changed at " & $now()', + skipRemaining: true, + }, + }, +}; + +export default detectUrlChanges; diff --git a/packages/server/test/detect-url-changes-smoke.js b/packages/server/test/detect-url-changes-smoke.js new file mode 100644 index 0000000..1493458 --- /dev/null +++ b/packages/server/test/detect-url-changes-smoke.js @@ -0,0 +1,161 @@ +import http from "node:http"; +import { migrate, db } from "../db.js"; +import { kvDelete } from "../kv-store.js"; +import { runScript } from "../script-sandbox.js"; +import { log } from "../logger.js"; + +await migrate(); + +const ns = "test/detect-url-changes-smoke"; + +function assert(cond, message) { + if (!cond) throw new Error(message); +} + +const state = { + status: 200, + contentType: "text/html; charset=utf-8", + body: "v1", + lastMethod: null, +}; + +const server = http.createServer((req, res) => { + state.lastMethod = req.method; + res.writeHead(state.status, { "content-type": state.contentType }); + res.end(typeof state.body === "string" ? state.body : JSON.stringify(state.body)); +}); + +await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); +const { port } = server.address(); +const base = `http://127.0.0.1:${port}`; + +function run(config) { + return runScript( + "detect-url-changes.js", + { data: {}, config }, + { log, workflowName: ns, owner: "default" }, + ); +} + +const cleanupKeys = []; +async function freshUrl(pathname) { + const url = `${base}${pathname}`; + await kvDelete(ns, url); + cleanupKeys.push(url); + return url; +} + +// 1. Raw-response change detection across runs. +{ + const url = await freshUrl("/raw"); + state.body = "v1"; + + const first = await run({ url }); + assert(first.data.hasChanges === true, "first run should report hasChanges=true"); + assert( + first.data.httpResponse === "v1", + "httpResponse should hold the raw body", + ); + assert(typeof first.data.fingerprint === "string", "fingerprint hash should be set"); + + const second = await run({ url }); + assert(second.data.hasChanges === false, "unchanged body should report hasChanges=false"); + + state.body = "v2 CHANGED"; + const third = await run({ url }); + assert(third.data.hasChanges === true, "changed body should report hasChanges=true"); + assert( + third.data.fingerprintPrevious === second.data.fingerprint, + "fingerprintPrevious should equal the prior hash", + ); +} + +// 2. fingerprint JSONata isolates the watched field; unrelated changes are ignored. +{ + const url = await freshUrl("/json"); + state.contentType = "application/json"; + state.body = { version: "1.0.0", servedAt: "2020-01-01T00:00:00Z" }; + + const first = await run({ url, fingerprint: "data.httpResponse.version" }); + assert(first.data.hasChanges === true, "json first run should report a change"); + + // Change only an unwatched field -> no change. + state.body = { version: "1.0.0", servedAt: "2020-06-01T00:00:00Z" }; + const second = await run({ url, fingerprint: "data.httpResponse.version" }); + assert( + second.data.hasChanges === false, + "changing an unwatched field should not report a change", + ); + + // Change the watched field -> change. + state.body = { version: "2.0.0", servedAt: "2020-06-01T00:00:00Z" }; + const third = await run({ url, fingerprint: "data.httpResponse.version" }); + assert(third.data.hasChanges === true, "changing the watched field should report a change"); + + state.contentType = "text/html; charset=utf-8"; +} + +// 3. transform + outputVar, and raw output when transform is omitted. +{ + const url = await freshUrl("/output"); + state.body = "hello"; + + const withTransform = await run({ + url, + outputVar: "message", + transform: '"changed=" & $string(data.hasChanges)', + }); + assert( + withTransform.data.message === "changed=true", + `transform should populate outputVar, got ${JSON.stringify(withTransform.data.message)}`, + ); + + const rawOutput = await run({ url: await freshUrl("/output-raw"), outputVar: "payload" }); + assert( + rawOutput.data.payload === rawOutput.data.httpResponse, + "outputVar without transform should store the raw response", + ); + + let threw = false; + try { + await run({ url, transform: "1" }); + } catch { + threw = true; + } + assert(threw, "transform without outputVar should throw"); +} + +// 4. optional skipRemaining only halts when unchanged. +{ + const url = await freshUrl("/skip"); + state.body = "stable"; + + const first = await run({ url, skipRemaining: true }); + assert(first.data.hasChanges === true, "skip test first run should change"); + assert(first.skipRemaining !== true, "changed run must not set skipRemaining"); + + const second = await run({ url, skipRemaining: true }); + assert(second.data.hasChanges === false, "skip test second run should be unchanged"); + assert(second.skipRemaining === true, "unchanged run with skipRemaining should halt"); + + // Default (skipRemaining off) never halts, so downstream can still notify. + const third = await run({ url }); + assert(third.skipRemaining !== true, "default should not set skipRemaining when unchanged"); +} + +// 5. non-GET methods are supported. +{ + const url = await freshUrl("/method"); + state.body = "ok"; + await run({ url, method: "POST", body: { ping: true } }); + assert(state.lastMethod === "POST", `server should have received POST, got ${state.lastMethod}`); +} + +for (const key of cleanupKeys) { + await kvDelete(ns, key); +} + +await new Promise((resolve) => server.close(resolve)); +await db.destroy(); +console.log("detect-url-changes smoke test passed"); +process.exit(0); diff --git a/packages/server/workflows/default/detect-example-changes.yaml b/packages/server/workflows/default/detect-example-changes.yaml new file mode 100644 index 0000000..dfaee2b --- /dev/null +++ b/packages/server/workflows/default/detect-example-changes.yaml @@ -0,0 +1,18 @@ +name: detect example.com changes +description: | + Fetch example.com, fingerprint the response, and report whether it changed + since the previous run. Uses detect-url-changes with a transform that builds + a human-readable message into ctx.data.message. +scripts: + - script: detect-url-changes.js + config: + url: https://example.com/ + outputVar: message + transform: > + data.hasChanges + ? "example.com changed (fingerprint " & data.fingerprint & ")" + : "example.com unchanged since " & data.fingerprintAt +triggers: + - type: HTTP + method: POST + path: /detect-example