feat(scripts): add detect-url-changes change-detection step

Co-authored-by: Nasyarobby Putra <nasyarobby@gmail.com>

packages/server/scripts/detect-url-changes.js

@@ -0,0 +1,218 @@
+import jsonata from "jsonata";
+
+function ensureDataObject(ctx) {
+  if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) {
+    ctx.data = {};
+  }
+}
+
+const ALLOWED_METHODS = new Set([
+  "GET",
+  "POST",
+  "PUT",
+  "PATCH",
+  "DELETE",
+  "HEAD",
+  "OPTIONS",
+]);
+
+function resolveMethod(raw) {
+  if (raw == null || raw === "") return "GET";
+  if (typeof raw !== "string") {
+    throw new Error("detect-url-changes: ctx.config.method must be a string");
+  }
+  const method = raw.trim().toUpperCase();
+  if (!ALLOWED_METHODS.has(method)) {
+    throw new Error(`detect-url-changes: unsupported method "${raw}"`);
+  }
+  return method;
+}
+
+function previewValue(value) {
+  const json = JSON.stringify(value);
+  if (json == null) return value;
+  if (json.length <= 500) return value;
+  return { preview: `${json.slice(0, 500)}...`, truncated: true };
+}
+
+async function detectUrlChanges(ctx) {
+  const url = ctx.config?.url;
+  if (typeof url !== "string" || url.length === 0) {
+    throw new Error("detect-url-changes: ctx.config.url is required");
+  }
+
+  const method = resolveMethod(ctx.config?.method);
+  const headers = ctx.config?.headers;
+  const body = ctx.config?.body;
+  const key =
+    typeof ctx.config?.key === "string" && ctx.config.key.length > 0
+      ? ctx.config.key
+      : url;
+  const fingerprintExpr = ctx.config?.fingerprint;
+  const outputVar = ctx.config?.outputVar;
+  const transformExpr = ctx.config?.transform;
+  const skipRemainingWhenUnchanged = ctx.config?.skipRemaining === true;
+
+  if (
+    typeof transformExpr === "string" &&
+    transformExpr.length > 0 &&
+    (typeof outputVar !== "string" || outputVar.length === 0)
+  ) {
+    throw new Error(
+      "detect-url-changes: ctx.config.outputVar is required when transform is set",
+    );
+  }
+
+  ensureDataObject(ctx);
+
+  log.info({ url, method, key }, "detect-url-changes: fetching url");
+  const response = await $axios.request({
+    method,
+    url,
+    ...(headers && typeof headers === "object" ? { headers } : {}),
+    ...(body !== undefined ? { data: body } : {}),
+  });
+  log.info(
+    { status: response.status },
+    "detect-url-changes: fetch complete",
+  );
+
+  ctx.data.httpResponse = response.data;
+
+  let fingerprintSource = ctx.data.httpResponse;
+  if (typeof fingerprintExpr === "string" && fingerprintExpr.length > 0) {
+    log.info(
+      { jsonata: fingerprintExpr },
+      "detect-url-changes: evaluating fingerprint jsonata",
+    );
+    fingerprintSource = await jsonata(fingerprintExpr).evaluate(ctx);
+  }
+
+  const result = await $fingerprint.claim(key, fingerprintSource, {
+    maxAge: ctx.config?.maxAge,
+  });
+
+  ctx.data.hasChanges = result.changed;
+  ctx.data.fingerprint = result.hash;
+  ctx.data.fingerprintChanged = result.changed;
+  ctx.data.fingerprintPrevious = result.previous;
+  ctx.data.fingerprintAt = result.changed ? result.at : result.previousAt;
+  ctx.data.fingerprintAge = result.ageMs;
+  ctx.data.fingerprintExpired = result.expired;
+
+  log.info(
+    {
+      key,
+      hasChanges: result.changed,
+      expired: result.expired,
+      ageMs: result.ageMs,
+    },
+    "detect-url-changes: result",
+  );
+
+  if (typeof outputVar === "string" && outputVar.length > 0) {
+    if (typeof transformExpr === "string" && transformExpr.length > 0) {
+      log.info(
+        { outputVar, jsonata: transformExpr },
+        "detect-url-changes: evaluating transform jsonata",
+      );
+      const transformed = await jsonata(transformExpr).evaluate(ctx);
+      ctx.data[outputVar] = transformed;
+      log.info(
+        { outputVar, value: previewValue(transformed) },
+        "detect-url-changes: saved transform result",
+      );
+    } else {
+      ctx.data[outputVar] = ctx.data.httpResponse;
+      log.info({ outputVar }, "detect-url-changes: saved raw response to outputVar");
+    }
+  }
+
+  if (skipRemainingWhenUnchanged && !result.changed) {
+    ctx.skipRemaining = true;
+  }
+
+  return ctx;
+}
+
+detectUrlChanges.meta = {
+  description:
+    "Fetch a URL, fingerprint the response (or a JSONata-derived value), and report whether it changed since the last run",
+  config: {
+    url: { type: "string", required: true, description: "URL to fetch" },
+    method: {
+      type: "string",
+      default: "GET",
+      description: "HTTP method (GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS)",
+    },
+    headers: {
+      type: "object",
+      required: false,
+      description: "Optional request headers",
+    },
+    body: {
+      type: "any",
+      required: false,
+      description: "Optional request body for non-GET methods",
+    },
+    key: {
+      type: "string",
+      required: false,
+      description: "KV key for the stored fingerprint. Defaults to the URL",
+    },
+    fingerprint: {
+      type: "string",
+      required: false,
+      description:
+        "JSONata against ctx used as the fingerprint source. Omitted fingerprints the raw response",
+    },
+    outputVar: {
+      type: "string",
+      required: false,
+      description: "ctx.data key to store output. Required when transform is set",
+    },
+    transform: {
+      type: "string",
+      required: false,
+      description:
+        "JSONata against ctx stored at outputVar. Omitted stores the raw response",
+    },
+    maxAge: {
+      type: "string",
+      required: false,
+      description:
+        "Optional age limit (e.g. 24h, 7d, or milliseconds). Older matching fingerprints report a change again",
+    },
+    skipRemaining: {
+      type: "boolean",
+      default: false,
+      description: "Skip later steps when the URL has not changed",
+    },
+  },
+  input: {},
+  output: {
+    hasChanges: {
+      type: "boolean",
+      description: "True when the URL changed since the last run",
+    },
+    httpResponse: { type: "any", description: "Raw response body" },
+    fingerprint: { type: "string", description: "SHA-256 hex of the source value" },
+    fingerprintChanged: { type: "boolean" },
+    fingerprintPrevious: { type: "string", required: false },
+    fingerprintAt: { type: "string", required: false, description: "ISO timestamp of the stored record" },
+    fingerprintAge: { type: "number", required: false, description: "Age in milliseconds" },
+    fingerprintExpired: { type: "boolean" },
+  },
+  example: {
+    data: {},
+    config: {
+      url: "https://example.com/",
+      fingerprint: "data.httpResponse",
+      outputVar: "message",
+      transform: '"example.com changed at " & $now()',
+      skipRemaining: true,
+    },
+  },
+};
+
+export default detectUrlChanges;

packages/server/test/detect-url-changes-smoke.js

@@ -0,0 +1,161 @@
+import http from "node:http";
+import { migrate, db } from "../db.js";
+import { kvDelete } from "../kv-store.js";
+import { runScript } from "../script-sandbox.js";
+import { log } from "../logger.js";
+
+await migrate();
+
+const ns = "test/detect-url-changes-smoke";
+
+function assert(cond, message) {
+  if (!cond) throw new Error(message);
+}
+
+const state = {
+  status: 200,
+  contentType: "text/html; charset=utf-8",
+  body: "<html><body>v1</body></html>",
+  lastMethod: null,
+};
+
+const server = http.createServer((req, res) => {
+  state.lastMethod = req.method;
+  res.writeHead(state.status, { "content-type": state.contentType });
+  res.end(typeof state.body === "string" ? state.body : JSON.stringify(state.body));
+});
+
+await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+const { port } = server.address();
+const base = `http://127.0.0.1:${port}`;
+
+function run(config) {
+  return runScript(
+    "detect-url-changes.js",
+    { data: {}, config },
+    { log, workflowName: ns, owner: "default" },
+  );
+}
+
+const cleanupKeys = [];
+async function freshUrl(pathname) {
+  const url = `${base}${pathname}`;
+  await kvDelete(ns, url);
+  cleanupKeys.push(url);
+  return url;
+}
+
+// 1. Raw-response change detection across runs.
+{
+  const url = await freshUrl("/raw");
+  state.body = "<html><body>v1</body></html>";
+
+  const first = await run({ url });
+  assert(first.data.hasChanges === true, "first run should report hasChanges=true");
+  assert(
+    first.data.httpResponse === "<html><body>v1</body></html>",
+    "httpResponse should hold the raw body",
+  );
+  assert(typeof first.data.fingerprint === "string", "fingerprint hash should be set");
+
+  const second = await run({ url });
+  assert(second.data.hasChanges === false, "unchanged body should report hasChanges=false");
+
+  state.body = "<html><body>v2 CHANGED</body></html>";
+  const third = await run({ url });
+  assert(third.data.hasChanges === true, "changed body should report hasChanges=true");
+  assert(
+    third.data.fingerprintPrevious === second.data.fingerprint,
+    "fingerprintPrevious should equal the prior hash",
+  );
+}
+
+// 2. fingerprint JSONata isolates the watched field; unrelated changes are ignored.
+{
+  const url = await freshUrl("/json");
+  state.contentType = "application/json";
+  state.body = { version: "1.0.0", servedAt: "2020-01-01T00:00:00Z" };
+
+  const first = await run({ url, fingerprint: "data.httpResponse.version" });
+  assert(first.data.hasChanges === true, "json first run should report a change");
+
+  // Change only an unwatched field -> no change.
+  state.body = { version: "1.0.0", servedAt: "2020-06-01T00:00:00Z" };
+  const second = await run({ url, fingerprint: "data.httpResponse.version" });
+  assert(
+    second.data.hasChanges === false,
+    "changing an unwatched field should not report a change",
+  );
+
+  // Change the watched field -> change.
+  state.body = { version: "2.0.0", servedAt: "2020-06-01T00:00:00Z" };
+  const third = await run({ url, fingerprint: "data.httpResponse.version" });
+  assert(third.data.hasChanges === true, "changing the watched field should report a change");
+
+  state.contentType = "text/html; charset=utf-8";
+}
+
+// 3. transform + outputVar, and raw output when transform is omitted.
+{
+  const url = await freshUrl("/output");
+  state.body = "<html><body>hello</body></html>";
+
+  const withTransform = await run({
+    url,
+    outputVar: "message",
+    transform: '"changed=" & $string(data.hasChanges)',
+  });
+  assert(
+    withTransform.data.message === "changed=true",
+    `transform should populate outputVar, got ${JSON.stringify(withTransform.data.message)}`,
+  );
+
+  const rawOutput = await run({ url: await freshUrl("/output-raw"), outputVar: "payload" });
+  assert(
+    rawOutput.data.payload === rawOutput.data.httpResponse,
+    "outputVar without transform should store the raw response",
+  );
+
+  let threw = false;
+  try {
+    await run({ url, transform: "1" });
+  } catch {
+    threw = true;
+  }
+  assert(threw, "transform without outputVar should throw");
+}
+
+// 4. optional skipRemaining only halts when unchanged.
+{
+  const url = await freshUrl("/skip");
+  state.body = "<html><body>stable</body></html>";
+
+  const first = await run({ url, skipRemaining: true });
+  assert(first.data.hasChanges === true, "skip test first run should change");
+  assert(first.skipRemaining !== true, "changed run must not set skipRemaining");
+
+  const second = await run({ url, skipRemaining: true });
+  assert(second.data.hasChanges === false, "skip test second run should be unchanged");
+  assert(second.skipRemaining === true, "unchanged run with skipRemaining should halt");
+
+  // Default (skipRemaining off) never halts, so downstream can still notify.
+  const third = await run({ url });
+  assert(third.skipRemaining !== true, "default should not set skipRemaining when unchanged");
+}
+
+// 5. non-GET methods are supported.
+{
+  const url = await freshUrl("/method");
+  state.body = "ok";
+  await run({ url, method: "POST", body: { ping: true } });
+  assert(state.lastMethod === "POST", `server should have received POST, got ${state.lastMethod}`);
+}
+
+for (const key of cleanupKeys) {
+  await kvDelete(ns, key);
+}
+
+await new Promise((resolve) => server.close(resolve));
+await db.destroy();
+console.log("detect-url-changes smoke test passed");
+process.exit(0);

packages/server/workflows/default/detect-example-changes.yaml

@@ -0,0 +1,18 @@
+name: detect example.com changes
+description: |
+  Fetch example.com, fingerprint the response, and report whether it changed
+  since the previous run. Uses detect-url-changes with a transform that builds
+  a human-readable message into ctx.data.message.
+scripts:
+  - script: detect-url-changes.js
+    config:
+      url: https://example.com/
+      outputVar: message
+      transform: >
+        data.hasChanges
+          ? "example.com changed (fingerprint " & data.fingerprint & ")"
+          : "example.com unchanged since " & data.fingerprintAt
+triggers:
+  - type: HTTP
+    method: POST
+    path: /detect-example