feat(server): implement KV store API and fingerprinting functionality

- Added a new KV store API with endpoints for querying namespaces and key-value pairs. - Introduced fingerprinting functionality to hash and manage data fingerprints, allowing for efficient change detection. - Enhanced existing scripts to utilize the new fingerprinting capabilities, enabling conditional execution based on data changes. - Updated the web interface to include a dedicated KV page for managing key-value entries and displaying their details. - Improved overall user experience with pagination and search capabilities in the KV management interface.
2026-08-15 06:00:30 +07:00
parent c5015bb391
commit 951e6f8371
15 changed files with 823 additions and 18 deletions
@@ -0,0 +1,99 @@
+import jsonata from "jsonata";
+
+function ensureDataObject(ctx) {
+  if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) {
+    ctx.data = {};
+  }
+}
+
+async function fingerprint(ctx) {
+  const key =
+    typeof ctx.config?.key === "string" && ctx.config.key.length > 0
+      ? ctx.config.key
+      : "fingerprint";
+  const jsonataExpr = ctx.config?.jsonata;
+  const skipRemaining = ctx.config?.skipRemaining !== false;
+
+  let source;
+  if (typeof jsonataExpr === "string" && jsonataExpr.length > 0) {
+    log.info({ jsonata: jsonataExpr }, "fingerprint: evaluating jsonata");
+    const expression = jsonata(jsonataExpr);
+    source = await expression.evaluate(ctx);
+  } else {
+    source = ctx.data;
+  }
+
+  log.info({ key }, "fingerprint: claiming");
+  const result = await $fingerprint.claim(key, source, {
+    maxAge: ctx.config?.maxAge,
+  });
+
+  ensureDataObject(ctx);
+  ctx.data.fingerprint = result.hash;
+  ctx.data.fingerprintChanged = result.changed;
+  ctx.data.fingerprintPrevious = result.previous;
+  ctx.data.fingerprintAt = result.changed ? result.at : result.previousAt;
+  ctx.data.fingerprintAge = result.ageMs;
+  ctx.data.fingerprintExpired = result.expired;
+
+  log.info(
+    {
+      key,
+      changed: result.changed,
+      expired: result.expired,
+      ageMs: result.ageMs,
+    },
+    "fingerprint: result",
+  );
+
+  if (!result.changed && skipRemaining) {
+    ctx.skipRemaining = true;
+  }
+
+  return ctx;
+}
+
+fingerprint.meta = {
+  description:
+    "Hash a value, compare it to the last stored fingerprint, and skip remaining steps when unchanged",
+  config: {
+    key: {
+      type: "string",
+      default: "fingerprint",
+      description: "KV key for the stored fingerprint record",
+    },
+    jsonata: {
+      type: "string",
+      required: false,
+      description: "JSONata against ctx; omitted hashes ctx.data",
+    },
+    skipRemaining: {
+      type: "boolean",
+      default: true,
+      description: "Skip later steps when the fingerprint is unchanged",
+    },
+    maxAge: {
+      type: "string",
+      required: false,
+      description: "Optional age limit (e.g. 24h, 7d, or milliseconds). Older matching hashes fire again",
+    },
+  },
+  input: {},
+  output: {
+    fingerprint: { type: "string", description: "SHA-256 hex of the source value" },
+    fingerprintChanged: { type: "boolean" },
+    fingerprintPrevious: { type: "string", required: false },
+    fingerprintAt: { type: "string", required: false, description: "ISO timestamp of the stored record" },
+    fingerprintAge: { type: "number", required: false, description: "Age in milliseconds" },
+    fingerprintExpired: { type: "boolean" },
+  },
+  example: {
+    data: { item: { guid: "https://example.com/post-1" } },
+    config: {
+      key: "latest-item",
+      jsonata: "data.item.guid ? data.item.guid : data.item.link",
+    },
+  },
+};
+
+export default fingerprint;
@@ -1,3 +1,5 @@
+import jsonata from "jsonata";
+
 function ntfyHeaders(ctx) {
  const headers = {};

@@ -9,6 +11,32 @@ function ntfyHeaders(ctx) {
  return headers;
 }

+function resolveFingerprintKey(config) {
+  const fingerprint = config?.fingerprint;
+  if (fingerprint === true) return "fingerprint:ntfy";
+  if (typeof fingerprint === "string" && fingerprint.length > 0) return fingerprint;
+  return null;
+}
+
+function defaultFingerprintSource(ctx) {
+  return {
+    title: ctx.data?.title,
+    message: ctx.data?.message,
+    attach: ctx.data?.attach,
+    filename: ctx.data?.filename,
+    contentType: ctx.data?.contentType,
+  };
+}
+
+async function resolveFingerprintValue(ctx) {
+  const expr = ctx.config?.fingerprintJsonata;
+  if (typeof expr === "string" && expr.length > 0) {
+    const expression = jsonata(expr);
+    return await expression.evaluate(ctx);
+  }
+  return defaultFingerprintSource(ctx);
+}
+
 async function ntfy(ctx) {
  const file = ctx.data?.file;
  const hasFile = Buffer.isBuffer(file) || file instanceof Uint8Array;
@@ -25,6 +53,35 @@ async function ntfy(ctx) {
    "ntfy incoming context",
  );

+  const fingerprintKey = resolveFingerprintKey(ctx.config);
+  /** @type {{ hash: string, previousAt: string | null, ageMs: number | null } | null} */
+  let fp = null;
+
+  if (fingerprintKey) {
+    const source = await resolveFingerprintValue(ctx);
+    const checked = await $fingerprint.check(fingerprintKey, source, {
+      maxAge: ctx.config?.fingerprintMaxAge,
+    });
+    if (!checked.changed) {
+      log.info(
+        {
+          key: fingerprintKey,
+          fingerprint: checked.hash,
+          ageMs: checked.ageMs,
+        },
+        "ntfy: skipped, fingerprint unchanged",
+      );
+      return {
+        sent: "false",
+        skipped: true,
+        fingerprint: checked.hash,
+        fingerprintAt: checked.previousAt,
+        fingerprintAge: checked.ageMs,
+      };
+    }
+    fp = checked;
+  }
+
  const headers = ntfyHeaders(ctx);
  const ntfyUrl = ctx.config?.url || "https://ntfy.sh/scrunner";

@@ -47,22 +104,29 @@ async function ntfy(ctx) {
      maxBodyLength: Infinity,
      maxContentLength: Infinity,
    });
-    return { sent: "true" };
+  } else {
+    if (ctx.data?.attach) {
+      log.info("ntfy: setting attach %s", ctx.data.attach);
+      headers.Attach = ctx.data.attach;
+    }
+
+    const truncatedMessage = ctx.data?.message?.substring(0, 100);
+    log.info("ntfy sending message to %s", ntfyUrl);
+    log.info("ntfy messsage: %s", truncatedMessage);
+
+    await $axios.post(ntfyUrl, ctx.data?.message || "Hello from scrunner", {
+      headers,
+    });
  }

-  if (ctx.data?.attach) {
-    log.info("ntfy: setting attach %s", ctx.data.attach);
-    headers.Attach = ctx.data.attach;
+  /** @type {Record<string, unknown>} */
+  const sent = { sent: "true" };
+  if (fingerprintKey && fp) {
+    const stored = await $fingerprint.remember(fingerprintKey, fp.hash);
+    sent.fingerprint = stored.hash;
+    sent.fingerprintAt = stored.at;
  }
-
-  const truncatedMessage = ctx.data?.message?.substring(0, 100);
-  log.info("ntfy sending message to %s", ntfyUrl);
-  log.info("ntfy messsage: %s", truncatedMessage);
-
-  await $axios.post(ntfyUrl, ctx.data?.message || "Hello from scrunner", {
-    headers,
-  });
-  return { sent: "true" };
+  return sent;
 }

 ntfy.meta = {
@@ -73,6 +137,21 @@ ntfy.meta = {
      default: "https://ntfy.sh/scrunner",
      description: "ntfy topic URL",
    },
+    fingerprint: {
+      type: "string",
+      required: false,
+      description: "true or a KV key; skip send when the payload fingerprint is unchanged",
+    },
+    fingerprintJsonata: {
+      type: "string",
+      required: false,
+      description: "JSONata against ctx; default hashes title, message, attach, filename, contentType",
+    },
+    fingerprintMaxAge: {
+      type: "string",
+      required: false,
+      description: "Optional age limit (e.g. 24h, 7d, or milliseconds)",
+    },
  },
  input: {
    title: { type: "string", required: false },
@@ -87,7 +166,7 @@ ntfy.meta = {
  },
  example: {
    data: { title: "Hello", message: "Hello from scrunner" },
-    config: { url: "https://ntfy.sh/scrunner" },
+    config: { url: "https://ntfy.sh/scrunner", fingerprint: true },
  },
 };