feat(scripts): enhance script return structure and context handling

- Introduced a new contract for script returns, requiring an object with `output`, `context`, and `skipRemaining` fields.
- Refactored existing scripts to align with the new return structure, ensuring compatibility with the updated context management.
- Added utility functions for normalizing context and step results, improving the handling of script execution context.
- Updated various scripts (e.g., fetch-binary, fetch-html, fetch-http) to return structured output and context, enhancing data flow and usability.
- Improved error handling for script returns, ensuring clearer feedback when invalid structures are returned.

This update significantly enhances the flexibility and clarity of script interactions within workflows.

packages/server/scripts/fetch-binary.js

@@ -1,7 +1,15 @@
-function ensureDataObject(ctx) {
-  if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) {
-    ctx.data = {};
+function passContext(ctx) {
+  if (ctx?.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)) {
+    return { ...ctx.context };
   }
+  return {};
+}
+
+function mergeData(data) {
+  if (data != null && typeof data === "object" && !Array.isArray(data)) {
+    return { ...data };
+  }
+  return {};
 }
 
 function filenameFromUrl(url) {
@@ -38,8 +46,6 @@ function resolveUrl(ctx) {
 }
 
 async function fetchBinary(ctx) {
-  ensureDataObject(ctx);
-
   const url = resolveUrl(ctx);
   if (!url) {
     throw new Error(
@@ -55,27 +61,32 @@ async function fetchBinary(ctx) {
   const response = await $axios.get(url, { responseType: "arraybuffer" });
   const file = Buffer.from(response.data ?? []);
   const contentType = String(response.headers?.["content-type"] ?? "application/octet-stream");
-  const filename = ctx.config?.filename || ctx.data.filename || filenameFromUrl(url);
+  const filename = ctx.config?.filename || ctx.data?.filename || filenameFromUrl(url);
 
-  ctx.data[outputVar] = file;
-  ctx.data.filename = filename;
-  ctx.data.contentType = contentType;
+  const extra = {
+    [outputVar]: file,
+    filename,
+    contentType,
+  };
 
   log.info(
     { outputVar, filename, contentType, length: file.length },
     "fetch-binary: saved",
   );
 
-  return ctx;
+  return {
+    output: { ...mergeData(ctx.data), ...extra },
+    context: { ...passContext(ctx), ...extra },
+  };
 }
 
 fetchBinary.meta = {
-  description: "Download a binary URL into ctx.data as a Buffer",
+  description: "Download a binary URL and add the Buffer to output (keeps previous data fields)",
   previewConfigKey: "url",
   config: {
     url: { type: "string", required: false, description: "Direct download URL" },
     urlVar: { type: "string", required: false, description: "Key in ctx.data that holds the URL" },
-    outputVar: { type: "string", default: "file", description: "ctx.data key for the Buffer" },
+    outputVar: { type: "string", default: "file", description: "output key for the Buffer" },
     filename: { type: "string", required: false, description: "Override saved filename" },
   },
   input: {
@@ -88,6 +99,11 @@ fetchBinary.meta = {
     filename: { type: "string" },
     contentType: { type: "string" },
   },
+  context: {
+    file: { type: "buffer", description: "Downloaded bytes (or ctx.config.outputVar)" },
+    filename: { type: "string" },
+    contentType: { type: "string" },
+  },
   example: {
     data: { attach: "https://example.com/image.png" },
     config: { outputVar: "file" },

packages/server/scripts/fetch-html.js

@@ -1,10 +1,11 @@
 import { parse } from "node-html-parser";
 import jsonata from "jsonata";
 
-function ensureDataObject(ctx) {
-  if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) {
-    ctx.data = {};
+function passContext(ctx) {
+  if (ctx?.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)) {
+    return { ...ctx.context };
   }
+  return {};
 }
 
 function serializeElement(el) {
@@ -43,8 +44,6 @@ async function fetchHtml(ctx) {
     throw new Error("fetch-html: ctx.config.outputVar is required when selector or jsonata is set");
   }
 
-  ensureDataObject(ctx);
-
   log.info({ url }, "fetch-html: fetching page");
   const response = await $axios.get(url);
   log.info(
@@ -52,19 +51,20 @@ async function fetchHtml(ctx) {
     "fetch-html: fetch complete",
   );
 
-  ctx.data.httpResponse = String(response.data ?? "");
+  /** @type {Record<string, unknown>} */
+  const output = { httpResponse: String(response.data ?? "") };
   log.info(
-    { length: ctx.data.httpResponse.length },
+    { length: String(output.httpResponse).length },
     "fetch-html: saved httpResponse",
   );
 
   if (selector) {
     log.info({ selector, outputVar }, "fetch-html: selecting elements");
-    const root = parse(ctx.data.httpResponse);
+    const root = parse(String(output.httpResponse));
     const elements = root.querySelectorAll(selector);
-    ctx.data[outputVar] = elements.map(serializeElement);
+    output[outputVar] = elements.map(serializeElement);
     log.info(
-      { outputVar, count: ctx.data[outputVar].length },
+      { outputVar, count: output[outputVar].length },
       "fetch-html: saved selector matches",
     );
   }
@@ -72,16 +72,15 @@ async function fetchHtml(ctx) {
   if (jsonataExpr) {
     log.info({ outputVar, jsonata: jsonataExpr }, "fetch-html: evaluating jsonata");
     const expression = jsonata(jsonataExpr);
-    const input = ctx.data[outputVar];
-    const result = await expression.evaluate(input);
-    ctx.data[outputVar] = result;
+    const result = await expression.evaluate(output[outputVar]);
+    output[outputVar] = result;
     log.info(
       { outputVar, result: previewValue(result) },
       "fetch-html: saved jsonata result",
     );
   }
 
-  return ctx;
+  return { output, context: { ...passContext(ctx), ...output } };
 }
 
 fetchHtml.meta = {
@@ -100,7 +99,10 @@ fetchHtml.meta = {
   },
   input: {},
   output: {
-    httpResponse: { type: "string", description: "Raw HTML" },
+    httpResponse: { type: "string", description: "Raw HTML (overwritten when outputVar is httpResponse)" },
+  },
+  context: {
+    httpResponse: { type: "any", description: "Same keys as output" },
   },
   example: {
     data: {},

packages/server/scripts/fetch-http.js

@@ -1,9 +1,10 @@
 const HTTP_METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"];
 
-function ensureDataObject(ctx) {
-  if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) {
-    ctx.data = {};
+function passContext(ctx) {
+  if (ctx?.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)) {
+    return { ...ctx.context };
   }
+  return {};
 }
 
 function isPlainObject(value) {
@@ -40,7 +41,6 @@ function normalizeHeaders(value, label) {
       headers[key] = String(val);
       continue;
     }
-    // Secret wrappers are unwrapped by $axios; do not String() them.
     if (typeof val === "object") {
       headers[key] = val;
       continue;
@@ -88,8 +88,6 @@ async function fetchHttp(ctx) {
   const headers = resolveHeaders(ctx);
   const body = resolveBody(ctx);
 
-  ensureDataObject(ctx);
-
   /** @type {Record<string, unknown>} */
   const request = { url, method, headers };
   if (body.present && method !== "HEAD") {
@@ -103,14 +101,16 @@ async function fetchHttp(ctx) {
     "fetch-http: fetch complete",
   );
 
-  ctx.data.httpResponse = response.data;
-  ctx.data.httpStatus = response.status;
+  const output = {
+    httpResponse: response.data,
+    httpStatus: response.status,
+  };
   log.info(
-    { status: ctx.data.httpStatus, length: responseSize(ctx.data.httpResponse) },
+    { status: output.httpStatus, length: responseSize(output.httpResponse) },
     "fetch-http: saved httpResponse",
   );
 
-  return ctx;
+  return { output, context: { ...passContext(ctx), ...output } };
 }
 
 fetchHttp.meta = {
@@ -154,6 +154,10 @@ fetchHttp.meta = {
     httpResponse: { type: "any", description: "Response body as returned by the server" },
     httpStatus: { type: "number", description: "HTTP status code" },
   },
+  context: {
+    httpResponse: { type: "any", description: "Same as output.httpResponse" },
+    httpStatus: { type: "number", description: "Same as output.httpStatus" },
+  },
   example: {
     data: {},
     config: {

packages/server/scripts/fetch-rss-feed.js

@@ -1,10 +1,11 @@
 import rssParser from "rss-parser";
 import jsonata from "jsonata";
 
-function ensureDataObject(ctx) {
-  if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) {
-    ctx.data = {};
+function passContext(ctx) {
+  if (ctx?.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)) {
+    return { ...ctx.context };
   }
+  return {};
 }
 
 function previewValue(value) {
@@ -28,8 +29,6 @@ async function fetchRssFeed(ctx) {
     throw new Error("fetch-rss-feed: ctx.config.outputVar is required when jsonata is set");
   }
 
-  ensureDataObject(ctx);
-
   log.info({ url }, "fetch-rss-feed: fetching feed");
   const parser = new rssParser({
     customFields: {
@@ -46,7 +45,8 @@ async function fetchRssFeed(ctx) {
     "fetch-rss-feed: fetch complete",
   );
 
-  ctx.data.rssFeed = feed;
+  /** @type {Record<string, unknown>} */
+  const output = { rssFeed: feed };
   log.info(
     { title: feed.title, itemCount },
     "fetch-rss-feed: saved rssFeed",
@@ -56,14 +56,14 @@ async function fetchRssFeed(ctx) {
     log.info({ outputVar, jsonata: jsonataExpr }, "fetch-rss-feed: evaluating jsonata");
     const expression = jsonata(jsonataExpr);
     const result = await expression.evaluate(feed);
-    ctx.data[outputVar] = result;
+    output[outputVar] = result;
     log.info(
       { outputVar, result: previewValue(result) },
       "fetch-rss-feed: saved jsonata result",
     );
   }
 
-  return ctx;
+  return { output, context: { ...passContext(ctx), ...output } };
 }
 
 fetchRssFeed.meta = {
@@ -75,7 +75,7 @@ fetchRssFeed.meta = {
     outputVar: {
       type: "string",
       required: false,
-      description: "Required when jsonata is set; destination on ctx.data",
+      description: "Required when jsonata is set; key on output and context",
     },
     jsonata: {
       type: "string",
@@ -89,6 +89,9 @@ fetchRssFeed.meta = {
   output: {
     rssFeed: { type: "object", description: "Parsed RSS/Atom feed from rss-parser" },
   },
+  context: {
+    rssFeed: { type: "object", description: "Same as output.rssFeed (plus outputVar when set)" },
+  },
   example: {
     data: {},
     config: {

packages/server/scripts/fingerprint.js

@@ -1,9 +1,17 @@
 import jsonata from "jsonata";
 
-function ensureDataObject(ctx) {
-  if (ctx.data == null || typeof ctx.data !== "object" || Array.isArray(ctx.data)) {
-    ctx.data = {};
+function passContext(ctx) {
+  if (ctx?.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)) {
+    return { ...ctx.context };
   }
+  return {};
+}
+
+function mergeData(data) {
+  if (data != null && typeof data === "object" && !Array.isArray(data)) {
+    return { ...data };
+  }
+  return {};
 }
 
 async function fingerprint(ctx) {
@@ -28,13 +36,14 @@ async function fingerprint(ctx) {
     maxAge: ctx.config?.maxAge,
   });
 
-  ensureDataObject(ctx);
-  ctx.data.fingerprint = result.hash;
-  ctx.data.fingerprintChanged = result.changed;
-  ctx.data.fingerprintPrevious = result.previous;
-  ctx.data.fingerprintAt = result.changed ? result.at : result.previousAt;
-  ctx.data.fingerprintAge = result.ageMs;
-  ctx.data.fingerprintExpired = result.expired;
+  const extra = {
+    fingerprint: result.hash,
+    fingerprintChanged: result.changed,
+    fingerprintPrevious: result.previous,
+    fingerprintAt: result.changed ? result.at : result.previousAt,
+    fingerprintAge: result.ageMs,
+    fingerprintExpired: result.expired,
+  };
 
   log.info(
     {
@@ -46,17 +55,22 @@ async function fingerprint(ctx) {
     "fingerprint: result",
   );
 
+  /** @type {{ output: Record<string, unknown>, context: Record<string, unknown>, skipRemaining?: true }} */
+  const envelope = {
+    output: { ...mergeData(ctx.data), ...extra },
+    context: { ...passContext(ctx), ...extra },
+  };
   if (!result.changed && skipRemaining) {
-    ctx.skipRemaining = true;
+    envelope.skipRemaining = true;
   }
-
-  return ctx;
+  return envelope;
 }
 
 fingerprint.meta = {
   description:
     "Hash a value, compare it to the last stored fingerprint, and skip remaining steps when unchanged",
   previewConfigKey: "key",
+  reads: "ctx",
   config: {
     key: {
       type: "string",
@@ -88,6 +102,14 @@ fingerprint.meta = {
     fingerprintAge: { type: "number", required: false, description: "Age in milliseconds" },
     fingerprintExpired: { type: "boolean" },
   },
+  context: {
+    fingerprint: { type: "string" },
+    fingerprintChanged: { type: "boolean" },
+    fingerprintPrevious: { type: "string", required: false },
+    fingerprintAt: { type: "string", required: false },
+    fingerprintAge: { type: "number", required: false },
+    fingerprintExpired: { type: "boolean" },
+  },
   example: {
     data: { item: { guid: "https://example.com/post-1" } },
     config: {

packages/server/scripts/get-current-time.js

@@ -1,22 +1,30 @@
-// this script will get current time
+function passContext(ctx) {
+  if (ctx?.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)) {
+    return { ...ctx.context };
+  }
+  return {};
+}
 
-function getCurrentTime() {
-    return {
-        data: {
-            datetime: new Date().toISOString(),
-            processId: "1234"
-        }
-    }
+function getCurrentTime(ctx) {
+  const output = {
+    datetime: new Date().toISOString(),
+    processId: "1234",
+  };
+  return { output, context: { ...passContext(ctx), ...output } };
 }
 
 getCurrentTime.meta = {
-  description: "Return the current time as ctx.data.datetime",
+  description: "Return the current time as output.datetime",
   config: {},
   input: {},
   output: {
     datetime: { type: "string", description: "ISO timestamp" },
     processId: { type: "string" },
   },
+  context: {
+    datetime: { type: "string", description: "ISO timestamp" },
+    processId: { type: "string" },
+  },
   example: {
     data: {},
     config: {},

packages/server/scripts/get-secret.js

@@ -1,3 +1,17 @@
+function passContext(ctx) {
+  if (ctx?.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)) {
+    return { ...ctx.context };
+  }
+  return {};
+}
+
+function mergeData(data) {
+  if (data != null && typeof data === "object" && !Array.isArray(data)) {
+    return { ...data };
+  }
+  return {};
+}
+
 async function getSecret(ctx) {
   const name = ctx.config?.name;
   if (typeof name !== "string" || name.length === 0) {
@@ -9,19 +23,16 @@ async function getSecret(ctx) {
       : name;
 
   const value = await $secrets.get(name);
-  const base =
-    ctx != null && typeof ctx === "object" && !Array.isArray(ctx) ? { ...ctx } : {};
-  const data =
-    base.data != null && typeof base.data === "object" && !Array.isArray(base.data)
-      ? { ...base.data }
-      : {};
-  data[as] = value;
-  return { ...base, data };
+  const extra = { [as]: value };
+  return {
+    output: { ...mergeData(ctx.data), ...extra },
+    context: { ...passContext(ctx), ...extra },
+  };
 }
 
 getSecret.meta = {
   description:
-    "Load a named secret for this workflow owner into ctx.data. The value is wrapped and redacted in logs.",
+    "Load a named secret for this workflow owner onto output and context. The value is wrapped and redacted in logs.",
   previewConfigKey: "name",
   config: {
     name: {
@@ -32,16 +43,12 @@ getSecret.meta = {
     as: {
       type: "string",
       required: false,
-      description: "ctx.data field to write (defaults to name)",
+      description: "Field name on output and context (defaults to name)",
     },
   },
   input: {},
-  output: {
-    data: {
-      type: "object",
-      description: "Previous ctx.data plus the retrieved Secret at [as]",
-    },
-  },
+  output: {},
+  context: {},
   example: {
     data: {},
     config: { name: "ntfy_token", as: "ntfyToken" },

packages/server/scripts/jsonata.js

@@ -1,26 +1,34 @@
 import jsonata from "jsonata";
 
 async function jsonataFn(ctx) {
-    log.info({ctx}, "jsonata: context")
-    log.info("jsonata: evaluating expression %s", ctx.config.expression);
-    const expression = jsonata(ctx.config.expression);
-    const result = await expression.evaluate(ctx.data);
-    log.info({result}, "jsonata: expression result");
-    return result;
+  log.info({ ctx }, "jsonata: context");
+  log.info("jsonata: evaluating expression %s", ctx.config.expression);
+  const expression = jsonata(ctx.config.expression);
+  const result = await expression.evaluate(ctx);
+  log.info({ result }, "jsonata: expression result");
+  return {
+    output: result,
+    context:
+      ctx.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)
+        ? ctx.context
+        : {},
+  };
 }
 
 jsonataFn.meta = {
-  description: "Evaluate a JSONata expression against ctx.data and return the result as the next context",
+  description:
+    "Evaluate a JSONata expression against the full ctx (data, context, config); the result is the next data",
   previewConfigKey: "expression",
+  reads: "ctx",
   config: {
-    expression: { type: "string", required: true, description: "JSONata expression" },
+    expression: { type: "string", required: true, description: "JSONata expression against ctx" },
   },
   input: {},
   output: {},
   example: {
     data: { title: "Hello", url: "https://example.com" },
     config: {
-      expression: '{"data": {"title": title, "message": title, "attach": url}}',
+      expression: '{"title": data.title, "message": data.title, "attach": data.url}',
     },
   },
 };

packages/server/scripts/ntfy.js

@@ -1,5 +1,12 @@
 import jsonata from "jsonata";
 
+function passContext(ctx) {
+  if (ctx?.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)) {
+    return { ...ctx.context };
+  }
+  return {};
+}
+
 function ntfyHeaders(ctx) {
   const headers = {};
 
@@ -72,11 +79,14 @@ async function ntfy(ctx) {
         "ntfy: skipped, fingerprint unchanged",
       );
       return {
-        sent: "false",
-        skipped: true,
-        fingerprint: checked.hash,
-        fingerprintAt: checked.previousAt,
-        fingerprintAge: checked.ageMs,
+        output: {
+          sent: "false",
+          skipped: true,
+          fingerprint: checked.hash,
+          fingerprintAt: checked.previousAt,
+          fingerprintAge: checked.ageMs,
+        },
+        context: passContext(ctx),
       };
     }
     fp = checked;
@@ -126,7 +136,7 @@ async function ntfy(ctx) {
     sent.fingerprint = stored.hash;
     sent.fingerprintAt = stored.at;
   }
-  return sent;
+  return { output: sent, context: passContext(ctx) };
 }
 
 ntfy.meta = {
@@ -147,7 +157,7 @@ ntfy.meta = {
     fingerprintJsonata: {
       type: "string",
       required: false,
-      description: "JSONata against ctx; default hashes title, message, attach, filename, contentType",
+      description: "JSONata against full ctx; default hashes data title, message, attach, filename, contentType",
     },
     fingerprintMaxAge: {
       type: "string",
@@ -163,8 +173,9 @@ ntfy.meta = {
     filename: { type: "string", required: false },
     contentType: { type: "string", required: false },
   },
+  reads: "ctx",
   output: {
-    sent: { type: "string", description: "Replaces the workflow context with { sent: \"true\" }" },
+    sent: { type: "string", description: '"true" when sent, "false" when fingerprint skipped' },
   },
   example: {
     data: { title: "Hello", message: "Hello from jerapah-flow" },

packages/server/scripts/render-template.js

@@ -70,9 +70,15 @@ async function renderTemplate(ctx) {
   const text = htmlToText(html);
 
   return {
-    html,
-    text,
-    template: templateName,
+    output: {
+      html,
+      text,
+      template: templateName,
+    },
+    context:
+      ctx.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)
+        ? ctx.context
+        : {},
   };
 }
 

packages/server/scripts/send-email.js

@@ -218,13 +218,19 @@ async function sendEmail(ctx) {
   log.info({ messageId: info.messageId }, "send-email: message sent");
 
   return {
-    sent: true,
-    messageId: info.messageId ?? null,
-    from,
-    to,
-    cc: cc ?? null,
-    bcc: bcc ?? null,
-    subject,
+    output: {
+      sent: true,
+      messageId: info.messageId ?? null,
+      from,
+      to,
+      cc: cc ?? null,
+      bcc: bcc ?? null,
+      subject,
+    },
+    context:
+      ctx.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)
+        ? ctx.context
+        : {},
   };
 }
 

packages/server/scripts/trigger-workflow.js

@@ -1,5 +1,12 @@
 import jsonata from "jsonata";
 
+function passContext(ctx) {
+  if (ctx?.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)) {
+    return { ...ctx.context };
+  }
+  return {};
+}
+
 async function triggerWorkflow(ctx) {
   const name = ctx.config?.name;
   if (typeof name !== "string" || name.length === 0) {
@@ -9,27 +16,23 @@ async function triggerWorkflow(ctx) {
   let data = ctx.data;
   const expression = ctx.config?.expression;
   if (typeof expression === "string" && expression.length > 0) {
-    const result = jsonata(expression).evaluate(ctx.data);
+    const result = jsonata(expression).evaluate(ctx);
     data = await result;
   }
 
   const started = await $workflows.trigger(name, data);
-  const triggered = { name, runId: started?.runId ?? null };
-
-  const base =
-    ctx != null && typeof ctx === "object" && !Array.isArray(ctx) ? { ...ctx } : {};
-
-  if (base.data != null && typeof base.data === "object" && !Array.isArray(base.data)) {
-    return { ...base, data: { ...base.data, triggered } };
-  }
-  return { ...base, triggered };
+  return {
+    output: { name, runId: started?.runId ?? null },
+    context: passContext(ctx),
+  };
 }
 
 triggerWorkflow.meta = {
   description:
-    "Fire-and-forget another workflow by YAML name (same owner). Destination must declare triggers: [{ type: workflow }]. Optionally reshape ctx.data with JSONata before sending.",
+    "Fire-and-forget another workflow by YAML name (same owner). Destination must declare triggers: [{ type: workflow }]. Optionally reshape the destination input with JSONata against full ctx.",
   previewConfigKey: "name",
   tags: ["trigger"],
+  reads: "ctx",
   config: {
     name: {
       type: "string",
@@ -40,15 +43,13 @@ triggerWorkflow.meta = {
       type: "string",
       required: false,
       description:
-        "Optional JSONata expression evaluated against ctx.data; result becomes the destination run input",
+        "Optional JSONata expression evaluated against ctx; result becomes the destination run input",
     },
   },
   input: {},
   output: {
-    triggered: {
-      type: "object",
-      description: "Record of the kicked-off run ({ name, runId }); under data when data is an object",
-    },
+    name: { type: "string", description: "Destination workflow name" },
+    runId: { type: "string", description: "Started run id (null if the destination failed to start)" },
   },
   example: {
     data: {
@@ -58,7 +59,7 @@ triggerWorkflow.meta = {
     config: {
       name: "notify-comic",
       expression:
-        '{ "title": title, "message": title, "attach": url }',
+        '{ "title": data.title, "message": data.title, "attach": data.url }',
     },
   },
 };