feat(variables): introduce variable management system

- Added a new module for managing variables, including creation, retrieval, updating, and deletion functionalities.
- Implemented a new API endpoint for variable operations, allowing users to interact with variables through HTTP requests.
- Enhanced the configuration reference system to support variable references, replacing the previous KV reference system.
- Updated existing workflows and scripts to utilize the new variable management features, improving flexibility in configuration handling.
- Introduced smoke tests to validate the functionality of the variable system, ensuring reliability and robustness.

This update significantly enhances the ability to manage dynamic variables within workflows, improving overall usability and maintainability.

packages/server/test/config-refs-smoke.js

@@ -1,6 +1,6 @@
 import { migrate, db } from "../db.js";
-import { kvSet, kvDelete } from "../kv-store.js";
 import { upsertSecret, deleteSecret } from "../secrets-store.js";
+import { deleteVariable, upsertVariable } from "../variables-store.js";
 import { Secret } from "../secret-value.js";
 import { parseConfigRef, resolveConfigRefs } from "../config-refs.js";
 
@@ -45,7 +45,8 @@ assertParse("  password123  ", null);
 assertParse("$SECRET_zte_modem_password", { kind: "secret", name: "zte_modem_password" });
 assertParse("  $SECRET_zte_modem_password  ", { kind: "secret", name: "zte_modem_password" });
 assertParse("Bearer $SECRET_x", null);
-assertParse("$KV_modem password", { kind: "kv", name: "modem password" });
+assertParse("$KV_modem password", null);
+assertParse("$VAR_ntfy_url", { kind: "var", name: "ntfy_url" });
 assertParse("$CONTEXT_token", { kind: "context", name: "token" });
 assertParse("$SECRET_", { kind: "secret", name: "" });
 assertParse("$CONTEXT_SECRET_foo", { kind: "context", name: "SECRET_foo" });
@@ -55,6 +56,8 @@ assertParse("$CONTEXT_SECRET_foo", { kind: "context", name: "SECRET_foo" });
   assert(literal === "password123", "literal passthrough");
   const unknown = await resolveConfigRefs("$FOO_bar", ctx);
   assert(unknown === "$FOO_bar", "$FOO_bar stays literal");
+  const kvLiteral = await resolveConfigRefs("$KV_modem_password", ctx);
+  assert(kvLiteral === "$KV_modem_password", "$KV_ stays literal");
   const embedded = await resolveConfigRefs("Bearer $SECRET_x", ctx);
   assert(embedded === "Bearer $SECRET_x", "mid-string stays literal");
   const number = await resolveConfigRefs(42, ctx);
@@ -66,9 +69,24 @@ const secret = await upsertSecret({
   name: "config_refs_smoke_token",
   value: "s3cret-ok",
 });
-await kvSet(workflowKey, "modem_password", "kv-pass-ok");
-await kvSet(workflowKey, "modem password", "kv-spaced-ok");
-await kvSet(workflowKey, "obj-key", { nested: true });
+const varUrl = await upsertVariable({
+  owner,
+  name: "config_refs_smoke_url",
+  type: "string",
+  value: "https://example.test",
+});
+const varRetry = await upsertVariable({
+  owner,
+  name: "config_refs_smoke_retry",
+  type: "number",
+  value: 3,
+});
+const varDebug = await upsertVariable({
+  owner,
+  name: "config_refs_smoke_debug",
+  type: "boolean",
+  value: false,
+});
 
 try {
   {
@@ -80,12 +98,18 @@ try {
     assert(resolved === "s3cret-ok", "secret resolve trimmed");
   }
   {
-    const resolved = await resolveConfigRefs("$KV_modem_password", ctx);
-    assert(resolved === "kv-pass-ok", "kv resolve");
+    const resolved = await resolveConfigRefs("$VAR_config_refs_smoke_url", ctx);
+    assert(resolved === "https://example.test", "var string");
   }
   {
-    const resolved = await resolveConfigRefs("$KV_modem password", ctx);
-    assert(resolved === "kv-spaced-ok", "kv spaced key");
+    const resolved = await resolveConfigRefs("$VAR_config_refs_smoke_retry", ctx);
+    assert(resolved === 3, "var number stays number");
+    assert(typeof resolved === "number", "var number type");
+  }
+  {
+    const resolved = await resolveConfigRefs("$VAR_config_refs_smoke_debug", ctx);
+    assert(resolved === false, "var boolean stays false");
+    assert(typeof resolved === "boolean", "var boolean type");
   }
   {
     const resolved = await resolveConfigRefs("$CONTEXT_token", {
@@ -113,16 +137,20 @@ try {
   {
     const nested = await resolveConfigRefs(
       {
-        url: "http://example.test",
+        url: "$VAR_config_refs_smoke_url",
+        retry: "$VAR_config_refs_smoke_retry",
+        debug: "$VAR_config_refs_smoke_debug",
         password: "$SECRET_config_refs_smoke_token",
         headers: { Authorization: "$KV_modem_password" },
         extra: ["$CONTEXT_token", "plain"],
       },
       { ...ctx, context: { token: "ctx-token-ok" } },
     );
-    assert(nested.url === "http://example.test", "nested literal");
+    assert(nested.url === "https://example.test", "nested var string");
+    assert(nested.retry === 3, "nested var number");
+    assert(nested.debug === false, "nested var boolean");
     assert(nested.password === "s3cret-ok", "nested secret");
-    assert(nested.headers.Authorization === "kv-pass-ok", "nested kv");
+    assert(nested.headers.Authorization === "$KV_modem_password", "nested $KV_ stays literal");
     assert(nested.extra[0] === "ctx-token-ok", "nested array context");
     assert(nested.extra[1] === "plain", "nested array literal");
   }
@@ -146,18 +174,6 @@ try {
     () => resolveConfigRefs("$SECRET_", ctx),
     "invalid secret name",
   );
-  await assertRejects(
-    () => resolveConfigRefs("$KV_missing-key", ctx),
-    'KV "missing-key" not found',
-  );
-  await assertRejects(
-    () => resolveConfigRefs("$KV_obj-key", ctx),
-    'KV "obj-key" is not a scalar',
-  );
-  await assertRejects(
-    () => resolveConfigRefs("$KV_", ctx),
-    "empty KV key",
-  );
   await assertRejects(
     () => resolveConfigRefs("$CONTEXT_missing", ctx),
     'context "missing" not found',
@@ -174,11 +190,23 @@ try {
     () => resolveConfigRefs("$CONTEXT_", ctx),
     "empty context key",
   );
+  await assertRejects(
+    () => resolveConfigRefs("$VAR_does_not_exist_xyz", ctx),
+    'variable "does_not_exist_xyz" not found',
+  );
+  await assertRejects(
+    () => resolveConfigRefs("$VAR_not valid", ctx),
+    "invalid variable name",
+  );
+  await assertRejects(
+    () => resolveConfigRefs("$VAR_", ctx),
+    "empty variable name",
+  );
 } finally {
   await deleteSecret(secret.id);
-  await kvDelete(workflowKey, "modem_password");
-  await kvDelete(workflowKey, "modem password");
-  await kvDelete(workflowKey, "obj-key");
+  await deleteVariable(varUrl.id);
+  await deleteVariable(varRetry.id);
+  await deleteVariable(varDebug.id);
 }
 
 console.log("config-refs smoke test passed");

packages/server/test/variables-smoke.js

@@ -0,0 +1,136 @@
+import { migrate, db } from "../db.js";
+import { runScriptSource } from "../script-sandbox.js";
+import { log } from "../logger.js";
+import {
+  deleteVariable,
+  encodeVariableValue,
+  getVariablePlain,
+  upsertVariable,
+} from "../variables-store.js";
+
+await migrate();
+
+function assert(cond, msg) {
+  if (!cond) throw new Error(msg);
+}
+
+async function assertThrows(fn, match) {
+  try {
+    await fn();
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (match && !message.includes(match)) {
+      throw new Error(`threw "${message}", expected to include "${match}"`);
+    }
+    return;
+  }
+  throw new Error(`expected to throw (${match ?? "any error"})`);
+}
+
+const owner = "default";
+
+await assertThrows(
+  () => encodeVariableValue("number", "3"),
+  "finite number",
+);
+await assertThrows(
+  () => encodeVariableValue("number", NaN),
+  "finite number",
+);
+await assertThrows(
+  () => encodeVariableValue("boolean", "true"),
+  "boolean",
+);
+await assertThrows(
+  () => encodeVariableValue("string", 3),
+  "string",
+);
+
+const created = [];
+try {
+  const str = await upsertVariable({
+    owner,
+    name: "variables_smoke_url",
+    type: "string",
+    value: "https://n.0dev.web.id/system",
+  });
+  created.push(str.id);
+  assert(str.value === "https://n.0dev.web.id/system", "string stored");
+  assert(
+    (await getVariablePlain(owner, "variables_smoke_url")) === str.value,
+    "string plain",
+  );
+
+  const num = await upsertVariable({
+    owner,
+    name: "variables_smoke_retry",
+    type: "number",
+    value: 3,
+  });
+  created.push(num.id);
+  assert(num.value === 3 && typeof num.value === "number", "number stored");
+
+  const flag = await upsertVariable({
+    owner,
+    name: "variables_smoke_debug",
+    type: "boolean",
+    value: false,
+  });
+  created.push(flag.id);
+  assert(flag.value === false, "boolean false stored");
+
+  const updated = await upsertVariable({
+    owner,
+    name: "variables_smoke_retry",
+    type: "number",
+    value: 9,
+  });
+  assert(updated.id === num.id, "upsert same id");
+  assert(updated.value === 9, "upsert number");
+
+  await assertThrows(
+    () =>
+      upsertVariable({
+        owner,
+        name: "variables_smoke_bad",
+        type: "number",
+        value: "abc",
+      }),
+    "finite number",
+  );
+
+  const scriptOut = await runScriptSource(
+    "variables-smoke.js",
+    `export default async function () {
+      const url = await $vars.get("variables_smoke_url");
+      const retry = await $vars.get("variables_smoke_retry");
+      const debug = await $vars.get("variables_smoke_debug");
+      return { url, retry, debug, retryType: typeof retry, debugType: typeof debug };
+    }`,
+    { data: {} },
+    { log, workflowName: "default/variables-smoke.yaml", owner },
+  );
+  assert(scriptOut.url === "https://n.0dev.web.id/system", "script $vars string");
+  assert(scriptOut.retry === 9 && scriptOut.retryType === "number", "script $vars number");
+  assert(scriptOut.debug === false && scriptOut.debugType === "boolean", "script $vars boolean");
+
+  await assertThrows(
+    () =>
+      runScriptSource(
+        "variables-smoke.js",
+        `export default async function () {
+          return $vars.get("does_not_exist_xyz");
+        }`,
+        { data: {} },
+        { log, workflowName: "default/variables-smoke.yaml", owner },
+      ),
+    'variable "does_not_exist_xyz" not found',
+  );
+} finally {
+  for (const id of created) {
+    await deleteVariable(id);
+  }
+}
+
+console.log("variables smoke test passed");
+await db.destroy();