diff --git a/.gitignore b/.gitignore index 2e4d176..943a64c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,8 @@ logs/ *.db-* .DS_Store packages/web/dist + +# Personal/local scripts and workflows (not for the repo) +packages/server/scripts/dev-* +packages/server/workflows/**/dev-* +debug-*.js diff --git a/packages/server/config-refs.js b/packages/server/config-refs.js new file mode 100644 index 0000000..7ab8aea --- /dev/null +++ b/packages/server/config-refs.js @@ -0,0 +1,150 @@ +import { coerceCredentialString } from "./http-trigger-auth.js"; +import { kvGet } from "./kv-store.js"; +import { assertSecretName, getSecretPlaintext } from "./secrets-store.js"; +import { isSecret } from "./secret-value.js"; + +/** Longest prefix first so `$CONTEXT_` is not confused with `$KV_`. */ +const PREFIXES = [ + { kind: "context", prefix: "$CONTEXT_" }, + { kind: "secret", prefix: "$SECRET_" }, + { kind: "kv", prefix: "$KV_" }, +]; + +/** + * @typedef {{ kind: "secret" | "kv" | "context", name: string, raw: string }} ConfigRef + * @typedef {{ owner: string, workflowKey: string, context?: unknown }} ConfigRefCtx + */ + +/** + * Parse a whole-value config placeholder. Returns null for literals. + * @param {unknown} value + * @returns {ConfigRef | null} + */ +export function parseConfigRef(value) { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + for (const { kind, prefix } of PREFIXES) { + if (trimmed.startsWith(prefix)) { + return { kind, name: trimmed.slice(prefix.length), raw: trimmed }; + } + } + return null; +} + +/** + * Walk config (objects/arrays) and replace whole-value `$SECRET_` / `$KV_` / `$CONTEXT_` + * strings. Does not walk trigger data. + * + * @param {unknown} value + * @param {ConfigRefCtx} ctx + * @param {WeakSet} [seen] + * @returns {Promise} + */ +export async function resolveConfigRefs(value, ctx, seen = new WeakSet()) { + if (typeof value === "string") { + return resolveStringRef(value, ctx); + } + if (value == null || typeof value !== "object") { + return value; + } + if (seen.has(value)) return value; + seen.add(value); + + if (Array.isArray(value)) { + const out = []; + for (const item of value) { + out.push(await resolveConfigRefs(item, ctx, seen)); + } + return out; + } + + /** @type {Record} */ + const out = {}; + for (const [key, child] of Object.entries(value)) { + out[key] = await resolveConfigRefs(child, ctx, seen); + } + return out; +} + +/** + * @param {string} value + * @param {ConfigRefCtx} ctx + * @returns {Promise} + */ +async function resolveStringRef(value, ctx) { + const ref = parseConfigRef(value); + if (!ref) return value; + + if (ref.kind === "secret") { + return resolveSecretRef(ref, ctx); + } + if (ref.kind === "kv") { + return resolveKvRef(ref, ctx); + } + return resolveContextRef(ref, ctx); +} + +/** + * @param {ConfigRef} ref + * @param {ConfigRefCtx} ctx + * @returns {Promise} + */ +async function resolveSecretRef(ref, ctx) { + try { + assertSecretName(ref.name); + } catch { + throw new Error(`config ref ${ref.raw}: invalid secret name`); + } + const plaintext = await getSecretPlaintext(ctx.owner, ref.name); + if (plaintext == null) { + throw new Error(`config ref ${ref.raw}: secret "${ref.name}" not found`); + } + return plaintext; +} + +/** + * @param {ConfigRef} ref + * @param {ConfigRefCtx} ctx + * @returns {Promise} + */ +async function resolveKvRef(ref, ctx) { + if (ref.name.length === 0) { + throw new Error(`config ref ${ref.raw}: empty KV key`); + } + const raw = await kvGet(ctx.workflowKey, ref.name); + if (raw == null) { + throw new Error(`config ref ${ref.raw}: KV "${ref.name}" not found`); + } + const coerced = coerceCredentialString(raw); + if (coerced == null) { + throw new Error(`config ref ${ref.raw}: KV "${ref.name}" is not a scalar`); + } + return coerced; +} + +/** + * @param {ConfigRef} ref + * @param {ConfigRefCtx} ctx + * @returns {string} + */ +function resolveContextRef(ref, ctx) { + if (ref.name.length === 0) { + throw new Error(`config ref ${ref.raw}: empty context key`); + } + const bag = + ctx.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context) + ? /** @type {Record} */ (ctx.context) + : {}; + if (!Object.prototype.hasOwnProperty.call(bag, ref.name)) { + throw new Error(`config ref ${ref.raw}: context "${ref.name}" not found`); + } + const raw = bag[ref.name]; + if (isSecret(raw)) { + return raw.reveal(); + } + const coerced = coerceCredentialString(raw); + if (coerced == null) { + throw new Error(`config ref ${ref.raw}: context "${ref.name}" is not a scalar`); + } + return coerced; +} diff --git a/packages/server/registry.js b/packages/server/registry.js index 3a00336..f4d56ff 100644 --- a/packages/server/registry.js +++ b/packages/server/registry.js @@ -30,6 +30,7 @@ import { sendHttpPageOrJson, sendSuccessPage, } from "./http-trigger-auth.js"; +import { resolveConfigRefs } from "./config-refs.js"; /** * @typedef {{ owner: string, file: string, workflow: any }} WorkflowEntry @@ -137,9 +138,12 @@ export function createRegistry(server) { const onDisk = fsStore.listOwnerYamlFiles(owner); for (const file of onDisk) { - if (!workflowFiles.includes(file)) { - log.warn(`Workflow file not in registers.yaml: ${owner}/${file}`); + if (workflowFiles.includes(file)) continue; + if (file.startsWith("dev-")) { + workflowFiles.push(file); + continue; } + log.warn(`Workflow file not in registers.yaml: ${owner}/${file}`); } for (const file of workflowFiles) { @@ -572,21 +576,26 @@ export function createRegistry(server) { depth, ) { const script = parsed.kind === "set" ? SET_STEP_SCRIPT : parsed.script; - const config = parsed.config; + const unresolvedConfig = parsed.config; const incomingContext = normalizeContext(ctx.context); - const stepCtx = { - data: ctx.data, - context: incomingContext, - config, - }; const step = await store.startStep({ runId, index, script, - config, + config: unresolvedConfig, }); const stepLog = runLog.child({ stepId: step.id, script }); try { + const config = await resolveConfigRefs(unresolvedConfig, { + owner, + workflowKey: key, + context: incomingContext, + }); + const stepCtx = { + data: ctx.data, + context: incomingContext, + config, + }; if (parsed.when) { const whenResult = await evaluateJsonata(parsed.when, stepCtx); if (!isJsonataTruthy(whenResult)) { diff --git a/packages/server/src/api/scripts.js b/packages/server/src/api/scripts.js index af9c5ef..6d175d0 100644 --- a/packages/server/src/api/scripts.js +++ b/packages/server/src/api/scripts.js @@ -7,6 +7,7 @@ import { import * as fsStore from "../../fs-store.js"; import { createDryRunLogger, safeSerialize } from "./dry-run-logger.js"; import { normalizeStepResult } from "../../step-result.js"; +import { resolveConfigRefs } from "../../config-refs.js"; /** * @param {{ referencedScripts: () => Set }} registry @@ -123,16 +124,21 @@ export default function scriptsPluginFactory(registry) { body.context != null && typeof body.context === "object" && !Array.isArray(body.context) ? body.context : {}; - const ctx = { - data: body.data ?? null, - context: incomingContext, - config: body.config ?? null, - }; const { log, logs } = createDryRunLogger(); const started = Date.now(); try { + const config = await resolveConfigRefs(body.config ?? null, { + owner, + workflowKey: "dry-run", + context: incomingContext, + }); + const ctx = { + data: body.data ?? null, + context: incomingContext, + config, + }; const { fn, meta, metaError } = instantiateScriptSource(name, body.content, { log, workflowName: "dry-run", diff --git a/packages/server/test/config-refs-smoke.js b/packages/server/test/config-refs-smoke.js new file mode 100644 index 0000000..774d1c4 --- /dev/null +++ b/packages/server/test/config-refs-smoke.js @@ -0,0 +1,185 @@ +import { migrate, db } from "../db.js"; +import { kvSet, kvDelete } from "../kv-store.js"; +import { upsertSecret, deleteSecret } from "../secrets-store.js"; +import { Secret } from "../secret-value.js"; +import { parseConfigRef, resolveConfigRefs } from "../config-refs.js"; + +await migrate(); + +function assert(cond, msg) { + if (!cond) throw new Error(msg); +} + +async function assertRejects(fn, match) { + try { + await fn(); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (match && !message.includes(match)) { + throw new Error(`rejected with "${message}", expected to include "${match}"`); + } + return; + } + throw new Error(`expected to reject (${match ?? "any error"})`); +} + +const owner = "default"; +const workflowKey = "default/config-refs-smoke.yaml"; +const ctx = { owner, workflowKey, context: {} }; + +function assertParse(value, expected) { + const got = parseConfigRef(value); + if (expected == null) { + assert(got == null, `expected null parse for ${JSON.stringify(value)}, got ${JSON.stringify(got)}`); + return; + } + assert(got != null, `expected parse for ${JSON.stringify(value)}`); + assert(got.kind === expected.kind, `kind ${got.kind} !== ${expected.kind}`); + assert(got.name === expected.name, `name ${JSON.stringify(got.name)} !== ${JSON.stringify(expected.name)}`); +} + +assertParse("password123", null); +assertParse("$FOO_bar", null); +assertParse("$SECRET", null); +assertParse(" password123 ", null); +assertParse("$SECRET_zte_modem_password", { kind: "secret", name: "zte_modem_password" }); +assertParse(" $SECRET_zte_modem_password ", { kind: "secret", name: "zte_modem_password" }); +assertParse("Bearer $SECRET_x", null); +assertParse("$KV_modem password", { kind: "kv", name: "modem password" }); +assertParse("$CONTEXT_token", { kind: "context", name: "token" }); +assertParse("$SECRET_", { kind: "secret", name: "" }); +assertParse("$CONTEXT_SECRET_foo", { kind: "context", name: "SECRET_foo" }); + +{ + const literal = await resolveConfigRefs("password123", ctx); + assert(literal === "password123", "literal passthrough"); + const unknown = await resolveConfigRefs("$FOO_bar", ctx); + assert(unknown === "$FOO_bar", "$FOO_bar stays literal"); + const embedded = await resolveConfigRefs("Bearer $SECRET_x", ctx); + assert(embedded === "Bearer $SECRET_x", "mid-string stays literal"); + const number = await resolveConfigRefs(42, ctx); + assert(number === 42, "number passthrough"); +} + +const secret = await upsertSecret({ + owner, + name: "config_refs_smoke_token", + value: "s3cret-ok", +}); +await kvSet(workflowKey, "modem_password", "kv-pass-ok"); +await kvSet(workflowKey, "modem password", "kv-spaced-ok"); +await kvSet(workflowKey, "obj-key", { nested: true }); + +try { + { + const resolved = await resolveConfigRefs("$SECRET_config_refs_smoke_token", ctx); + assert(resolved === "s3cret-ok", "secret resolve"); + } + { + const resolved = await resolveConfigRefs(" $SECRET_config_refs_smoke_token ", ctx); + assert(resolved === "s3cret-ok", "secret resolve trimmed"); + } + { + const resolved = await resolveConfigRefs("$KV_modem_password", ctx); + assert(resolved === "kv-pass-ok", "kv resolve"); + } + { + const resolved = await resolveConfigRefs("$KV_modem password", ctx); + assert(resolved === "kv-spaced-ok", "kv spaced key"); + } + { + const resolved = await resolveConfigRefs("$CONTEXT_token", { + ...ctx, + context: { token: "ctx-token-ok" }, + }); + assert(resolved === "ctx-token-ok", "context string"); + } + { + const resolved = await resolveConfigRefs("$CONTEXT_n", { + ...ctx, + context: { n: 7 }, + }); + assert(resolved === "7", "context number stringify"); + } + { + const wrapped = new Secret("wrapped-secret-ok"); + const resolved = await resolveConfigRefs("$CONTEXT_tok", { + ...ctx, + context: { tok: wrapped }, + }); + assert(resolved === "wrapped-secret-ok", "context Secret unwrap"); + } + + { + const nested = await resolveConfigRefs( + { + url: "http://example.test", + password: "$SECRET_config_refs_smoke_token", + headers: { Authorization: "$KV_modem_password" }, + extra: ["$CONTEXT_token", "plain"], + }, + { ...ctx, context: { token: "ctx-token-ok" } }, + ); + assert(nested.url === "http://example.test", "nested literal"); + assert(nested.password === "s3cret-ok", "nested secret"); + assert(nested.headers.Authorization === "kv-pass-ok", "nested kv"); + assert(nested.extra[0] === "ctx-token-ok", "nested array context"); + assert(nested.extra[1] === "plain", "nested array literal"); + } + + const data = { password: "$SECRET_config_refs_smoke_token" }; + const config = { password: "$SECRET_config_refs_smoke_token" }; + const resolvedConfig = await resolveConfigRefs(config, ctx); + assert(resolvedConfig.password === "s3cret-ok", "config resolved"); + assert(data.password === "$SECRET_config_refs_smoke_token", "data not walked"); + assert(config.password === "$SECRET_config_refs_smoke_token", "input config not mutated"); + + await assertRejects( + () => resolveConfigRefs("$SECRET_does_not_exist_xyz", ctx), + 'secret "does_not_exist_xyz" not found', + ); + await assertRejects( + () => resolveConfigRefs("$SECRET_not valid", ctx), + "invalid secret name", + ); + await assertRejects( + () => resolveConfigRefs("$SECRET_", ctx), + "invalid secret name", + ); + await assertRejects( + () => resolveConfigRefs("$KV_missing-key", ctx), + 'KV "missing-key" not found', + ); + await assertRejects( + () => resolveConfigRefs("$KV_obj-key", ctx), + 'KV "obj-key" is not a scalar', + ); + await assertRejects( + () => resolveConfigRefs("$KV_", ctx), + "empty KV key", + ); + await assertRejects( + () => resolveConfigRefs("$CONTEXT_missing", ctx), + 'context "missing" not found', + ); + await assertRejects( + () => + resolveConfigRefs("$CONTEXT_obj", { + ...ctx, + context: { obj: { a: 1 } }, + }), + 'context "obj" is not a scalar', + ); + await assertRejects( + () => resolveConfigRefs("$CONTEXT_", ctx), + "empty context key", + ); +} finally { + await deleteSecret(secret.id); + await kvDelete(workflowKey, "modem_password"); + await kvDelete(workflowKey, "modem password"); + await kvDelete(workflowKey, "obj-key"); +} + +console.log("config-refs smoke test passed"); +await db.destroy(); diff --git a/packages/web/src/components/DuplicateWorkflowDialog.jsx b/packages/web/src/components/DuplicateWorkflowDialog.jsx index 63ac181..d45cb58 100644 --- a/packages/web/src/components/DuplicateWorkflowDialog.jsx +++ b/packages/web/src/components/DuplicateWorkflowDialog.jsx @@ -2,10 +2,12 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { errorMessage } from "../api/client.js"; import { useDuplicateWorkflow, useOwners, useWorkflows } from "../api/hooks.js"; import { ensureWorkflowFilename, suggestCopyFilename } from "../lib/workflow-doc.js"; +import { useNotifications } from "../notifications.jsx"; const EMPTY_WORKFLOWS = []; export function DuplicateWorkflowDialog({ source, warnUnsaved, onClose, onDuplicated }) { + const { notify } = useNotifications(); const { data: owners = [] } = useOwners(); const { data: workflows = EMPTY_WORKFLOWS } = useWorkflows(); const duplicate = useDuplicateWorkflow(); @@ -40,6 +42,7 @@ export function DuplicateWorkflowDialog({ source, warnUnsaved, onClose, onDuplic }, { onSuccess: (data) => { + notify.success(`Duplicated to ${data.owner}/${data.file}`); onDuplicated?.(data); }, }, diff --git a/packages/web/src/components/workflow/ConfigFields.jsx b/packages/web/src/components/workflow/ConfigFields.jsx index 2b846e1..dc2eaac 100644 --- a/packages/web/src/components/workflow/ConfigFields.jsx +++ b/packages/web/src/components/workflow/ConfigFields.jsx @@ -6,6 +6,33 @@ import { FieldLabel } from "./FieldHelp.jsx"; const MULTILINE_KEYS = new Set(["expression", "jsonata"]); +const CONFIG_REF_PREFIXES = [ + { prefix: "$SECRET_", label: "secret" }, + { prefix: "$CONTEXT_", label: "context" }, + { prefix: "$KV_", label: "KV" }, +]; + +function describeConfigRef(value) { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + for (const { prefix, label } of CONFIG_REF_PREFIXES) { + if (trimmed.startsWith(prefix) && trimmed.length > prefix.length) { + return { label, name: trimmed.slice(prefix.length) }; + } + } + return null; +} + +function ConfigRefHint({ value }) { + const ref = describeConfigRef(value); + if (!ref) return null; + return ( +

+ from {ref.label} {ref.name} +

+ ); +} + function fieldSpec(meta, key) { const spec = meta?.config?.[key]; if (spec && typeof spec === "object") return spec; @@ -261,11 +288,14 @@ function ObjectFields({ value, onChange }) { onChange={(nv) => setVal(k, nv)} /> ) : ( - setVal(k, nv)} - className="font-mono text-xs" - /> +
+ setVal(k, nv)} + className="font-mono text-xs" + /> + +
)} + + ))} + + ); +} + +export function useNotifications() { + const ctx = useContext(NotificationContext); + if (!ctx) { + throw new Error("useNotifications must be used within NotificationProvider"); + } + return ctx; +} diff --git a/packages/web/src/pages/ScriptDryRunPage.jsx b/packages/web/src/pages/ScriptDryRunPage.jsx index ddc332a..48e3d0f 100644 --- a/packages/web/src/pages/ScriptDryRunPage.jsx +++ b/packages/web/src/pages/ScriptDryRunPage.jsx @@ -19,8 +19,10 @@ import { contextFromMeta, prettyJson, } from "../lib/script.js"; +import { useNotifications } from "../notifications.jsx"; export function ScriptDryRunPage() { + const { notify } = useNotifications(); const { name: rawName } = useParams(); const name = decodeURIComponent(rawName ?? ""); const location = useLocation(); @@ -129,7 +131,10 @@ export function ScriptDryRunPage() { } function onSave() { - save.mutate({ name, content }); + save.mutate( + { name, content }, + { onSuccess: () => notify.success("Script saved") }, + ); } return ( @@ -194,9 +199,6 @@ export function ScriptDryRunPage() { {save.isError ? (

{errorMessage(save.error)}

) : null} - {save.isSuccess ? ( -

Script saved

- ) : null}
diff --git a/packages/web/src/pages/ScriptEditPage.jsx b/packages/web/src/pages/ScriptEditPage.jsx index 506e24e..d0ee3fb 100644 --- a/packages/web/src/pages/ScriptEditPage.jsx +++ b/packages/web/src/pages/ScriptEditPage.jsx @@ -7,6 +7,7 @@ import { CodeEditor } from "../components/CodeEditor.jsx"; import { ScriptIcon } from "../components/ScriptIcon.jsx"; import { ScriptMetaPanel } from "../components/ScriptMetaPanel.jsx"; import { NEW_SCRIPT_TEMPLATE, normalizeScriptName } from "../lib/script.js"; +import { useNotifications } from "../notifications.jsx"; export function ScriptNewPage() { const navigate = useNavigate(); @@ -88,6 +89,7 @@ export function ScriptEditPage() { const { name: rawName } = useParams(); const name = decodeURIComponent(rawName ?? ""); const navigate = useNavigate(); + const { notify } = useNotifications(); const existing = useScript(name); const save = useSaveScript(); const [content, setContent] = useState(""); @@ -103,7 +105,10 @@ export function ScriptEditPage() { function onSave(e) { e.preventDefault(); - save.mutate({ name, content }); + save.mutate( + { name, content }, + { onSuccess: () => notify.success("Script saved") }, + ); } function openDryRun() { @@ -172,9 +177,6 @@ export function ScriptEditPage() { {save.isError ? (

{errorMessage(save.error)}

) : null} - {save.isSuccess ? ( -

Script saved

- ) : null}
); } diff --git a/packages/web/src/pages/WorkflowEditPage.jsx b/packages/web/src/pages/WorkflowEditPage.jsx index 03c6d1e..3f8ae0d 100644 --- a/packages/web/src/pages/WorkflowEditPage.jsx +++ b/packages/web/src/pages/WorkflowEditPage.jsx @@ -12,6 +12,7 @@ import { DuplicateWorkflowDialog } from "../components/DuplicateWorkflowDialog.j import { WorkflowFileIcon } from "../components/WorkflowFileIcon.jsx"; import { WorkflowVisualEditor } from "../components/workflow/WorkflowVisualEditor.jsx"; import { NEW_WORKFLOW_YAML, parseWorkflowYaml } from "../lib/workflow-doc.js"; +import { useNotifications } from "../notifications.jsx"; function WorkflowEditorLayout({ title, @@ -19,7 +20,6 @@ function WorkflowEditorLayout({ savePending, saveDisabled, saveError, - saveSuccess, onSave, onTest, onDuplicate, @@ -81,7 +81,6 @@ function WorkflowEditorLayout({ {children} {saveError ?

{saveError}

: null} {enableError ?

{enableError}

: null} - {saveSuccess ?

Workflow saved

: null} ); } @@ -154,6 +153,7 @@ export function WorkflowNewPage() { export function WorkflowEditPage() { const navigate = useNavigate(); + const { notify } = useNotifications(); const { owner: rawOwner, file: rawFile } = useParams(); const owner = decodeURIComponent(rawOwner ?? ""); const file = decodeURIComponent(rawFile ?? ""); @@ -190,7 +190,10 @@ export function WorkflowEditPage() { save.mutate( { owner, file, content }, { - onSuccess: () => setSavedYaml(content), + onSuccess: () => { + setSavedYaml(content); + notify.success("Workflow saved"); + }, }, ); } @@ -227,7 +230,6 @@ export function WorkflowEditPage() { savePending={save.isPending} saveDisabled={!contentReady} saveError={save.isError ? errorMessage(save.error) : null} - saveSuccess={save.isSuccess} onSave={onSave} onTest={() => setTestOpen(true)} onDuplicate={() => setDuplicateOpen(true)}