feat(migration): migrate legacy owner data and update config reference handling
- Implemented a migration process to move resources from the legacy owner "default" to the new default owner "local", ensuring no data loss during the transition.
- Updated configuration reference handling to replace legacy `$VAR_`, `$SECRET_`, and `$CONTEXT_` prefixes with mustache-style `{{ vars.name }}`, `{{ secrets.name }}`, and `{{ context.name }}`.
- Enhanced YAML configuration files and scripts to reflect the new mustache syntax, improving consistency across the application.
- Added tests to validate the migration process and ensure proper handling of legacy references.
- Updated documentation to guide users on the new configuration reference format.
This commit is contained in:
@@ -120,7 +120,7 @@ Return `{ output, context?, skipRemaining? }`. Do not return `ctx`. Mutations of
|
||||
|---|---|
|
||||
| `ctx.data` | Step input (trigger payload, previous `output`, or DAG `needs`) |
|
||||
| `ctx.context` | Run clipboard (plain object) |
|
||||
| `ctx.config` | YAML `config` (secrets already unwrapped from `$SECRET_name`) |
|
||||
| `ctx.config` | YAML `config` (mustache refs like `{{ secrets.name }}` already resolved) |
|
||||
| `output` | Next step’s `data` |
|
||||
| `context` | Next clipboard. Omit to keep incoming |
|
||||
|
||||
@@ -142,7 +142,7 @@ scripts:
|
||||
script: plugin/my-plugin
|
||||
config:
|
||||
url: http://10.8.0.6:3030/notes
|
||||
token: $SECRET_joplin_api_token
|
||||
token: "{{ secrets.joplin_api_token }}"
|
||||
```
|
||||
|
||||
Optional `name` is the display title in the editor and graph (falls back to the script filename). Canonical ref is `plugin/<id>` (`.js` suffix is optional).
|
||||
@@ -153,4 +153,4 @@ Optional `name` is the display title in the editor and graph (falls back to the
|
||||
- Use an id that matches a core script file (`ntfy`, `jsonata`, …).
|
||||
- Mismatch folder name and `jerapah-plugin.json` `id` (plugin is disabled).
|
||||
- Commit `plugins/.staging-*` or `plugins/*/node_modules/`.
|
||||
- Put secrets in `script.js`; use YAML `$SECRET_name` / `$VAR_name`.
|
||||
- Put secrets in `script.js`; use YAML `{{ secrets.name }}` / `{{ vars.name }}` (quote if the value starts with `{`).
|
||||
|
||||
@@ -83,13 +83,34 @@ Each script is `async function main(ctx)` and **must** return:
|
||||
|---|---|
|
||||
| `ctx.data` | This step’s input (trigger payload, previous `output`, or DAG `needs`) |
|
||||
| `ctx.context` | Run clipboard (plain object, default `{}`) |
|
||||
| `ctx.config` | This step’s YAML config |
|
||||
| `ctx.config` | This step’s YAML config (mustache refs already resolved) |
|
||||
| `output` | Becomes the **next** step’s `data` |
|
||||
| `context` | Next snapshot of the bag. Omitted → keep incoming |
|
||||
| `skipRemaining` | Stop later steps. Sibling of `output`/`context`, not inside `output` |
|
||||
|
||||
Returning the full `ctx` is an error. Mutating `ctx.data` or `ctx.context` does not persist unless returned.
|
||||
|
||||
### Config interpolation
|
||||
|
||||
YAML `config` strings may use mustache paths. Quote values that start with `{`.
|
||||
|
||||
```yaml
|
||||
url: "{{ vars.ntfy_channel }}"
|
||||
token: "{{ secrets.joplin_api_token }}"
|
||||
id: "{{ context.user.id }}"
|
||||
title: "{{ data.httpResponse.data.date }}"
|
||||
topic: "{{ vars.ntfy_prefix }}/{{ data.channel }}"
|
||||
```
|
||||
|
||||
| Root | Meaning |
|
||||
|---|---|
|
||||
| `vars` | Owner variable; remaining segments are the flat name (`{{ vars.foo.bar }}` → variable `foo.bar`) |
|
||||
| `secrets` | Same for secrets |
|
||||
| `context` | Run clipboard (nested) |
|
||||
| `data` | This step’s input (nested; numeric segments index arrays) |
|
||||
|
||||
A string that is exactly one `{{ path }}` keeps the native type (object/array/number/boolean). Mixed strings concatenate as text. Bare name fields such as `passwordSecret: gmail_app_password` stay names for `$secrets.get` — do not wrap them in `{{ secrets.… }}`. Legacy `$VAR_` / `$SECRET_` / `$CONTEXT_` whole-value refs throw; use mustache instead. Script APIs `$vars.get` / `$secrets.get` are unchanged.
|
||||
|
||||
YAML **SET** evaluates JSONata against the full `ctx`; the result is `output` (the next step’s data). `jsonata.js` does the same.
|
||||
|
||||
DAG `needs` assemble this step’s `data` from upstream **outputs**. Independent steps in the same wave share a context snapshot; sibling writes to the same context key fail the run.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
name: Comic - monkeyuser to ntfy
|
||||
description: |
|
||||
Scrape the latest MonkeyUser comic and send it to ntfy (requires $VAR_ntfy_channel).
|
||||
Scrape the latest MonkeyUser comic and send it to ntfy (requires {{ vars.ntfy_channel }}).
|
||||
scripts:
|
||||
- script: fetch-html.js
|
||||
config:
|
||||
@@ -20,7 +20,7 @@ scripts:
|
||||
- script: fetch-binary.js
|
||||
- script: ntfy.js
|
||||
config:
|
||||
url: $VAR_ntfy_channel
|
||||
url: "{{ vars.ntfy_channel }}"
|
||||
triggers:
|
||||
- type: HTTP
|
||||
method: POST
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* Rewrite legacy `$VAR_` / `$SECRET_` / `$CONTEXT_` placeholders to mustache.
|
||||
* Safe for passwordSecret-style fields (those store bare names, not $SECRET_ prefixes).
|
||||
*
|
||||
* @param {string} text
|
||||
* @returns {{ text: string, changed: boolean }}
|
||||
*/
|
||||
export function rewriteLegacyConfigRefsInText(text) {
|
||||
if (typeof text !== "string" || text.length === 0) {
|
||||
return { text: text ?? "", changed: false };
|
||||
}
|
||||
const next = text
|
||||
.replace(/\$VAR_([A-Za-z0-9._-]+)/g, "{{ vars.$1 }}")
|
||||
.replace(/\$SECRET_([A-Za-z0-9._-]+)/g, "{{ secrets.$1 }}")
|
||||
.replace(/\$CONTEXT_([A-Za-z0-9._-]+)/g, "{{ context.$1 }}");
|
||||
return { text: next, changed: next !== text };
|
||||
}
|
||||
+182
-66
@@ -3,36 +3,49 @@ import { assertSecretName, getSecretPlaintext } from "./secrets-store.js";
|
||||
import { isSecret } from "./secret-value.js";
|
||||
import { assertVariableName, getVariablePlain } from "./variables-store.js";
|
||||
|
||||
const PREFIXES = [
|
||||
{ kind: "context", prefix: "$CONTEXT_" },
|
||||
{ kind: "secret", prefix: "$SECRET_" },
|
||||
{ kind: "var", prefix: "$VAR_" },
|
||||
];
|
||||
const FORBIDDEN_SEGMENTS = new Set(["__proto__", "constructor", "prototype"]);
|
||||
const MUSTACHE_TOKEN_RE =
|
||||
/\{\{\s*([A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z0-9_]+)*)\s*\}\}/g;
|
||||
const WHOLE_MUSTACHE_RE =
|
||||
/^\{\{\s*([A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z0-9_]+)*)\s*\}\}$/;
|
||||
const LEGACY_PREFIX_RE = /^\s*\$(VAR|SECRET|CONTEXT)_([A-Za-z0-9._-]*)\s*$/;
|
||||
|
||||
/**
|
||||
* @typedef {{ kind: "secret" | "context" | "var", name: string, raw: string }} ConfigRef
|
||||
* @typedef {{ owner: string, workflowKey: string, context?: unknown }} ConfigRefCtx
|
||||
* @typedef {{
|
||||
* owner: string,
|
||||
* workflowKey?: string,
|
||||
* context?: unknown,
|
||||
* data?: unknown,
|
||||
* }} ConfigRefCtx
|
||||
*/
|
||||
|
||||
/**
|
||||
* Parse a whole-value config placeholder. Returns null for literals.
|
||||
* Detect leftover `$VAR_` / `$SECRET_` / `$CONTEXT_` whole-value refs.
|
||||
* @param {unknown} value
|
||||
* @returns {ConfigRef | null}
|
||||
* @returns {{ kind: "var" | "secret" | "context", name: string, raw: string } | null}
|
||||
*/
|
||||
export function parseConfigRef(value) {
|
||||
export function parseLegacyConfigRef(value) {
|
||||
if (typeof value !== "string") return null;
|
||||
const trimmed = value.trim();
|
||||
for (const { kind, prefix } of PREFIXES) {
|
||||
if (trimmed.startsWith(prefix)) {
|
||||
return { kind, name: trimmed.slice(prefix.length), raw: trimmed };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
const match = LEGACY_PREFIX_RE.exec(value);
|
||||
if (!match) return null;
|
||||
const kind =
|
||||
match[1] === "VAR" ? "var" : match[1] === "SECRET" ? "secret" : "context";
|
||||
return { kind, name: match[2] ?? "", raw: value.trim() };
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk config (objects/arrays) and replace whole-value `$SECRET_` / `$CONTEXT_` / `$VAR_`
|
||||
* strings. Does not walk trigger data.
|
||||
* @param {"var" | "secret" | "context"} kind
|
||||
* @param {string} name
|
||||
*/
|
||||
function legacyRenameHint(kind, name) {
|
||||
if (kind === "var") return `use {{ vars.${name || "name"} }}`;
|
||||
if (kind === "secret") return `use {{ secrets.${name || "name"} }}`;
|
||||
return `use {{ context.${name || "name"} }}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk config (objects/arrays) and interpolate `{{ path }}` strings.
|
||||
* Does not walk trigger data.
|
||||
*
|
||||
* @param {unknown} value
|
||||
* @param {ConfigRefCtx} ctx
|
||||
@@ -68,83 +81,186 @@ export async function resolveConfigRefs(value, ctx, seen = new WeakSet()) {
|
||||
/**
|
||||
* @param {string} value
|
||||
* @param {ConfigRefCtx} ctx
|
||||
* @returns {Promise<string | number | boolean>}
|
||||
* @returns {Promise<unknown>}
|
||||
*/
|
||||
async function resolveStringRef(value, ctx) {
|
||||
const ref = parseConfigRef(value);
|
||||
if (!ref) return value;
|
||||
const legacy = parseLegacyConfigRef(value);
|
||||
if (legacy) {
|
||||
throw new Error(
|
||||
`config ref ${legacy.raw}: removed; ${legacyRenameHint(legacy.kind, legacy.name)}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (ref.kind === "secret") {
|
||||
return resolveSecretRef(ref, ctx);
|
||||
const whole = WHOLE_MUSTACHE_RE.exec(value);
|
||||
if (whole && whole[0] === value) {
|
||||
return resolvePath(whole[1], ctx, { raw: value, allowObject: true });
|
||||
}
|
||||
if (ref.kind === "var") {
|
||||
return resolveVarRef(ref, ctx);
|
||||
|
||||
if (!value.includes("{{")) {
|
||||
return value;
|
||||
}
|
||||
return resolveContextRef(ref, ctx);
|
||||
|
||||
MUSTACHE_TOKEN_RE.lastIndex = 0;
|
||||
let out = "";
|
||||
let lastIndex = 0;
|
||||
let match;
|
||||
while ((match = MUSTACHE_TOKEN_RE.exec(value)) != null) {
|
||||
out += value.slice(lastIndex, match.index);
|
||||
const resolved = await resolvePath(match[1], ctx, {
|
||||
raw: match[0],
|
||||
allowObject: false,
|
||||
});
|
||||
out += stringifyScalar(resolved, match[0]);
|
||||
lastIndex = match.index + match[0].length;
|
||||
}
|
||||
out += value.slice(lastIndex);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ConfigRef} ref
|
||||
* @param {string} pathExpr
|
||||
* @param {ConfigRefCtx} ctx
|
||||
* @returns {Promise<string>}
|
||||
* @param {{ raw: string, allowObject: boolean }} opts
|
||||
*/
|
||||
async function resolveSecretRef(ref, ctx) {
|
||||
try {
|
||||
assertSecretName(ref.name);
|
||||
} catch {
|
||||
throw new Error(`config ref ${ref.raw}: invalid secret name`);
|
||||
async function resolvePath(pathExpr, ctx, opts) {
|
||||
const segments = pathExpr.split(".");
|
||||
if (segments.length === 0 || segments.some((s) => !s)) {
|
||||
throw new Error(`config ref ${opts.raw}: empty path`);
|
||||
}
|
||||
const plaintext = await getSecretPlaintext(ctx.owner, ref.name);
|
||||
if (plaintext == null) {
|
||||
throw new Error(`config ref ${ref.raw}: secret "${ref.name}" not found`);
|
||||
for (const seg of segments) {
|
||||
if (FORBIDDEN_SEGMENTS.has(seg)) {
|
||||
throw new Error(`config ref ${opts.raw}: forbidden path segment "${seg}"`);
|
||||
}
|
||||
}
|
||||
return plaintext;
|
||||
|
||||
const root = segments[0];
|
||||
const rest = segments.slice(1);
|
||||
|
||||
if (root === "vars") {
|
||||
return resolveNamedStore("var", rest, ctx, opts);
|
||||
}
|
||||
if (root === "secrets") {
|
||||
return resolveNamedStore("secret", rest, ctx, opts);
|
||||
}
|
||||
if (root === "context") {
|
||||
return walkObject(ctx.context, rest, opts);
|
||||
}
|
||||
if (root === "data") {
|
||||
return walkObject(ctx.data, rest, opts);
|
||||
}
|
||||
throw new Error(
|
||||
`config ref ${opts.raw}: unknown root "${root}" (use vars, secrets, context, or data)`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ConfigRef} ref
|
||||
* @param {"var" | "secret"} kind
|
||||
* @param {string[]} rest
|
||||
* @param {ConfigRefCtx} ctx
|
||||
* @returns {Promise<string | number | boolean>}
|
||||
* @param {{ raw: string, allowObject: boolean }} opts
|
||||
*/
|
||||
async function resolveVarRef(ref, ctx) {
|
||||
if (ref.name.length === 0) {
|
||||
throw new Error(`config ref ${ref.raw}: empty variable name`);
|
||||
async function resolveNamedStore(kind, rest, ctx, opts) {
|
||||
if (rest.length === 0) {
|
||||
throw new Error(`config ref ${opts.raw}: empty ${kind} name`);
|
||||
}
|
||||
const name = rest.join(".");
|
||||
try {
|
||||
assertVariableName(ref.name);
|
||||
if (kind === "secret") assertSecretName(name);
|
||||
else assertVariableName(name);
|
||||
} catch {
|
||||
throw new Error(`config ref ${ref.raw}: invalid variable name`);
|
||||
throw new Error(`config ref ${opts.raw}: invalid ${kind} name`);
|
||||
}
|
||||
const value = await getVariablePlain(ctx.owner, ref.name);
|
||||
|
||||
if (kind === "secret") {
|
||||
const plaintext = await getSecretPlaintext(ctx.owner, name);
|
||||
if (plaintext == null) {
|
||||
throw new Error(`config ref ${opts.raw}: secret "${name}" not found`);
|
||||
}
|
||||
return plaintext;
|
||||
}
|
||||
|
||||
const value = await getVariablePlain(ctx.owner, name);
|
||||
if (value == null) {
|
||||
throw new Error(`config ref ${ref.raw}: variable "${ref.name}" not found`);
|
||||
throw new Error(`config ref ${opts.raw}: variable "${name}" not found`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ConfigRef} ref
|
||||
* @param {ConfigRefCtx} ctx
|
||||
* @returns {string}
|
||||
* @param {unknown} root
|
||||
* @param {string[]} rest
|
||||
* @param {{ raw: string, allowObject: boolean }} opts
|
||||
*/
|
||||
function resolveContextRef(ref, ctx) {
|
||||
if (ref.name.length === 0) {
|
||||
throw new Error(`config ref ${ref.raw}: empty context key`);
|
||||
function walkObject(root, rest, opts) {
|
||||
if (rest.length === 0) {
|
||||
return unwrapValue(root, opts);
|
||||
}
|
||||
const bag =
|
||||
ctx.context != null && typeof ctx.context === "object" && !Array.isArray(ctx.context)
|
||||
? /** @type {Record<string, unknown>} */ (ctx.context)
|
||||
: {};
|
||||
if (!Object.prototype.hasOwnProperty.call(bag, ref.name)) {
|
||||
throw new Error(`config ref ${ref.raw}: context "${ref.name}" not found`);
|
||||
|
||||
let cur = root;
|
||||
for (const seg of rest) {
|
||||
if (cur == null || typeof cur !== "object") {
|
||||
throw new Error(`config ref ${opts.raw}: path not found`);
|
||||
}
|
||||
if (Array.isArray(cur)) {
|
||||
if (!/^\d+$/.test(seg)) {
|
||||
throw new Error(`config ref ${opts.raw}: path not found`);
|
||||
}
|
||||
const idx = Number(seg);
|
||||
if (!Number.isInteger(idx) || idx < 0 || idx >= cur.length) {
|
||||
throw new Error(`config ref ${opts.raw}: path not found`);
|
||||
}
|
||||
cur = cur[idx];
|
||||
continue;
|
||||
}
|
||||
const bag = /** @type {Record<string, unknown>} */ (cur);
|
||||
if (!Object.prototype.hasOwnProperty.call(bag, seg)) {
|
||||
throw new Error(`config ref ${opts.raw}: path not found`);
|
||||
}
|
||||
cur = bag[seg];
|
||||
}
|
||||
const raw = bag[ref.name];
|
||||
if (isSecret(raw)) {
|
||||
return raw.reveal();
|
||||
return unwrapValue(cur, opts);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {unknown} value
|
||||
* @param {{ raw: string, allowObject: boolean }} opts
|
||||
*/
|
||||
function unwrapValue(value, opts) {
|
||||
if (isSecret(value)) {
|
||||
return value.reveal();
|
||||
}
|
||||
const coerced = coerceCredentialString(raw);
|
||||
if (coerced == null) {
|
||||
throw new Error(`config ref ${ref.raw}: context "${ref.name}" is not a scalar`);
|
||||
if (!opts.allowObject && value != null && typeof value === "object") {
|
||||
throw new Error(`config ref ${opts.raw}: value is not a scalar`);
|
||||
}
|
||||
return coerced;
|
||||
// Whole-value context/data may be any JSON type; mixed strings need scalars only.
|
||||
if (opts.allowObject) {
|
||||
if (value != null && typeof value === "object") return value;
|
||||
if (
|
||||
typeof value === "string" ||
|
||||
typeof value === "number" ||
|
||||
typeof value === "boolean"
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
// Prefer credential coercion for odd primitives (e.g. bigint) when whole-value.
|
||||
const coerced = coerceCredentialString(value);
|
||||
if (coerced != null) return coerced;
|
||||
return value;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {unknown} value
|
||||
* @param {string} raw
|
||||
*/
|
||||
function stringifyScalar(value, raw) {
|
||||
if (value == null) {
|
||||
throw new Error(`config ref ${raw}: value is null`);
|
||||
}
|
||||
if (typeof value === "string") return value;
|
||||
if (typeof value === "number" || typeof value === "boolean") {
|
||||
return String(value);
|
||||
}
|
||||
throw new Error(`config ref ${raw}: value is not a scalar`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
/**
|
||||
* No-op Knex marker: owner + config-ref migrate runs in owner-migrate.js at startup
|
||||
* (needs filesystem + DB together). Kept so deploy tooling sees a versioned step.
|
||||
*
|
||||
* @param {import("knex").Knex} _knex
|
||||
*/
|
||||
export async function up(_knex) {
|
||||
// Intentionally empty — see migrateDefaultOwnerIfNeeded().
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("knex").Knex} _knex
|
||||
*/
|
||||
export async function down(_knex) {
|
||||
// Irreversible data migrate.
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { DEFAULT_OWNER } from "@jerapah-flow/shared";
|
||||
import { db } from "./db.js";
|
||||
import { log } from "./logger.js";
|
||||
import { EXAMPLE_WORKFLOWS_DIR, WORKFLOWS_DIR } from "./paths.js";
|
||||
import { TRASH_WORKFLOWS_DIR } from "./workflow-trash.js";
|
||||
import { rewriteLegacyConfigRefsInText } from "./config-ref-rewrite.js";
|
||||
|
||||
const LEGACY_OWNER = "default";
|
||||
|
||||
/**
|
||||
* @param {string} dir
|
||||
* @returns {string[]}
|
||||
*/
|
||||
function listFilesRecursive(dir) {
|
||||
if (!fs.existsSync(dir)) return [];
|
||||
/** @type {string[]} */
|
||||
const out = [];
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) out.push(...listFilesRecursive(full));
|
||||
else out.push(full);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Move files from legacy owner dir into DEFAULT_OWNER. Skip name collisions.
|
||||
* @param {string} rootDir
|
||||
* @returns {{ moved: number, skipped: number }}
|
||||
*/
|
||||
function mergeOwnerDir(rootDir) {
|
||||
const fromDir = path.join(rootDir, LEGACY_OWNER);
|
||||
const toDir = path.join(rootDir, DEFAULT_OWNER);
|
||||
if (!fs.existsSync(fromDir)) return { moved: 0, skipped: 0 };
|
||||
|
||||
fs.mkdirSync(toDir, { recursive: true });
|
||||
let moved = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const name of fs.readdirSync(fromDir)) {
|
||||
const from = path.join(fromDir, name);
|
||||
const to = path.join(toDir, name);
|
||||
const st = fs.statSync(from);
|
||||
if (!st.isFile()) {
|
||||
skipped += 1;
|
||||
log.warn({ from }, "owner migrate: skip non-file under legacy owner dir");
|
||||
continue;
|
||||
}
|
||||
if (fs.existsSync(to)) {
|
||||
skipped += 1;
|
||||
log.warn(
|
||||
{ from, to },
|
||||
"owner migrate: skip YAML collision (local already has file)",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
fs.renameSync(from, to);
|
||||
moved += 1;
|
||||
}
|
||||
|
||||
const remaining = fs.existsSync(fromDir) ? fs.readdirSync(fromDir) : [];
|
||||
if (remaining.length === 0 && fs.existsSync(fromDir)) {
|
||||
fs.rmdirSync(fromDir);
|
||||
}
|
||||
|
||||
return { moved, skipped };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} filePath
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function rewriteFileInPlace(filePath) {
|
||||
const raw = fs.readFileSync(filePath, "utf8");
|
||||
const { text, changed } = rewriteLegacyConfigRefsInText(raw);
|
||||
if (!changed) return false;
|
||||
fs.writeFileSync(filePath, text, "utf8");
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} rootDir
|
||||
* @returns {number}
|
||||
*/
|
||||
function rewriteYamlTree(rootDir) {
|
||||
let n = 0;
|
||||
for (const file of listFilesRecursive(rootDir)) {
|
||||
if (!/\.ya?ml$/i.test(file)) continue;
|
||||
if (rewriteFileInPlace(file)) n += 1;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("knex").Knex} knex
|
||||
* @param {string} table
|
||||
* @param {"name" | "file" | null} uniqueCol
|
||||
*/
|
||||
async function migrateOwnerColumn(knex, table, uniqueCol) {
|
||||
const legacyRows = await knex(table).where({ owner: LEGACY_OWNER }).select("*");
|
||||
let moved = 0;
|
||||
let skipped = 0;
|
||||
for (const row of legacyRows) {
|
||||
if (uniqueCol != null) {
|
||||
const conflict = await knex(table)
|
||||
.where({ owner: DEFAULT_OWNER, [uniqueCol]: row[uniqueCol] })
|
||||
.first();
|
||||
if (conflict) {
|
||||
skipped += 1;
|
||||
log.warn(
|
||||
{ table, id: row.id, [uniqueCol]: row[uniqueCol] },
|
||||
"owner migrate: skip row collision",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
await knex(table).where({ id: row.id }).update({ owner: DEFAULT_OWNER });
|
||||
moved += 1;
|
||||
}
|
||||
return { moved, skipped };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("knex").Knex} knex
|
||||
*/
|
||||
async function migrateWorkflowRuns(knex) {
|
||||
const n = await knex("workflow_runs")
|
||||
.where({ owner: LEGACY_OWNER })
|
||||
.update({ owner: DEFAULT_OWNER });
|
||||
return { moved: Number(n) || 0, skipped: 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("knex").Knex} knex
|
||||
*/
|
||||
async function migrateWorkflowRevisionsOwner(knex) {
|
||||
const n = await knex("workflow_revisions")
|
||||
.where({ owner: LEGACY_OWNER })
|
||||
.update({ owner: DEFAULT_OWNER });
|
||||
return { moved: Number(n) || 0, skipped: 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("knex").Knex} knex
|
||||
*/
|
||||
async function migrateScriptStateNamespaces(knex) {
|
||||
const rows = await knex("script_state")
|
||||
.where("namespace", "like", `${LEGACY_OWNER}/%`)
|
||||
.select("namespace", "key");
|
||||
let moved = 0;
|
||||
let skipped = 0;
|
||||
for (const row of rows) {
|
||||
const nextNs = `${DEFAULT_OWNER}${row.namespace.slice(LEGACY_OWNER.length)}`;
|
||||
const conflict = await knex("script_state")
|
||||
.where({ namespace: nextNs, key: row.key })
|
||||
.first();
|
||||
if (conflict) {
|
||||
skipped += 1;
|
||||
log.warn(
|
||||
{ namespace: row.namespace, key: row.key, nextNs },
|
||||
"owner migrate: skip script_state collision",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
await knex("script_state")
|
||||
.where({ namespace: row.namespace, key: row.key })
|
||||
.update({ namespace: nextNs });
|
||||
moved += 1;
|
||||
}
|
||||
return { moved, skipped };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("knex").Knex} knex
|
||||
*/
|
||||
async function rewriteDbConfigStrings(knex) {
|
||||
let profiles = 0;
|
||||
let revisions = 0;
|
||||
|
||||
const profileRows = await knex("profiles").select("id", "config");
|
||||
for (const row of profileRows) {
|
||||
const { text, changed } = rewriteLegacyConfigRefsInText(String(row.config ?? ""));
|
||||
if (!changed) continue;
|
||||
await knex("profiles").where({ id: row.id }).update({ config: text });
|
||||
profiles += 1;
|
||||
}
|
||||
|
||||
const revisionRows = await knex("workflow_revisions").select("id", "content");
|
||||
for (const row of revisionRows) {
|
||||
const { text, changed } = rewriteLegacyConfigRefsInText(String(row.content ?? ""));
|
||||
if (!changed) continue;
|
||||
await knex("workflow_revisions").where({ id: row.id }).update({ content: text });
|
||||
revisions += 1;
|
||||
}
|
||||
|
||||
return { profiles, revisions };
|
||||
}
|
||||
|
||||
/**
|
||||
* One-shot: move owner `default` → `local`, rewrite prefix refs to mustache.
|
||||
* Idempotent when there is no remaining `default` data / prefix refs.
|
||||
*/
|
||||
export async function migrateDefaultOwnerIfNeeded() {
|
||||
const knex = db;
|
||||
|
||||
const yamlLive = mergeOwnerDir(WORKFLOWS_DIR);
|
||||
const yamlTrash = mergeOwnerDir(TRASH_WORKFLOWS_DIR);
|
||||
|
||||
const variables = await migrateOwnerColumn(knex, "variables", "name");
|
||||
const secrets = await migrateOwnerColumn(knex, "secrets", "name");
|
||||
const profiles = await migrateOwnerColumn(knex, "profiles", "name");
|
||||
const trash = await migrateOwnerColumn(knex, "workflow_trash", "file");
|
||||
const runs = await migrateWorkflowRuns(knex);
|
||||
const revisionsOwner = await migrateWorkflowRevisionsOwner(knex);
|
||||
const scriptState = await migrateScriptStateNamespaces(knex);
|
||||
|
||||
const yamlRewritten =
|
||||
rewriteYamlTree(path.join(WORKFLOWS_DIR, DEFAULT_OWNER)) +
|
||||
rewriteYamlTree(path.join(TRASH_WORKFLOWS_DIR, DEFAULT_OWNER)) +
|
||||
rewriteYamlTree(path.join(WORKFLOWS_DIR, LEGACY_OWNER)) +
|
||||
rewriteYamlTree(path.join(TRASH_WORKFLOWS_DIR, LEGACY_OWNER));
|
||||
|
||||
let examplesRewritten = 0;
|
||||
if (fs.existsSync(EXAMPLE_WORKFLOWS_DIR)) {
|
||||
examplesRewritten = rewriteYamlTree(EXAMPLE_WORKFLOWS_DIR);
|
||||
}
|
||||
|
||||
const dbStrings = await rewriteDbConfigStrings(knex);
|
||||
|
||||
const summary = {
|
||||
yamlLive,
|
||||
yamlTrash,
|
||||
variables,
|
||||
secrets,
|
||||
profiles,
|
||||
trash,
|
||||
runs,
|
||||
revisionsOwner,
|
||||
scriptState,
|
||||
yamlRewritten,
|
||||
examplesRewritten,
|
||||
dbStrings,
|
||||
};
|
||||
|
||||
const touched =
|
||||
yamlLive.moved +
|
||||
yamlTrash.moved +
|
||||
variables.moved +
|
||||
secrets.moved +
|
||||
profiles.moved +
|
||||
trash.moved +
|
||||
runs.moved +
|
||||
revisionsOwner.moved +
|
||||
scriptState.moved +
|
||||
yamlRewritten +
|
||||
examplesRewritten +
|
||||
dbStrings.profiles +
|
||||
dbStrings.revisions >
|
||||
0;
|
||||
|
||||
if (touched) {
|
||||
log.info(summary, "migrated owner default → local and rewrote config refs");
|
||||
}
|
||||
|
||||
return summary;
|
||||
}
|
||||
@@ -17,7 +17,9 @@
|
||||
"test:workflow-history": "JFLOW_WORKFLOWS_DIR=./data/workflow-history-smoke JFLOW_DB_PATH=./data/workflow-history-smoke.db node test/workflow-history-smoke.js",
|
||||
"reset-admin": "node reset-admin.js",
|
||||
"test:profiles": "node test/profiles-smoke.js",
|
||||
"test:set-dry-run": "node test/set-dry-run-smoke.js"
|
||||
"test:set-dry-run": "node test/set-dry-run-smoke.js",
|
||||
"test:config-refs": "node test/config-refs-smoke.js",
|
||||
"test:owner-migrate": "node test/owner-migrate-smoke.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@jerapah-flow/shared": "workspace:*",
|
||||
|
||||
@@ -511,6 +511,7 @@ export function createRegistry(server, opts = {}) {
|
||||
owner,
|
||||
workflowKey: key,
|
||||
context: incomingContext,
|
||||
data: ctx.data,
|
||||
});
|
||||
const stepCtx = {
|
||||
data: ctx.data,
|
||||
|
||||
@@ -11,6 +11,7 @@ import { isSecret, Secret, unwrapSecretsDeep } from "./secret-value.js";
|
||||
import { getHttpPageByName, getHttpTemplateByName } from "./http-pages-store.js";
|
||||
import { getSecretPlaintext } from "./secrets-store.js";
|
||||
import { getVariablePlain } from "./variables-store.js";
|
||||
import { DEFAULT_OWNER } from "@jerapah-flow/shared";
|
||||
|
||||
const hostRequire = createRequire(import.meta.url);
|
||||
|
||||
@@ -443,7 +444,7 @@ function createScriptSandbox({
|
||||
log,
|
||||
script,
|
||||
workflowName,
|
||||
owner = "default",
|
||||
owner = DEFAULT_OWNER,
|
||||
$workflows = $workflowsStub,
|
||||
pluginDir = null,
|
||||
}) {
|
||||
@@ -563,7 +564,7 @@ export function instantiateScriptSource(script, source, opts = {}) {
|
||||
log: opts.log ?? inspectLog,
|
||||
script,
|
||||
workflowName: opts.workflowName ?? "inspect",
|
||||
owner: opts.owner ?? "default",
|
||||
owner: opts.owner ?? DEFAULT_OWNER,
|
||||
$workflows: opts.$workflows,
|
||||
pluginDir: opts.pluginDir ?? null,
|
||||
});
|
||||
|
||||
@@ -31,6 +31,7 @@ import { getAppVersion } from "../../app-version.js";
|
||||
import { EXAMPLE_PLUGINS_DIR } from "../../paths.js";
|
||||
import { pluginScriptRef } from "../../plugin-manifest.js";
|
||||
import { evaluateJsonata, SET_STEP_SCRIPT } from "../../workflow-parse.js";
|
||||
import { DEFAULT_OWNER } from "@jerapah-flow/shared";
|
||||
|
||||
/**
|
||||
* @param {{ referencedScripts: () => Set<string> }} registry
|
||||
@@ -258,7 +259,7 @@ export default function scriptsPluginFactory(registry) {
|
||||
req.body ?? {}
|
||||
);
|
||||
|
||||
let owner = "local";
|
||||
let owner = DEFAULT_OWNER;
|
||||
if (body.owner != null && body.owner !== "") {
|
||||
try {
|
||||
owner = fsStore.assertOwner(String(body.owner));
|
||||
@@ -304,6 +305,7 @@ export default function scriptsPluginFactory(registry) {
|
||||
owner,
|
||||
workflowKey: "dry-run",
|
||||
context: incomingContext,
|
||||
data: incomingData,
|
||||
},
|
||||
);
|
||||
const ctx = {
|
||||
@@ -365,6 +367,7 @@ export default function scriptsPluginFactory(registry) {
|
||||
owner,
|
||||
workflowKey: "dry-run",
|
||||
context: incomingContext,
|
||||
data: incomingData,
|
||||
});
|
||||
const ctx = {
|
||||
data: incomingData,
|
||||
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
} from "./workflow-queue.js";
|
||||
import { purgeExpiredTrash } from "./workflow-trash.js";
|
||||
import { migrateLegacyWorkflowsIfNeeded } from "./workflow-migrate.js";
|
||||
import { migrateDefaultOwnerIfNeeded } from "./owner-migrate.js";
|
||||
import {
|
||||
getConfigGeneration,
|
||||
startHeartbeatLoop,
|
||||
@@ -135,6 +136,12 @@ export async function startApp(opts = {}) {
|
||||
log.warn({ err }, "legacy workflow migrate failed");
|
||||
}
|
||||
|
||||
try {
|
||||
await migrateDefaultOwnerIfNeeded();
|
||||
} catch (err) {
|
||||
log.warn({ err }, "owner default→local migrate failed");
|
||||
}
|
||||
|
||||
const registry = createRegistry(server, {
|
||||
queue: workflowQueue,
|
||||
// Cron + HTTP triggers enqueue jobs; only the API process may own them.
|
||||
|
||||
@@ -2,7 +2,8 @@ import { migrate, db } from "../db.js";
|
||||
import { upsertSecret, deleteSecret } from "../secrets-store.js";
|
||||
import { deleteVariable, upsertVariable } from "../variables-store.js";
|
||||
import { Secret } from "../secret-value.js";
|
||||
import { parseConfigRef, resolveConfigRefs } from "../config-refs.js";
|
||||
import { parseLegacyConfigRef, resolveConfigRefs } from "../config-refs.js";
|
||||
import { rewriteLegacyConfigRefsInText } from "../config-ref-rewrite.js";
|
||||
|
||||
await migrate();
|
||||
|
||||
@@ -23,126 +24,171 @@ async function assertRejects(fn, match) {
|
||||
throw new Error(`expected to reject (${match ?? "any error"})`);
|
||||
}
|
||||
|
||||
const owner = "default";
|
||||
const workflowKey = "default/config-refs-smoke.yaml";
|
||||
const ctx = { owner, workflowKey, context: {} };
|
||||
const owner = "config_refs_smoke_owner";
|
||||
const ctx = { owner, workflowKey: `${owner}/config-refs-smoke.yaml`, context: {}, data: {} };
|
||||
|
||||
function assertParse(value, expected) {
|
||||
const got = parseConfigRef(value);
|
||||
if (expected == null) {
|
||||
assert(got == null, `expected null parse for ${JSON.stringify(value)}, got ${JSON.stringify(got)}`);
|
||||
return;
|
||||
}
|
||||
assert(got != null, `expected parse for ${JSON.stringify(value)}`);
|
||||
assert(got.kind === expected.kind, `kind ${got.kind} !== ${expected.kind}`);
|
||||
assert(got.name === expected.name, `name ${JSON.stringify(got.name)} !== ${JSON.stringify(expected.name)}`);
|
||||
{
|
||||
const rewritten = rewriteLegacyConfigRefsInText(
|
||||
'url: $VAR_ntfy\ntoken: $SECRET_tok\nid: $CONTEXT_user',
|
||||
);
|
||||
assert(rewritten.changed, "rewrite detects legacy refs");
|
||||
assert(
|
||||
rewritten.text.includes("{{ vars.ntfy }}") &&
|
||||
rewritten.text.includes("{{ secrets.tok }}") &&
|
||||
rewritten.text.includes("{{ context.user }}"),
|
||||
"rewrite maps prefixes",
|
||||
);
|
||||
assert(!rewriteLegacyConfigRefsInText("passwordSecret: gmail_app").changed, "bare names untouched");
|
||||
}
|
||||
|
||||
assertParse("password123", null);
|
||||
assertParse("$FOO_bar", null);
|
||||
assertParse("$SECRET", null);
|
||||
assertParse(" password123 ", null);
|
||||
assertParse("$SECRET_zte_modem_password", { kind: "secret", name: "zte_modem_password" });
|
||||
assertParse(" $SECRET_zte_modem_password ", { kind: "secret", name: "zte_modem_password" });
|
||||
assertParse("Bearer $SECRET_x", null);
|
||||
assertParse("$KV_modem password", null);
|
||||
assertParse("$VAR_ntfy_url", { kind: "var", name: "ntfy_url" });
|
||||
assertParse("$CONTEXT_token", { kind: "context", name: "token" });
|
||||
assertParse("$SECRET_", { kind: "secret", name: "" });
|
||||
assertParse("$CONTEXT_SECRET_foo", { kind: "context", name: "SECRET_foo" });
|
||||
assert(parseLegacyConfigRef("$VAR_ntfy_url")?.kind === "var", "legacy var parse");
|
||||
assert(parseLegacyConfigRef("$SECRET_x")?.kind === "secret", "legacy secret parse");
|
||||
assert(parseLegacyConfigRef("$CONTEXT_token")?.kind === "context", "legacy context parse");
|
||||
assert(parseLegacyConfigRef("{{ vars.x }}") == null, "mustache is not legacy");
|
||||
assert(parseLegacyConfigRef("Bearer $SECRET_x") == null, "mid-string not whole-value legacy");
|
||||
|
||||
{
|
||||
const literal = await resolveConfigRefs("password123", ctx);
|
||||
assert(literal === "password123", "literal passthrough");
|
||||
const unknown = await resolveConfigRefs("$FOO_bar", ctx);
|
||||
assert(unknown === "$FOO_bar", "$FOO_bar stays literal");
|
||||
const kvLiteral = await resolveConfigRefs("$KV_modem_password", ctx);
|
||||
assert(kvLiteral === "$KV_modem_password", "$KV_ stays literal");
|
||||
const embedded = await resolveConfigRefs("Bearer $SECRET_x", ctx);
|
||||
assert(embedded === "Bearer $SECRET_x", "mid-string stays literal");
|
||||
assert(embedded === "Bearer $SECRET_x", "mid-string legacy stays literal");
|
||||
const number = await resolveConfigRefs(42, ctx);
|
||||
assert(number === 42, "number passthrough");
|
||||
}
|
||||
|
||||
const secret = await upsertSecret({
|
||||
owner,
|
||||
name: "config_refs_smoke_token",
|
||||
value: "s3cret-ok",
|
||||
});
|
||||
const varUrl = await upsertVariable({
|
||||
owner,
|
||||
name: "config_refs_smoke_url",
|
||||
type: "string",
|
||||
value: "https://example.test",
|
||||
});
|
||||
const varRetry = await upsertVariable({
|
||||
owner,
|
||||
name: "config_refs_smoke_retry",
|
||||
type: "number",
|
||||
value: 3,
|
||||
});
|
||||
const varDebug = await upsertVariable({
|
||||
owner,
|
||||
name: "config_refs_smoke_debug",
|
||||
type: "boolean",
|
||||
value: false,
|
||||
});
|
||||
await assertRejects(
|
||||
() => resolveConfigRefs("$VAR_ntfy_channel", ctx),
|
||||
"removed",
|
||||
);
|
||||
await assertRejects(
|
||||
() => resolveConfigRefs("$SECRET_tok", ctx),
|
||||
"use {{ secrets.tok }}",
|
||||
);
|
||||
await assertRejects(
|
||||
() => resolveConfigRefs("$CONTEXT_token", ctx),
|
||||
"use {{ context.token }}",
|
||||
);
|
||||
|
||||
const created = [];
|
||||
try {
|
||||
const secret = await upsertSecret({
|
||||
owner,
|
||||
name: "config_refs_smoke_token",
|
||||
value: "s3cret-ok",
|
||||
});
|
||||
created.push(["secret", secret.id]);
|
||||
|
||||
const varUrl = await upsertVariable({
|
||||
owner,
|
||||
name: "config_refs_smoke_url",
|
||||
type: "string",
|
||||
value: "https://example.test",
|
||||
});
|
||||
created.push(["var", varUrl.id]);
|
||||
|
||||
const varRetry = await upsertVariable({
|
||||
owner,
|
||||
name: "config_refs_smoke_retry",
|
||||
type: "number",
|
||||
value: 3,
|
||||
});
|
||||
created.push(["var", varRetry.id]);
|
||||
|
||||
const varDebug = await upsertVariable({
|
||||
owner,
|
||||
name: "config_refs_smoke_debug",
|
||||
type: "boolean",
|
||||
value: false,
|
||||
});
|
||||
created.push(["var", varDebug.id]);
|
||||
|
||||
{
|
||||
const resolved = await resolveConfigRefs("$SECRET_config_refs_smoke_token", ctx);
|
||||
assert(resolved === "s3cret-ok", "secret resolve");
|
||||
const resolved = await resolveConfigRefs("{{ secrets.config_refs_smoke_token }}", ctx);
|
||||
assert(resolved === "s3cret-ok", "secret whole-value");
|
||||
}
|
||||
{
|
||||
const resolved = await resolveConfigRefs(" $SECRET_config_refs_smoke_token ", ctx);
|
||||
assert(resolved === "s3cret-ok", "secret resolve trimmed");
|
||||
}
|
||||
{
|
||||
const resolved = await resolveConfigRefs("$VAR_config_refs_smoke_url", ctx);
|
||||
const resolved = await resolveConfigRefs("{{ vars.config_refs_smoke_url }}", ctx);
|
||||
assert(resolved === "https://example.test", "var string");
|
||||
}
|
||||
{
|
||||
const resolved = await resolveConfigRefs("$VAR_config_refs_smoke_retry", ctx);
|
||||
assert(resolved === 3, "var number stays number");
|
||||
assert(typeof resolved === "number", "var number type");
|
||||
const resolved = await resolveConfigRefs("{{ vars.config_refs_smoke_retry }}", ctx);
|
||||
assert(resolved === 3, "var number keeps type");
|
||||
}
|
||||
{
|
||||
const resolved = await resolveConfigRefs("$VAR_config_refs_smoke_debug", ctx);
|
||||
assert(resolved === false, "var boolean stays false");
|
||||
assert(typeof resolved === "boolean", "var boolean type");
|
||||
const resolved = await resolveConfigRefs("{{ vars.config_refs_smoke_debug }}", ctx);
|
||||
assert(resolved === false, "var boolean keeps type");
|
||||
}
|
||||
{
|
||||
const resolved = await resolveConfigRefs("$CONTEXT_token", {
|
||||
const resolved = await resolveConfigRefs("{{ context.token }}", {
|
||||
...ctx,
|
||||
context: { token: "ctx-token-ok" },
|
||||
});
|
||||
assert(resolved === "ctx-token-ok", "context string");
|
||||
}
|
||||
{
|
||||
const resolved = await resolveConfigRefs("$CONTEXT_n", {
|
||||
const resolved = await resolveConfigRefs("{{ context.n }}", {
|
||||
...ctx,
|
||||
context: { n: 7 },
|
||||
});
|
||||
assert(resolved === "7", "context number stringify");
|
||||
assert(resolved === 7, "context number keeps type");
|
||||
}
|
||||
{
|
||||
const wrapped = new Secret("wrapped-secret-ok");
|
||||
const resolved = await resolveConfigRefs("$CONTEXT_tok", {
|
||||
const resolved = await resolveConfigRefs("{{ context.tok }}", {
|
||||
...ctx,
|
||||
context: { tok: wrapped },
|
||||
});
|
||||
assert(resolved === "wrapped-secret-ok", "context Secret unwrap");
|
||||
}
|
||||
|
||||
{
|
||||
const resolved = await resolveConfigRefs("{{ context.user }}", {
|
||||
...ctx,
|
||||
context: { user: { id: "u1", role: "admin" } },
|
||||
});
|
||||
assert(
|
||||
resolved && typeof resolved === "object" && resolved.id === "u1",
|
||||
"whole-value object pass-through",
|
||||
);
|
||||
}
|
||||
{
|
||||
const resolved = await resolveConfigRefs("{{ context.user.id }}", {
|
||||
...ctx,
|
||||
context: { user: { id: "nested-id" } },
|
||||
});
|
||||
assert(resolved === "nested-id", "nested context path");
|
||||
}
|
||||
{
|
||||
const resolved = await resolveConfigRefs("{{ data.items.0.id }}", {
|
||||
...ctx,
|
||||
data: { items: [{ id: "row-0" }] },
|
||||
});
|
||||
assert(resolved === "row-0", "array index path");
|
||||
}
|
||||
{
|
||||
const resolved = await resolveConfigRefs(
|
||||
"{{ vars.config_refs_smoke_url }}/{{ data.channel }}",
|
||||
{ ...ctx, data: { channel: "alerts" } },
|
||||
);
|
||||
assert(resolved === "https://example.test/alerts", "concatenation");
|
||||
}
|
||||
{
|
||||
const resolved = await resolveConfigRefs("Bearer {{ context.token }}", {
|
||||
...ctx,
|
||||
context: { token: "abc" },
|
||||
});
|
||||
assert(resolved === "Bearer abc", "mixed string");
|
||||
}
|
||||
{
|
||||
const nested = await resolveConfigRefs(
|
||||
{
|
||||
url: "$VAR_config_refs_smoke_url",
|
||||
retry: "$VAR_config_refs_smoke_retry",
|
||||
debug: "$VAR_config_refs_smoke_debug",
|
||||
password: "$SECRET_config_refs_smoke_token",
|
||||
url: "{{ vars.config_refs_smoke_url }}",
|
||||
retry: "{{ vars.config_refs_smoke_retry }}",
|
||||
debug: "{{ vars.config_refs_smoke_debug }}",
|
||||
password: "{{ secrets.config_refs_smoke_token }}",
|
||||
headers: { Authorization: "$KV_modem_password" },
|
||||
extra: ["$CONTEXT_token", "plain"],
|
||||
extra: ["{{ context.token }}", "plain"],
|
||||
},
|
||||
{ ...ctx, context: { token: "ctx-token-ok" } },
|
||||
);
|
||||
@@ -155,59 +201,47 @@ try {
|
||||
assert(nested.extra[1] === "plain", "nested array literal");
|
||||
}
|
||||
|
||||
const data = { password: "$SECRET_config_refs_smoke_token" };
|
||||
const config = { password: "$SECRET_config_refs_smoke_token" };
|
||||
const data = { password: "{{ secrets.config_refs_smoke_token }}" };
|
||||
const config = { password: "{{ secrets.config_refs_smoke_token }}" };
|
||||
const resolvedConfig = await resolveConfigRefs(config, ctx);
|
||||
assert(resolvedConfig.password === "s3cret-ok", "config resolved");
|
||||
assert(data.password === "$SECRET_config_refs_smoke_token", "data not walked");
|
||||
assert(config.password === "$SECRET_config_refs_smoke_token", "input config not mutated");
|
||||
assert(data.password === "{{ secrets.config_refs_smoke_token }}", "data not walked");
|
||||
assert(config.password === "{{ secrets.config_refs_smoke_token }}", "input config not mutated");
|
||||
|
||||
await assertRejects(
|
||||
() => resolveConfigRefs("$SECRET_does_not_exist_xyz", ctx),
|
||||
() => resolveConfigRefs("{{ secrets.does_not_exist_xyz }}", ctx),
|
||||
'secret "does_not_exist_xyz" not found',
|
||||
);
|
||||
await assertRejects(
|
||||
() => resolveConfigRefs("$SECRET_not valid", ctx),
|
||||
"invalid secret name",
|
||||
);
|
||||
await assertRejects(
|
||||
() => resolveConfigRefs("$SECRET_", ctx),
|
||||
"invalid secret name",
|
||||
);
|
||||
await assertRejects(
|
||||
() => resolveConfigRefs("$CONTEXT_missing", ctx),
|
||||
'context "missing" not found',
|
||||
() => resolveConfigRefs("{{ context.missing }}", ctx),
|
||||
"path not found",
|
||||
);
|
||||
await assertRejects(
|
||||
() =>
|
||||
resolveConfigRefs("$CONTEXT_obj", {
|
||||
resolveConfigRefs("Bearer {{ context.obj }}", {
|
||||
...ctx,
|
||||
context: { obj: { a: 1 } },
|
||||
}),
|
||||
'context "obj" is not a scalar',
|
||||
"not a scalar",
|
||||
);
|
||||
await assertRejects(
|
||||
() => resolveConfigRefs("$CONTEXT_", ctx),
|
||||
"empty context key",
|
||||
() => resolveConfigRefs("{{ vars }}", ctx),
|
||||
"empty var name",
|
||||
);
|
||||
await assertRejects(
|
||||
() => resolveConfigRefs("$VAR_does_not_exist_xyz", ctx),
|
||||
'variable "does_not_exist_xyz" not found',
|
||||
() => resolveConfigRefs("{{ title }}", ctx),
|
||||
"unknown root",
|
||||
);
|
||||
await assertRejects(
|
||||
() => resolveConfigRefs("$VAR_not valid", ctx),
|
||||
"invalid variable name",
|
||||
);
|
||||
await assertRejects(
|
||||
() => resolveConfigRefs("$VAR_", ctx),
|
||||
"empty variable name",
|
||||
() => resolveConfigRefs("{{ context.__proto__.x }}", ctx),
|
||||
"forbidden path segment",
|
||||
);
|
||||
} finally {
|
||||
await deleteSecret(secret.id);
|
||||
await deleteVariable(varUrl.id);
|
||||
await deleteVariable(varRetry.id);
|
||||
await deleteVariable(varDebug.id);
|
||||
for (const [kind, id] of created.reverse()) {
|
||||
if (kind === "secret") await deleteSecret(id);
|
||||
else await deleteVariable(id);
|
||||
}
|
||||
await db.destroy();
|
||||
}
|
||||
|
||||
console.log("config-refs smoke test passed");
|
||||
await db.destroy();
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { rewriteLegacyConfigRefsInText } from "../config-ref-rewrite.js";
|
||||
import { migrateDefaultOwnerIfNeeded } from "../owner-migrate.js";
|
||||
import { migrate, db } from "../db.js";
|
||||
|
||||
function assert(cond, msg) {
|
||||
if (!cond) throw new Error(msg);
|
||||
}
|
||||
|
||||
{
|
||||
const { text, changed } = rewriteLegacyConfigRefsInText(
|
||||
"a: $VAR_x\nb: $SECRET_y\nc: $CONTEXT_z\nd: passwordSecret: gmail_app",
|
||||
);
|
||||
assert(changed, "detects legacy");
|
||||
assert(text.includes("{{ vars.x }}"), "var rewrite");
|
||||
assert(text.includes("{{ secrets.y }}"), "secret rewrite");
|
||||
assert(text.includes("{{ context.z }}"), "context rewrite");
|
||||
assert(text.includes("passwordSecret: gmail_app"), "bare secret name untouched");
|
||||
}
|
||||
|
||||
await migrate();
|
||||
const first = await migrateDefaultOwnerIfNeeded();
|
||||
const second = await migrateDefaultOwnerIfNeeded();
|
||||
assert(second.secrets.moved === 0, "second pass moves no secrets");
|
||||
assert(second.runs.moved === 0, "second pass moves no runs");
|
||||
await db.destroy();
|
||||
|
||||
console.log("owner-migrate smoke passed", {
|
||||
firstSecretsMoved: first.secrets.moved,
|
||||
secondSecretsMoved: second.secrets.moved,
|
||||
});
|
||||
@@ -71,11 +71,11 @@ const created = await upsertProfile({
|
||||
owner,
|
||||
name,
|
||||
script: "ntfy.js",
|
||||
config: { url: "$VAR_ntfy_channel" },
|
||||
config: { url: "{{ vars.ntfy_channel }}" },
|
||||
description: "smoke",
|
||||
});
|
||||
assert(created.name === name, "created");
|
||||
assert(created.config.url === "$VAR_ntfy_channel", "config roundtrip");
|
||||
assert(created.config.url === "{{ vars.ntfy_channel }}", "config roundtrip");
|
||||
assert(created.script === "ntfy.js", "script locked on profile");
|
||||
|
||||
const fetched = await getProfilePlain(owner, name);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { HTTP_METHODS } from "@jerapah-flow/shared";
|
||||
import { HTTP_METHODS, DEFAULT_OWNER } from "@jerapah-flow/shared";
|
||||
import {
|
||||
checkAnyHttpAuth,
|
||||
resolveAuthMechanisms,
|
||||
@@ -85,8 +85,14 @@ export function createHttpTriggerHandler({
|
||||
return async function dispatchHttpTrigger(req, reply) {
|
||||
const wildcard = /** @type {{ "*": string }} */ (req.params)["*"] ?? "";
|
||||
const url = `/u/${String(wildcard).replace(/^\/+/, "")}`;
|
||||
// Compat: leftover webhooks still hitting /u/default/... after owner rename.
|
||||
const compatUrl = url.startsWith("/u/default/")
|
||||
? `/u/${DEFAULT_OWNER}/${url.slice("/u/default/".length)}`
|
||||
: url === "/u/default"
|
||||
? `/u/${DEFAULT_OWNER}`
|
||||
: url;
|
||||
const method = String(req.method ?? "GET").toUpperCase();
|
||||
const routeKey = `${method} ${url}`;
|
||||
const routeKey = `${method} ${compatUrl}`;
|
||||
const mapped = httpRoutes.get(routeKey);
|
||||
|
||||
if (!mapped) {
|
||||
@@ -104,7 +110,7 @@ export function createHttpTriggerHandler({
|
||||
if (t?.type !== "HTTP") return false;
|
||||
const m = String(t.method ?? "POST").toUpperCase();
|
||||
const p = namespacedPath(entry.owner, t.path);
|
||||
return m === method && p === url;
|
||||
return m === method && p === compatUrl;
|
||||
}) ?? mapped.trigger;
|
||||
|
||||
if (
|
||||
|
||||
@@ -2,3 +2,4 @@ export { isPlainObject } from "./is-plain-object.js";
|
||||
export { mergeProfileConfig, overlayFromMerged, configHasOverlay } from "./profile-config.js";
|
||||
export { ensureWorkflowFilename, suggestCopyFilename } from "./workflow-filename.js";
|
||||
export { HTTP_METHODS, namespacedPath, hasWorkflowTrigger } from "./workflow-path.js";
|
||||
export { DEFAULT_OWNER } from "./tenant.js";
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
/** Default owner folder for new resources (latent tenant id). */
|
||||
export const DEFAULT_OWNER = "local";
|
||||
@@ -66,7 +66,8 @@ export function SecretEditorModal({ mode, initial, onClose, onSaved }) {
|
||||
</label>
|
||||
<p className="text-xs opacity-60">
|
||||
Values are encrypted at rest and never shown again after save. Values shorter than 8
|
||||
characters are not redacted from logs.
|
||||
characters are not redacted from logs. In workflows use{" "}
|
||||
<span className="font-mono">{"{{ secrets.name }}"}</span> (quote in YAML).
|
||||
</p>
|
||||
{upsert.isError ? (
|
||||
<p className="text-error text-sm">{errorMessage(upsert.error)}</p>
|
||||
|
||||
@@ -129,7 +129,9 @@ export function VariableEditorModal({ mode, initial, onClose, onSaved }) {
|
||||
</label>
|
||||
<p className="text-xs opacity-60">
|
||||
Stored in plaintext. Use Secrets for credentials. In workflows use{" "}
|
||||
<span className="font-mono">$VAR_name</span> as a whole field.
|
||||
<span className="font-mono">{"{{ vars.name }}"}</span> (quote strings that
|
||||
start with <span className="font-mono">{"{"}</span>). Nested:{" "}
|
||||
<span className="font-mono">{"{{ context.user.id }}"}</span>.
|
||||
</p>
|
||||
{formError ? <p className="text-error text-sm">{formError}</p> : null}
|
||||
{upsert.isError ? (
|
||||
|
||||
@@ -4,22 +4,40 @@ import { LuEye, LuEyeOff, LuExternalLink } from "react-icons/lu";
|
||||
import { useVariables } from "../../api/hooks.js";
|
||||
|
||||
const VAR_PEEK_MAX = 48;
|
||||
const MUSTACHE_RE =
|
||||
/\{\{\s*([A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z0-9_]+)*)\s*\}\}/g;
|
||||
const LEGACY_PREFIX_RE = /^\s*\$(VAR|SECRET|CONTEXT)_([A-Za-z0-9._-]*)\s*$/;
|
||||
|
||||
const CONFIG_REF_PREFIXES = [
|
||||
{ prefix: "$SECRET_", label: "secret" },
|
||||
{ prefix: "$CONTEXT_", label: "context" },
|
||||
{ prefix: "$VAR_", label: "variable" },
|
||||
];
|
||||
|
||||
/**
|
||||
* @param {unknown} value
|
||||
* @returns {{
|
||||
* kind: "mustache" | "legacy",
|
||||
* path?: string,
|
||||
* root?: string,
|
||||
* name?: string,
|
||||
* legacyKind?: string,
|
||||
* legacyName?: string,
|
||||
* } | null}
|
||||
*/
|
||||
function describeConfigRef(value) {
|
||||
if (typeof value !== "string") return null;
|
||||
const trimmed = value.trim();
|
||||
for (const { prefix, label } of CONFIG_REF_PREFIXES) {
|
||||
if (trimmed.startsWith(prefix) && trimmed.length > prefix.length) {
|
||||
return { label, name: trimmed.slice(prefix.length), kind: prefix };
|
||||
}
|
||||
const legacy = LEGACY_PREFIX_RE.exec(trimmed);
|
||||
if (legacy) {
|
||||
const legacyKind =
|
||||
legacy[1] === "VAR" ? "variable" : legacy[1] === "SECRET" ? "secret" : "context";
|
||||
return {
|
||||
kind: "legacy",
|
||||
legacyKind,
|
||||
legacyName: legacy[2] ?? "",
|
||||
};
|
||||
}
|
||||
return null;
|
||||
MUSTACHE_RE.lastIndex = 0;
|
||||
const match = MUSTACHE_RE.exec(trimmed);
|
||||
if (!match) return null;
|
||||
const path = match[1];
|
||||
const root = path.split(".")[0];
|
||||
return { kind: "mustache", path, root, name: path.slice(root.length + 1) };
|
||||
}
|
||||
|
||||
function formatVarDisplay(value) {
|
||||
@@ -33,7 +51,7 @@ function truncatePeek(text, maxLen = VAR_PEEK_MAX) {
|
||||
return `${text.slice(0, Math.max(0, maxLen - 1))}…`;
|
||||
}
|
||||
|
||||
/** Edit-time lookup for `$VAR_` against workflow owner (not a runtime guarantee). */
|
||||
/** Edit-time lookup for `{{ vars.name }}` against workflow owner (not a runtime guarantee). */
|
||||
function lookupVariable(variables, owner, name) {
|
||||
const list = Array.isArray(variables) ? variables : [];
|
||||
const match = list.find((v) => v.owner === owner && v.name === name);
|
||||
@@ -62,7 +80,7 @@ function variablesDeepLink({ owner, name, missing }) {
|
||||
|
||||
export function ConfigRefHint({ value, owner }) {
|
||||
const ref = describeConfigRef(value);
|
||||
const isVar = ref?.kind === "$VAR_";
|
||||
const isVar = ref?.kind === "mustache" && ref.root === "vars" && Boolean(ref.name);
|
||||
const { data: variables = [], isPending } = useVariables(undefined, { enabled: isVar });
|
||||
const [revealed, setRevealed] = useState(false);
|
||||
|
||||
@@ -72,10 +90,32 @@ export function ConfigRefHint({ value, owner }) {
|
||||
|
||||
if (!ref) return null;
|
||||
|
||||
if (ref.kind === "legacy") {
|
||||
const hint =
|
||||
ref.legacyKind === "variable"
|
||||
? `{{ vars.${ref.legacyName || "name"} }}`
|
||||
: ref.legacyKind === "secret"
|
||||
? `{{ secrets.${ref.legacyName || "name"} }}`
|
||||
: `{{ context.${ref.legacyName || "name"} }}`;
|
||||
return (
|
||||
<p className="text-xs text-error">
|
||||
legacy ${ref.legacyKind} ref — use <span className="font-mono">{hint}</span>
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
if (!isVar) {
|
||||
const label =
|
||||
ref.root === "secrets"
|
||||
? "secret"
|
||||
: ref.root === "context"
|
||||
? "context"
|
||||
: ref.root === "data"
|
||||
? "data"
|
||||
: "expression";
|
||||
return (
|
||||
<p className="text-xs opacity-60">
|
||||
from {ref.label} <span className="font-mono">{ref.name}</span>
|
||||
from {label} <span className="font-mono">{ref.path}</span>
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
/** Default owner folder for new resources (latent tenant id). */
|
||||
export const DEFAULT_OWNER = "local";
|
||||
export { DEFAULT_OWNER } from "@jerapah-flow/shared";
|
||||
|
||||
@@ -179,7 +179,7 @@ export function VariablesPage() {
|
||||
title={confirmDelete ? `Delete ${confirmDelete.name}?` : ""}
|
||||
message={
|
||||
confirmDelete
|
||||
? `This cannot be undone. Workflows that reference $VAR_${confirmDelete.name} will fail.`
|
||||
? `This cannot be undone. Workflows that reference {{ vars.${confirmDelete.name} }} will fail.`
|
||||
: ""
|
||||
}
|
||||
error={del.isError ? errorMessage(del.error) : null}
|
||||
|
||||
@@ -198,7 +198,7 @@ joplinHttp.meta = {
|
||||
token: {
|
||||
type: "string",
|
||||
required: true,
|
||||
description: "Bearer token ($SECRET_); Bearer prefix is added if missing",
|
||||
description: "Bearer token ({{ secrets.* }}); Bearer prefix is added if missing",
|
||||
},
|
||||
timeoutMs: {
|
||||
type: "number",
|
||||
@@ -237,7 +237,7 @@ joplinHttp.meta = {
|
||||
config: {
|
||||
url: "http://10.8.0.6:3030/notes",
|
||||
method: "GET",
|
||||
token: "$SECRET_joplin_api_token",
|
||||
token: "{{ secrets.joplin_api_token }}",
|
||||
timeoutMs: 60000,
|
||||
},
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user