From 611511ae60107894f7343d299d95edd563cf6cce Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 19 Aug 2026 16:04:00 +0000 Subject: [PATCH] feat(plugins): core read-only scripts and user plugin system Introduce plugin/ installs (zip, HTTPS git, example, fork), jerapah-plugin.json manifests with jerapah semver ranges, isolated plugin dirs under data/plugins, and app version 0.1.0. Core scripts are non-editable; get-current-time moves to examples/plugins. Co-authored-by: Nasyarobby Putra --- README.md | 20 + .../get-current-time/jerapah-plugin.json | 8 + .../plugins/get-current-time/package.json | 7 + .../plugins/get-current-time/script.js | 0 package.json | 2 +- packages/server/app-version.js | 79 +++ packages/server/package.json | 5 +- packages/server/paths.js | 8 + packages/server/plugin-install.js | 218 +++++++++ packages/server/plugin-manifest.js | 159 ++++++ packages/server/plugin-store.js | 456 ++++++++++++++++++ packages/server/script-sandbox.js | 116 ++++- packages/server/src/api/scripts.js | 400 +++++++++++++-- packages/server/test/plugins-smoke.js | 118 +++++ .../workflows/default/cron-example.yaml | 2 +- packages/server/workflows/default/test.yaml | 2 +- .../default/time-and-comic-to-ntfy.yaml | 2 +- .../default/time-to-ntfy-example.yaml | 2 +- packages/web/package.json | 2 +- packages/web/src/api/hooks.js | 41 ++ packages/web/src/lib/workflow-doc.js | 2 +- packages/web/src/pages/ScriptEditPage.jsx | 153 ++++-- packages/web/src/pages/ScriptsPage.jsx | 173 ++++++- 23 files changed, 1843 insertions(+), 132 deletions(-) create mode 100644 examples/plugins/get-current-time/jerapah-plugin.json create mode 100644 examples/plugins/get-current-time/package.json rename packages/server/scripts/get-current-time.js => examples/plugins/get-current-time/script.js (100%) create mode 100644 packages/server/app-version.js create mode 100644 packages/server/plugin-install.js create mode 100644 packages/server/plugin-manifest.js create mode 100644 packages/server/plugin-store.js create mode 100644 packages/server/test/plugins-smoke.js diff --git a/README.md b/README.md index 17191c3..d67ae25 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,26 @@ The first account created becomes **admin**. Later accounts are created from Use `pnpm dev:pm2` is the mode for Ops (start/stop HTTP, scale workers, drain restart). Control owns SQLite migrations; HTTP/workers do not migrate. +## Scripts (core vs plugins) + +| Kind | Name in YAML | Editable | Location | +|---|---|---|---| +| **Core** | `fetch-http.js`, `s3.js`, … | No (fork only) | `packages/server/scripts/` | +| **Plugin** | `plugin/` | Yes | `data/plugins//` | + +- App version is **`0.1.0`** (root `package.json`). Plugin manifests declare `jerapah: ">=0.1.0 <1.0.0"`. +- Install plugins via admin API: zip (base64), HTTPS git URL, example, or fork a core script. +- Install/update/uninstall sets **restart-needed** — drain-restart HTTP + workers under `pnpm dev:pm2`. +- Example plugin: `examples/plugins/get-current-time` → `plugin/get-current-time`. + +```bash +# Smoke +JFLOW_PLUGINS_DIR=packages/server/data/plugins-smoke-test \ +JFLOW_DB_PATH=packages/server/data/plugins-smoke.db \ +node packages/server/test/plugins-smoke.js +``` + + ## Script contract Each script is `async function main(ctx)` and **must** return: diff --git a/examples/plugins/get-current-time/jerapah-plugin.json b/examples/plugins/get-current-time/jerapah-plugin.json new file mode 100644 index 0000000..35294d2 --- /dev/null +++ b/examples/plugins/get-current-time/jerapah-plugin.json @@ -0,0 +1,8 @@ +{ + "id": "get-current-time", + "name": "Get current time", + "version": "0.1.0", + "jerapah": ">=0.1.0 <1.0.0", + "main": "script.js", + "description": "Example user plugin: return the current time as output.datetime" +} diff --git a/examples/plugins/get-current-time/package.json b/examples/plugins/get-current-time/package.json new file mode 100644 index 0000000..2ca2114 --- /dev/null +++ b/examples/plugins/get-current-time/package.json @@ -0,0 +1,7 @@ +{ + "name": "jflow-plugin-get-current-time", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Example JerapahFlow plugin (no npm dependencies)" +} diff --git a/packages/server/scripts/get-current-time.js b/examples/plugins/get-current-time/script.js similarity index 100% rename from packages/server/scripts/get-current-time.js rename to examples/plugins/get-current-time/script.js diff --git a/package.json b/package.json index 528db8e..8cc5674 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jerapah-flow", - "version": "1.0.0", + "version": "0.1.0", "private": true, "description": "Script/workflow runner with admin UI", "type": "module", diff --git a/packages/server/app-version.js b/packages/server/app-version.js new file mode 100644 index 0000000..c81629f --- /dev/null +++ b/packages/server/app-version.js @@ -0,0 +1,79 @@ +import fs from "fs"; +import path from "path"; +import { fileURLToPath } from "url"; +import { SERVER_ROOT } from "./paths.js"; + +const ROOT_PKG = path.resolve(SERVER_ROOT, "../../package.json"); + +/** + * JerapahFlow app version from the monorepo root package.json. + * @returns {string} + */ +export function getAppVersion() { + try { + const raw = JSON.parse(fs.readFileSync(ROOT_PKG, "utf8")); + if (typeof raw.version === "string" && raw.version.trim()) { + return raw.version.trim(); + } + } catch { + // fall through + } + return "0.1.0"; +} + +/** + * @param {string} version + * @returns {[number, number, number]} + */ +function parseSemver(version) { + const cleaned = String(version).trim().replace(/^v/i, ""); + const core = cleaned.split("-")[0].split("+")[0]; + const parts = core.split(".").map((p) => Number(p)); + return [parts[0] || 0, parts[1] || 0, parts[2] || 0]; +} + +/** + * @param {[number, number, number]} a + * @param {[number, number, number]} b + */ +function cmp(a, b) { + for (let i = 0; i < 3; i++) { + if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1; + } + return 0; +} + +/** + * Minimal semver range check for patterns used in manifests: + * `1.2.3`, `>=0.1.0`, `<1.0.0`, `>=0.1.0 <1.0.0` + * + * @param {string} version + * @param {string} range + * @returns {boolean} + */ +export function satisfiesRange(version, range) { + if (typeof range !== "string" || !range.trim()) return false; + const ver = parseSemver(version); + const tokens = range.trim().split(/\s+/); + for (const token of tokens) { + if (/^\d+\.\d+\.\d+/.test(token) && !token.startsWith(">") && !token.startsWith("<")) { + if (cmp(ver, parseSemver(token)) !== 0) return false; + continue; + } + const m = /^(>=|<=|>|<|=)?\s*v?(\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)$/.exec( + token, + ); + if (!m) return false; + const op = m[1] || "="; + const bound = parseSemver(m[2]); + const c = cmp(ver, bound); + if (op === ">=" && c < 0) return false; + if (op === "<=" && c > 0) return false; + if (op === ">" && c <= 0) return false; + if (op === "<" && c >= 0) return false; + if (op === "=" && c !== 0) return false; + } + return true; +} + +void fileURLToPath; diff --git a/packages/server/package.json b/packages/server/package.json index 8a38c26..39d66a7 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "@jerapah-flow/server", - "version": "1.0.0", + "version": "0.1.0", "private": true, "type": "module", "main": "runner.js", @@ -11,7 +11,8 @@ "start:api": "node server.js", "start:worker": "node worker.js", "start:control": "node control.js", - "migrate": "node -e \"import('./db.js').then((m) => m.migrate().then(() => process.exit(0)))\"" + "migrate": "node -e \"import('./db.js').then((m) => m.migrate().then(() => process.exit(0)))\"", + "test:plugins": "JFLOW_PLUGINS_DIR=./data/plugins-smoke-test JFLOW_DB_PATH=./data/plugins-smoke.db node test/plugins-smoke.js" }, "dependencies": { "@aws-sdk/client-s3": "^3.1111.0", diff --git a/packages/server/paths.js b/packages/server/paths.js index f84f6a5..55043c2 100644 --- a/packages/server/paths.js +++ b/packages/server/paths.js @@ -5,5 +5,13 @@ export const SERVER_ROOT = path.dirname(fileURLToPath(import.meta.url)); export const SCRIPTS_DIR = path.join(SERVER_ROOT, "scripts"); export const WORKFLOWS_DIR = path.join(SERVER_ROOT, "workflows"); export const DATA_DIR = path.join(SERVER_ROOT, "data"); +/** Installed user plugins (outside the pnpm workspace). */ +export const PLUGINS_DIR = + process.env.JFLOW_PLUGINS_DIR ?? path.join(DATA_DIR, "plugins"); +/** Example plugin sources shipped with the repo. */ +export const EXAMPLE_PLUGINS_DIR = path.resolve( + SERVER_ROOT, + "../../examples/plugins", +); export const LOGS_DIR = path.join(SERVER_ROOT, "logs"); export const WEB_DIST = path.resolve(SERVER_ROOT, "../web/dist"); diff --git a/packages/server/plugin-install.js b/packages/server/plugin-install.js new file mode 100644 index 0000000..06c1f05 --- /dev/null +++ b/packages/server/plugin-install.js @@ -0,0 +1,218 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { EXAMPLE_PLUGINS_DIR, PLUGINS_DIR } from "./paths.js"; +import { + installPluginFromDirectory, + pluginDir, +} from "./plugin-store.js"; +import { readManifestFile } from "./plugin-manifest.js"; + +const execFileAsync = promisify(execFile); + +/** + * @param {string} url + */ +export function assertHttpsGitUrl(url) { + let parsed; + try { + parsed = new URL(url); + } catch { + const err = new Error("invalid git URL"); + err.statusCode = 400; + throw err; + } + if (parsed.protocol !== "https:") { + const err = new Error("git URL must use https://"); + err.statusCode = 400; + throw err; + } + return parsed.toString(); +} + +/** + * Run pnpm install in a plugin directory (ignore lifecycle scripts). + * @param {string} dir + */ +export async function pnpmInstallPlugin(dir) { + const pkg = path.join(dir, "package.json"); + if (!fs.existsSync(pkg)) return { skipped: true }; + let hasDeps = false; + try { + const raw = JSON.parse(fs.readFileSync(pkg, "utf8")); + hasDeps = Boolean( + (raw.dependencies && Object.keys(raw.dependencies).length) || + (raw.optionalDependencies && + Object.keys(raw.optionalDependencies).length), + ); + } catch { + hasDeps = true; + } + if (!hasDeps) return { skipped: true }; + + await execFileAsync( + "pnpm", + ["install", "--dir", dir, "--ignore-scripts", "--prefer-offline"], + { + cwd: dir, + env: { ...process.env, CI: "1" }, + timeout: 5 * 60_000, + maxBuffer: 10 * 1024 * 1024, + }, + ); + return { skipped: false }; +} + +/** + * @param {string} url + * @param {{ ref?: string, overwrite?: boolean }} [opts] + */ +export async function installPluginFromGit(url, opts = {}) { + const httpsUrl = assertHttpsGitUrl(url); + const staging = path.join( + PLUGINS_DIR, + `.staging-git-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + fs.mkdirSync(PLUGINS_DIR, { recursive: true }); + fs.mkdirSync(staging, { recursive: true }); + try { + const args = ["clone", "--depth", "1"]; + if (opts.ref) { + args.push("--branch", String(opts.ref)); + } + args.push(httpsUrl, staging); + await execFileAsync("git", args, { + timeout: 5 * 60_000, + maxBuffer: 5 * 1024 * 1024, + }); + // Remove .git to keep plugins lean + fs.rmSync(path.join(staging, ".git"), { recursive: true, force: true }); + const installed = installPluginFromDirectory(staging, { + overwrite: Boolean(opts.overwrite), + markRestart: false, + }); + await pnpmInstallPlugin(installed.dir); + const { bumpGeneration } = await import("./control-state.js"); + bumpGeneration(`plugin:${installed.id} installed from git`); + return installed; + } finally { + fs.rmSync(staging, { recursive: true, force: true }); + } +} + +/** + * @param {string} zipPath + * @param {{ overwrite?: boolean }} [opts] + */ +export async function installPluginFromZipFile(zipPath, opts = {}) { + const abs = path.resolve(zipPath); + if (!fs.existsSync(abs)) { + const err = new Error("zip file not found"); + err.statusCode = 400; + throw err; + } + const staging = path.join( + PLUGINS_DIR, + `.staging-zip-${Date.now()}-${Math.random().toString(36).slice(2)}`, + ); + const extractDir = path.join(staging, "extract"); + fs.mkdirSync(extractDir, { recursive: true }); + try { + await execFileAsync("unzip", ["-q", abs, "-d", extractDir], { + timeout: 120_000, + }); + const root = findPluginRoot(extractDir); + const installed = installPluginFromDirectory(root, { + overwrite: Boolean(opts.overwrite), + markRestart: false, + }); + await pnpmInstallPlugin(installed.dir); + const { bumpGeneration } = await import("./control-state.js"); + bumpGeneration(`plugin:${installed.id} installed from zip`); + return installed; + } finally { + fs.rmSync(staging, { recursive: true, force: true }); + } +} + +/** + * @param {Buffer} buffer + * @param {{ overwrite?: boolean }} [opts] + */ +export async function installPluginFromZipBuffer(buffer, opts = {}) { + const tmp = path.join( + os.tmpdir(), + `jflow-plugin-${Date.now()}-${Math.random().toString(36).slice(2)}.zip`, + ); + fs.writeFileSync(tmp, buffer); + try { + return await installPluginFromZipFile(tmp, opts); + } finally { + fs.unlinkSync(tmp); + } +} + +/** + * Find directory containing jerapah-plugin.json (zip may have a single top folder). + * @param {string} extractDir + */ +function findPluginRoot(extractDir) { + const direct = path.join(extractDir, "jerapah-plugin.json"); + if (fs.existsSync(direct)) return extractDir; + const entries = fs.readdirSync(extractDir, { withFileTypes: true }); + const dirs = entries.filter((e) => e.isDirectory() && !e.name.startsWith(".")); + if (dirs.length === 1) { + const nested = path.join(extractDir, dirs[0].name); + if (fs.existsSync(path.join(nested, "jerapah-plugin.json"))) return nested; + } + for (const e of dirs) { + const nested = path.join(extractDir, e.name); + if (fs.existsSync(path.join(nested, "jerapah-plugin.json"))) return nested; + } + const err = new Error("zip missing jerapah-plugin.json"); + err.statusCode = 400; + throw err; +} + +/** + * Install a shipped example plugin by id (from examples/plugins/). + * @param {string} exampleId + * @param {{ overwrite?: boolean }} [opts] + */ +export async function installExamplePlugin(exampleId, opts = {}) { + const src = path.join(EXAMPLE_PLUGINS_DIR, exampleId); + if (!fs.existsSync(src)) { + const err = new Error(`example plugin not found: ${exampleId}`); + err.statusCode = 404; + throw err; + } + const installed = installPluginFromDirectory(src, { + overwrite: Boolean(opts.overwrite), + markRestart: false, + }); + await pnpmInstallPlugin(installed.dir); + const { bumpGeneration } = await import("./control-state.js"); + bumpGeneration(`plugin:${installed.id} installed from example`); + return installed; +} + +/** + * Copy example into plugins if missing (used by smoke / first-run helpers). + * @param {string} exampleId + */ +export function ensureExampleInstalled(exampleId) { + const dest = pluginDir(exampleId); + if (fs.existsSync(dest)) { + return { id: exampleId, already: true, dir: dest }; + } + const src = path.join(EXAMPLE_PLUGINS_DIR, exampleId); + const installed = installPluginFromDirectory(src, { + overwrite: false, + markRestart: false, + }); + return { ...installed, already: false }; +} + +void readManifestFile; diff --git a/packages/server/plugin-manifest.js b/packages/server/plugin-manifest.js new file mode 100644 index 0000000..e03ea69 --- /dev/null +++ b/packages/server/plugin-manifest.js @@ -0,0 +1,159 @@ +import fs from "fs"; +import path from "path"; +import { getAppVersion, satisfiesRange } from "./app-version.js"; + +export const PLUGIN_MANIFEST = "jerapah-plugin.json"; +export const PLUGIN_PREFIX = "plugin/"; + +/** + * @param {string} id + * @returns {string} + */ +export function assertPluginId(id) { + if (typeof id !== "string" || !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(id)) { + const err = new Error( + "invalid plugin id (use lowercase letters, numbers, hyphens)", + ); + err.statusCode = 400; + throw err; + } + if (id.length > 64) { + const err = new Error("plugin id too long"); + err.statusCode = 400; + throw err; + } + return id; +} + +/** + * @param {string} scriptRef e.g. plugin/foo or plugin/foo.js + * @returns {{ id: string, scriptRef: string } | null} + */ +export function parsePluginScriptRef(scriptRef) { + if (typeof scriptRef !== "string") return null; + let rest = scriptRef; + if (rest.startsWith(PLUGIN_PREFIX)) { + rest = rest.slice(PLUGIN_PREFIX.length); + } else { + return null; + } + if (rest.endsWith(".js")) rest = rest.slice(0, -3); + if (!rest || rest.includes("/") || rest.includes("\\")) return null; + try { + const id = assertPluginId(rest); + return { id, scriptRef: `${PLUGIN_PREFIX}${id}` }; + } catch { + return null; + } +} + +/** + * @param {string} id + * @returns {string} + */ +export function pluginScriptRef(id) { + return `${PLUGIN_PREFIX}${assertPluginId(id)}`; +} + +/** + * @param {unknown} raw + * @returns {{ + * id: string, + * name: string, + * version: string, + * jerapah: string, + * main: string, + * description: string | null, + * }} + */ +export function validateManifest(raw) { + if (raw == null || typeof raw !== "object" || Array.isArray(raw)) { + const err = new Error("manifest must be an object"); + err.statusCode = 400; + throw err; + } + const id = assertPluginId(String(/** @type {any} */ (raw).id ?? "")); + const version = String(/** @type {any} */ (raw).version ?? "").trim(); + if (!/^\d+\.\d+\.\d+/.test(version)) { + const err = new Error("manifest.version must be semver (e.g. 0.1.0)"); + err.statusCode = 400; + throw err; + } + const jerapah = String(/** @type {any} */ (raw).jerapah ?? "").trim(); + if (!jerapah) { + const err = new Error("manifest.jerapah range is required"); + err.statusCode = 400; + throw err; + } + const main = String(/** @type {any} */ (raw).main ?? "script.js").trim(); + if (!main || main.includes("..") || path.isAbsolute(main)) { + const err = new Error("manifest.main must be a relative file path"); + err.statusCode = 400; + throw err; + } + const name = + String(/** @type {any} */ (raw).name ?? id).trim() || id; + const descriptionRaw = /** @type {any} */ (raw).description; + const description = + typeof descriptionRaw === "string" && descriptionRaw.trim() + ? descriptionRaw.trim() + : null; + return { id, name, version, jerapah, main, description }; +} + +/** + * @param {string} pluginDir + */ +export function readManifestFile(pluginDir) { + const filePath = path.join(pluginDir, PLUGIN_MANIFEST); + if (!fs.existsSync(filePath)) { + const err = new Error(`missing ${PLUGIN_MANIFEST}`); + err.statusCode = 400; + throw err; + } + let raw; + try { + raw = JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch { + const err = new Error(`invalid ${PLUGIN_MANIFEST} JSON`); + err.statusCode = 400; + throw err; + } + return validateManifest(raw); +} + +/** + * @param {ReturnType} manifest + * @returns {{ ok: true } | { ok: false, error: string }} + */ +export function checkJerapahCompat(manifest) { + const appVersion = getAppVersion(); + if (!satisfiesRange(appVersion, manifest.jerapah)) { + return { + ok: false, + error: `plugin requires JerapahFlow ${manifest.jerapah}; app is ${appVersion}`, + }; + } + return { ok: true }; +} + +/** + * @param {{ + * id: string, + * name?: string, + * version?: string, + * jerapah?: string, + * main?: string, + * description?: string | null, + * }} opts + */ +export function buildManifest(opts) { + return validateManifest({ + id: opts.id, + name: opts.name ?? opts.id, + version: opts.version ?? "0.1.0", + jerapah: opts.jerapah ?? ">=0.1.0 <1.0.0", + main: opts.main ?? "script.js", + description: opts.description ?? null, + }); +} diff --git a/packages/server/plugin-store.js b/packages/server/plugin-store.js new file mode 100644 index 0000000..e10201c --- /dev/null +++ b/packages/server/plugin-store.js @@ -0,0 +1,456 @@ +import fs from "fs"; +import path from "path"; +import { createRequire } from "node:module"; +import { PLUGINS_DIR, SCRIPTS_DIR } from "./paths.js"; +import { + PLUGIN_MANIFEST, + assertPluginId, + buildManifest, + checkJerapahCompat, + parsePluginScriptRef, + pluginScriptRef, + readManifestFile, + validateManifest, +} from "./plugin-manifest.js"; +import { listScriptFiles } from "./fs-store.js"; +import { bumpGeneration } from "./control-state.js"; + +/** + * @param {string} id + */ +export function pluginDir(id) { + return path.join(PLUGINS_DIR, assertPluginId(id)); +} + +export function ensurePluginsDir() { + fs.mkdirSync(PLUGINS_DIR, { recursive: true }); +} + +/** + * Core script file names (*.js) currently shipped under SCRIPTS_DIR. + * @returns {string[]} + */ +export function listCoreScriptNames() { + return listScriptFiles(); +} + +/** + * Bare core names without .js (for collision checks). + * @returns {Set} + */ +export function coreBareNames() { + return new Set( + listCoreScriptNames().map((n) => (n.endsWith(".js") ? n.slice(0, -3) : n)), + ); +} + +/** + * @returns {Array<{ + * id: string, + * scriptRef: string, + * dir: string, + * manifest: ReturnType, + * compatible: boolean, + * compatError: string | null, + * disabled: boolean, + * }>} + */ +export function listInstalledPlugins() { + ensurePluginsDir(); + if (!fs.existsSync(PLUGINS_DIR)) return []; + /** @type {Array} */ + const out = []; + for (const entry of fs.readdirSync(PLUGINS_DIR, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + let id; + try { + id = assertPluginId(entry.name); + } catch { + continue; + } + const dir = pluginDir(id); + try { + const manifest = readManifestFile(dir); + if (manifest.id !== id) { + out.push({ + id, + scriptRef: pluginScriptRef(id), + dir, + manifest, + compatible: false, + compatError: `manifest id "${manifest.id}" does not match folder "${id}"`, + disabled: true, + }); + continue; + } + const compat = checkJerapahCompat(manifest); + const disabledFlag = fs.existsSync(path.join(dir, ".disabled")); + out.push({ + id, + scriptRef: pluginScriptRef(id), + dir, + manifest, + compatible: compat.ok, + compatError: compat.ok ? null : compat.error, + disabled: disabledFlag || !compat.ok, + }); + } catch (err) { + out.push({ + id, + scriptRef: pluginScriptRef(id), + dir, + manifest: null, + compatible: false, + compatError: err instanceof Error ? err.message : String(err), + disabled: true, + }); + } + } + return out.sort((a, b) => a.id.localeCompare(b.id)); +} + +/** + * @param {string} id + */ +export function getInstalledPlugin(id) { + const needle = assertPluginId(id); + return listInstalledPlugins().find((p) => p.id === needle) ?? null; +} + +/** + * Resolve a workflow script ref to a filesystem path + kind. + * + * @param {string} scriptRef + * @returns {{ + * kind: "core" | "plugin", + * scriptRef: string, + * filePath: string, + * pluginId?: string, + * pluginDir?: string, + * disabled?: boolean, + * error?: string, + * }} + */ +export function resolveScriptRef(scriptRef) { + if (typeof scriptRef !== "string" || !scriptRef.trim()) { + return { + kind: "core", + scriptRef: String(scriptRef), + filePath: "", + error: "invalid script ref", + }; + } + + const plugin = parsePluginScriptRef(scriptRef); + if (plugin) { + const installed = getInstalledPlugin(plugin.id); + if (!installed) { + return { + kind: "plugin", + scriptRef: plugin.scriptRef, + pluginId: plugin.id, + filePath: "", + error: `plugin not installed: ${plugin.scriptRef}`, + }; + } + if (installed.disabled) { + return { + kind: "plugin", + scriptRef: plugin.scriptRef, + pluginId: plugin.id, + pluginDir: installed.dir, + filePath: "", + disabled: true, + error: + installed.compatError || + `plugin disabled: ${plugin.scriptRef}`, + }; + } + const mainPath = path.join(installed.dir, installed.manifest.main); + if (!fs.existsSync(mainPath)) { + return { + kind: "plugin", + scriptRef: plugin.scriptRef, + pluginId: plugin.id, + pluginDir: installed.dir, + filePath: "", + error: `plugin main missing: ${installed.manifest.main}`, + }; + } + return { + kind: "plugin", + scriptRef: plugin.scriptRef, + pluginId: plugin.id, + pluginDir: installed.dir, + filePath: mainPath, + }; + } + + // Core: must be a plain *.js filename + if ( + scriptRef.includes("/") || + scriptRef.includes("\\") || + scriptRef.includes("..") + ) { + return { + kind: "core", + scriptRef, + filePath: "", + error: "invalid core script name", + }; + } + const name = scriptRef.endsWith(".js") ? scriptRef : `${scriptRef}.js`; + const filePath = path.join(SCRIPTS_DIR, name); + if (!fs.existsSync(filePath)) { + return { + kind: "core", + scriptRef: name, + filePath: "", + error: `core script not found: ${name}`, + }; + } + return { kind: "core", scriptRef: name, filePath }; +} + +/** + * Copy a prepared plugin directory into PLUGINS_DIR. + * + * @param {string} sourceDir directory containing jerapah-plugin.json + * @param {{ overwrite?: boolean, markRestart?: boolean, reason?: string }} [opts] + */ +export function installPluginFromDirectory(sourceDir, opts = {}) { + const abs = path.resolve(sourceDir); + if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) { + const err = new Error("plugin source directory not found"); + err.statusCode = 400; + throw err; + } + const manifest = readManifestFile(abs); + const compat = checkJerapahCompat(manifest); + if (!compat.ok) { + const err = new Error(compat.error); + err.statusCode = 409; + throw err; + } + if (coreBareNames().has(manifest.id)) { + const err = new Error( + `plugin id "${manifest.id}" collides with a core script name`, + ); + err.statusCode = 409; + throw err; + } + + const mainPath = path.join(abs, manifest.main); + if (!fs.existsSync(mainPath)) { + const err = new Error(`manifest.main not found: ${manifest.main}`); + err.statusCode = 400; + throw err; + } + + ensurePluginsDir(); + const dest = pluginDir(manifest.id); + if (fs.existsSync(dest)) { + if (!opts.overwrite) { + const err = new Error(`plugin already installed: ${manifest.id}`); + err.statusCode = 409; + throw err; + } + fs.rmSync(dest, { recursive: true, force: true }); + } + + fs.cpSync(abs, dest, { recursive: true }); + + // Ensure package.json exists (fork / thin plugins). + const pkgPath = path.join(dest, "package.json"); + if (!fs.existsSync(pkgPath)) { + fs.writeFileSync( + pkgPath, + `${JSON.stringify( + { + name: `jflow-plugin-${manifest.id}`, + version: manifest.version, + private: true, + type: "module", + }, + null, + 2, + )}\n`, + "utf8", + ); + } + + if (opts.markRestart !== false) { + bumpGeneration(opts.reason ?? `plugin:${manifest.id} installed`); + } + + return { + id: manifest.id, + scriptRef: pluginScriptRef(manifest.id), + dir: dest, + manifest, + }; +} + +/** + * @param {string} id + * @param {{ markRestart?: boolean }} [opts] + */ +export function uninstallPlugin(id, opts = {}) { + const pluginId = assertPluginId(id); + const dir = pluginDir(pluginId); + if (!fs.existsSync(dir)) { + const err = new Error("plugin not found"); + err.statusCode = 404; + throw err; + } + fs.rmSync(dir, { recursive: true, force: true }); + if (opts.markRestart !== false) { + bumpGeneration(`plugin:${pluginId} uninstalled`); + } + return { ok: true, id: pluginId }; +} + +/** + * Fork a core script into a new plugin. + * + * @param {string} coreName e.g. fetch-http.js + * @param {string} newId + * @param {{ description?: string }} [opts] + */ +export function forkCoreScript(coreName, newId, opts = {}) { + const id = assertPluginId(newId); + if (coreBareNames().has(id)) { + const err = new Error(`plugin id collides with core script: ${id}`); + err.statusCode = 409; + throw err; + } + if (fs.existsSync(pluginDir(id))) { + const err = new Error(`plugin already exists: ${id}`); + err.statusCode = 409; + throw err; + } + + const coreFile = coreName.endsWith(".js") ? coreName : `${coreName}.js`; + const src = path.join(SCRIPTS_DIR, coreFile); + if (!fs.existsSync(src)) { + const err = new Error(`core script not found: ${coreFile}`); + err.statusCode = 404; + throw err; + } + + const staging = path.join(PLUGINS_DIR, `.staging-fork-${id}-${Date.now()}`); + fs.mkdirSync(staging, { recursive: true }); + try { + const main = "script.js"; + fs.copyFileSync(src, path.join(staging, main)); + const manifest = buildManifest({ + id, + name: id, + version: "0.1.0", + jerapah: ">=0.1.0 <1.0.0", + main, + description: + opts.description ?? `Fork of core script ${coreFile}`, + }); + fs.writeFileSync( + path.join(staging, PLUGIN_MANIFEST), + `${JSON.stringify(manifest, null, 2)}\n`, + "utf8", + ); + fs.writeFileSync( + path.join(staging, "package.json"), + `${JSON.stringify( + { + name: `jflow-plugin-${id}`, + version: "0.1.0", + private: true, + type: "module", + }, + null, + 2, + )}\n`, + "utf8", + ); + return installPluginFromDirectory(staging, { + overwrite: false, + reason: `plugin:${id} forked from ${coreFile}`, + }); + } finally { + fs.rmSync(staging, { recursive: true, force: true }); + } +} + +/** + * @param {string} pluginDirectory + * @returns {((id: string) => unknown) | null} + */ +export function createPluginRequire(pluginDirectory) { + const pkg = path.resolve(pluginDirectory, "package.json"); + if (!fs.existsSync(pkg)) return null; + return createRequire(pkg); +} + +/** + * Create an empty plugin from the new-script template. + * @param {string} newId + * @param {string} source + * @param {{ description?: string }} [opts] + */ +export function createBlankPlugin(newId, source, opts = {}) { + const id = assertPluginId(newId); + if (coreBareNames().has(id)) { + const err = new Error(`plugin id collides with core script: ${id}`); + err.statusCode = 409; + throw err; + } + if (fs.existsSync(pluginDir(id))) { + const err = new Error(`plugin already exists: ${id}`); + err.statusCode = 409; + throw err; + } + if (typeof source !== "string") { + const err = new Error("source content is required"); + err.statusCode = 400; + throw err; + } + + const staging = path.join(PLUGINS_DIR, `.staging-new-${id}-${Date.now()}`); + fs.mkdirSync(staging, { recursive: true }); + try { + const main = "script.js"; + fs.writeFileSync(path.join(staging, main), source, "utf8"); + const manifest = buildManifest({ + id, + name: id, + version: "0.1.0", + jerapah: ">=0.1.0 <1.0.0", + main, + description: opts.description ?? null, + }); + fs.writeFileSync( + path.join(staging, PLUGIN_MANIFEST), + `${JSON.stringify(manifest, null, 2)}\n`, + "utf8", + ); + fs.writeFileSync( + path.join(staging, "package.json"), + `${JSON.stringify( + { + name: `jflow-plugin-${id}`, + version: "0.1.0", + private: true, + type: "module", + }, + null, + 2, + )}\n`, + "utf8", + ); + return installPluginFromDirectory(staging, { + overwrite: false, + reason: `plugin:${id} created`, + }); + } finally { + fs.rmSync(staging, { recursive: true, force: true }); + } +} diff --git a/packages/server/script-sandbox.js b/packages/server/script-sandbox.js index 067ad02..7996f7a 100644 --- a/packages/server/script-sandbox.js +++ b/packages/server/script-sandbox.js @@ -6,7 +6,7 @@ import axios from "axios"; import pino from "pino"; import { createKvApi } from "./kv-store.js"; import { createFingerprintApi } from "./script-fingerprint.js"; -import { SCRIPTS_DIR } from "./paths.js"; +import { resolveScriptRef, createPluginRequire } from "./plugin-store.js"; import { isSecret, Secret, unwrapSecretsDeep } from "./secret-value.js"; import { getHttpPageByName, getHttpTemplateByName } from "./http-pages-store.js"; import { getSecretPlaintext } from "./secrets-store.js"; @@ -296,15 +296,60 @@ function createScreenedAxios(log) { }); } -function createRestrictedRequire(screenedAxios) { +const BLOCKED_PLUGIN_MODULES = new Set([ + "child_process", + "node:child_process", + "cluster", + "node:cluster", + "fs", + "node:fs", + "fs/promises", + "node:fs/promises", + "module", + "node:module", + "vm", + "node:vm", + "worker_threads", + "node:worker_threads", + "v8", + "node:v8", + "inspector", + "node:inspector", + "sqlite", + "node:sqlite", +]); + +/** + * @param {import("axios").AxiosInstance} screenedAxios + * @param {string | null} [pluginDirectory] + */ +function createRestrictedRequire(screenedAxios, pluginDirectory = null) { + const pluginRequire = pluginDirectory + ? createPluginRequire(pluginDirectory) + : null; + return function restrictedRequire(id) { - if (typeof id !== "string" || !ALLOWED_MODULES.has(id)) { + if (typeof id !== "string") { throw new Error(`require(${JSON.stringify(id)}) is not allowed`); } - if (id === "axios") { - return screenedAxios; + if (BLOCKED_PLUGIN_MODULES.has(id)) { + throw new Error(`require(${JSON.stringify(id)}) is not allowed`); } - return hostRequire(id); + if (ALLOWED_MODULES.has(id)) { + if (id === "axios") return screenedAxios; + return hostRequire(id); + } + if (pluginRequire) { + try { + return pluginRequire(id); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + throw new Error( + `require(${JSON.stringify(id)}) failed in plugin: ${msg}`, + ); + } + } + throw new Error(`require(${JSON.stringify(id)}) is not allowed`); }; } @@ -391,6 +436,7 @@ const $workflowsStub = { * workflowName: string, * owner?: string, * $workflows?: { trigger: (name: string, data?: unknown) => Promise }, + * pluginDir?: string | null, * }} opts */ function createScriptSandbox({ @@ -399,6 +445,7 @@ function createScriptSandbox({ workflowName, owner = "default", $workflows = $workflowsStub, + pluginDir = null, }) { const scriptLog = log.child({ workflow: workflowName, script }); const $axios = createScreenedAxios(scriptLog); @@ -418,7 +465,7 @@ function createScriptSandbox({ $vars, $responses, $workflows, - require: createRestrictedRequire($axios), + require: createRestrictedRequire($axios, pluginDir), }; vm.createContext(sandbox, { @@ -470,8 +517,29 @@ export function extractScriptMeta(fn) { * $workflows?: { trigger: (name: string, data?: unknown) => Promise }, * }} opts */ -function instantiateCompiled(compiled, { log, script, workflowName, owner, $workflows }) { - const sandbox = createScriptSandbox({ log, script, workflowName, owner, $workflows }); +/** + * @param {import("vm").Script} compiled + * @param {{ + * log: import("pino").Logger, + * script: string, + * workflowName: string, + * owner?: string, + * $workflows?: { trigger: (name: string, data?: unknown) => Promise }, + * pluginDir?: string | null, + * }} opts + */ +function instantiateCompiled( + compiled, + { log, script, workflowName, owner, $workflows, pluginDir = null }, +) { + const sandbox = createScriptSandbox({ + log, + script, + workflowName, + owner, + $workflows, + pluginDir, + }); return compiled.runInContext(sandbox); } @@ -486,6 +554,7 @@ function instantiateCompiled(compiled, { log, script, workflowName, owner, $work * workflowName?: string, * owner?: string, * $workflows?: { trigger: (name: string, data?: unknown) => Promise }, + * pluginDir?: string | null, * }} [opts] */ export function instantiateScriptSource(script, source, opts = {}) { @@ -496,6 +565,7 @@ export function instantiateScriptSource(script, source, opts = {}) { workflowName: opts.workflowName ?? "inspect", owner: opts.owner ?? "default", $workflows: opts.$workflows, + pluginDir: opts.pluginDir ?? null, }); return { fn, ...extractScriptMeta(fn) }; } @@ -519,17 +589,28 @@ export function inspectScriptSource(script, source) { } function loadCompiledScript(script) { - const filePath = path.join(SCRIPTS_DIR, script); + const resolved = resolveScriptRef(script); + if (resolved.error || !resolved.filePath) { + throw new Error(resolved.error || `script not found: ${script}`); + } + const filePath = resolved.filePath; const { mtimeMs } = fs.statSync(filePath); - const cached = scriptCache.get(script); + const cacheKey = `${resolved.kind}:${resolved.scriptRef}:${filePath}`; + const cached = scriptCache.get(cacheKey); if (cached && cached.mtimeMs === mtimeMs) { - return cached.compiled; + return cached; } const source = fs.readFileSync(filePath, "utf8"); const compiled = compileScriptSource(source, filePath); - scriptCache.set(script, { compiled, mtimeMs }); - return compiled; + const entry = { + compiled, + mtimeMs, + pluginDir: resolved.pluginDir ?? null, + scriptRef: resolved.scriptRef, + }; + scriptCache.set(cacheKey, entry); + return entry; } /** @@ -545,13 +626,14 @@ function loadCompiledScript(script) { * }} opts */ export async function runScript(script, ctx, { log, workflowName, owner, $workflows }) { - const compiled = loadCompiledScript(script); - const fn = instantiateCompiled(compiled, { + const loaded = loadCompiledScript(script); + const fn = instantiateCompiled(loaded.compiled, { log, - script, + script: loaded.scriptRef, workflowName, owner, $workflows, + pluginDir: loaded.pluginDir, }); return await fn(ctx); } diff --git a/packages/server/src/api/scripts.js b/packages/server/src/api/scripts.js index 6d175d0..989f0b1 100644 --- a/packages/server/src/api/scripts.js +++ b/packages/server/src/api/scripts.js @@ -1,13 +1,32 @@ import fs from "fs"; +import path from "path"; import { clearScriptCache, inspectScriptSource, instantiateScriptSource, } from "../../script-sandbox.js"; import * as fsStore from "../../fs-store.js"; +import { + forkCoreScript, + getInstalledPlugin, + listCoreScriptNames, + listInstalledPlugins, + resolveScriptRef, + uninstallPlugin, + createBlankPlugin, +} from "../../plugin-store.js"; +import { + installExamplePlugin, + installPluginFromDirectory, + installPluginFromGit, + installPluginFromZipBuffer, +} from "../../plugin-install.js"; import { createDryRunLogger, safeSerialize } from "./dry-run-logger.js"; import { normalizeStepResult } from "../../step-result.js"; import { resolveConfigRefs } from "../../config-refs.js"; +import { getAppVersion } from "../../app-version.js"; +import { EXAMPLE_PLUGINS_DIR } from "../../paths.js"; +import { pluginScriptRef } from "../../plugin-manifest.js"; /** * @param {{ referencedScripts: () => Set }} registry @@ -18,19 +37,58 @@ export default function scriptsPluginFactory(registry) { */ return async function scriptsPlugin(fastify) { fastify.get("/scripts", async () => { - const scripts = fsStore.listScriptFiles().map((name) => { + const core = listCoreScriptNames().map((name) => { const content = fsStore.readScript(name); const inspected = content == null ? { meta: null, metaError: "script not found" } : inspectScriptSource(name, content); - return { name, hasIcon: fsStore.scriptHasIcon(name), ...inspected }; + return { + name, + kind: "core", + editable: false, + hasIcon: fsStore.scriptHasIcon(name), + ...inspected, + }; }); - return { scripts }; + + const plugins = listInstalledPlugins().map((p) => { + let inspected = { meta: null, metaError: null }; + if (!p.disabled && p.manifest) { + try { + const mainPath = path.join(p.dir, p.manifest.main); + const content = fs.readFileSync(mainPath, "utf8"); + inspected = inspectScriptSource(p.scriptRef, content); + } catch (err) { + inspected = { + meta: null, + metaError: err instanceof Error ? err.message : String(err), + }; + } + } else if (p.compatError) { + inspected = { meta: null, metaError: p.compatError }; + } + return { + name: p.scriptRef, + kind: "plugin", + editable: true, + pluginId: p.id, + disabled: p.disabled, + version: p.manifest?.version ?? null, + hasIcon: false, + ...inspected, + }; + }); + + return { + scripts: [...core, ...plugins], + appVersion: getAppVersion(), + }; }); fastify.get("/scripts/:name/icon", async (req, reply) => { const { name } = /** @type {{ name: string }} */ (req.params); + // Icons only for core scripts today try { fsStore.assertScriptName(name); } catch (err) { @@ -46,64 +104,152 @@ export default function scriptsPluginFactory(registry) { }); fastify.get("/scripts/:name", async (req, reply) => { - const { name } = /** @type {{ name: string }} */ (req.params); - try { - fsStore.assertScriptName(name); - } catch (err) { - return reply.code(err.statusCode ?? 400).send({ error: err.message }); + const rawName = decodeURIComponent( + /** @type {{ name: string }} */ (req.params).name, + ); + const resolved = resolveScriptRef(rawName); + if (resolved.error || !resolved.filePath) { + return reply + .code(404) + .send({ error: resolved.error || "script not found" }); } - const content = fsStore.readScript(name); - if (content == null) return reply.code(404).send({ error: "script not found" }); - return { name, content, hasIcon: fsStore.scriptHasIcon(name), ...inspectScriptSource(name, content) }; + const content = fs.readFileSync(resolved.filePath, "utf8"); + const inspected = inspectScriptSource(resolved.scriptRef, content); + return { + name: resolved.scriptRef, + kind: resolved.kind, + editable: resolved.kind === "plugin", + pluginId: resolved.pluginId ?? null, + content, + hasIcon: + resolved.kind === "core" + ? fsStore.scriptHasIcon(resolved.scriptRef) + : false, + ...inspected, + }; }); + // Core scripts are read-only. Creating/editing bare *.js writes is disabled. + // New user scripts must be plugins (fork / zip / git). fastify.put("/scripts/:name", async (req, reply) => { - const { name } = /** @type {{ name: string }} */ (req.params); - try { - fsStore.assertScriptName(name); - } catch (err) { - return reply.code(err.statusCode ?? 400).send({ error: err.message }); + const rawName = decodeURIComponent( + /** @type {{ name: string }} */ (req.params).name, + ); + const pluginRef = resolveScriptRef(rawName); + if (pluginRef.kind === "core" || !rawName.startsWith("plugin/")) { + // Attempt to treat as core name + try { + fsStore.assertScriptName( + rawName.endsWith(".js") ? rawName : `${rawName}.js`, + ); + } catch { + // continue + } + if (!rawName.startsWith("plugin/")) { + return reply.code(403).send({ + error: + "core scripts are read-only; fork to a plugin or install a plugin", + }); + } } + const body = /** @type {{ content?: string }} */ (req.body ?? {}); if (typeof body.content !== "string") { return reply.code(400).send({ error: "content is required" }); } - const existed = fsStore.readScript(name) != null; - fsStore.writeScript(name, body.content); + + const resolved = resolveScriptRef(rawName); + if (resolved.kind !== "plugin" || !resolved.filePath || !resolved.pluginDir) { + return reply.code(404).send({ + error: resolved.error || "plugin not found (install or fork first)", + }); + } + if (resolved.disabled) { + return reply.code(409).send({ error: resolved.error || "plugin disabled" }); + } + + fs.writeFileSync(resolved.filePath, body.content, "utf8"); clearScriptCache(); - return reply.code(existed ? 200 : 201).send({ - name, - ...inspectScriptSource(name, body.content), + return reply.send({ + name: resolved.scriptRef, + kind: "plugin", + editable: true, + ...inspectScriptSource(resolved.scriptRef, body.content), }); }); fastify.delete("/scripts/:name", async (req, reply) => { - const { name } = /** @type {{ name: string }} */ (req.params); - try { - fsStore.assertScriptName(name); - } catch (err) { - return reply.code(err.statusCode ?? 400).send({ error: err.message }); + const rawName = decodeURIComponent( + /** @type {{ name: string }} */ (req.params).name, + ); + if (!rawName.startsWith("plugin/")) { + return reply.code(403).send({ + error: "core scripts cannot be deleted", + }); } - if (registry.referencedScripts().has(name)) { + const resolved = resolveScriptRef(rawName); + const id = resolved.pluginId; + if (!id) { + // may be installed but disabled — still allow uninstall via plugin id parse + const installed = listInstalledPlugins().find( + (p) => p.scriptRef === rawName || `plugin/${p.id}` === rawName, + ); + if (!installed) { + return reply.code(404).send({ error: "plugin not found" }); + } + if (registry.referencedScripts().has(installed.scriptRef)) { + return reply + .code(409) + .send({ error: "plugin is referenced by a workflow" }); + } + uninstallPlugin(installed.id); + clearScriptCache(); + return { ok: true, restartNeeded: true }; + } + if (registry.referencedScripts().has(resolved.scriptRef)) { return reply .code(409) - .send({ error: "script is referenced by a workflow" }); - } - if (!fsStore.deleteScript(name)) { - return reply.code(404).send({ error: "script not found" }); + .send({ error: "plugin is referenced by a workflow" }); } + uninstallPlugin(id); clearScriptCache(); - return { ok: true }; + return { ok: true, restartNeeded: true }; + }); + + fastify.post("/scripts/:name/fork", async (req, reply) => { + const rawName = decodeURIComponent( + /** @type {{ name: string }} */ (req.params).name, + ); + const body = /** @type {{ id?: string, description?: string }} */ ( + req.body ?? {} + ); + if (typeof body.id !== "string" || !body.id.trim()) { + return reply.code(400).send({ error: "id is required" }); + } + try { + const coreName = rawName.endsWith(".js") ? rawName : `${rawName}.js`; + fsStore.assertScriptName(coreName); + const installed = forkCoreScript(coreName, body.id.trim(), { + description: body.description, + }); + clearScriptCache(); + return reply.code(201).send({ + ...installed, + restartNeeded: true, + warning: + "Plugins run as the JerapahFlow process user. Review code before install.", + }); + } catch (err) { + return reply + .code(/** @type {any} */ (err).statusCode ?? 500) + .send({ error: err instanceof Error ? err.message : String(err) }); + } }); fastify.post("/scripts/:name/dry-run", async (req, reply) => { - const { name } = /** @type {{ name: string }} */ (req.params); - try { - fsStore.assertScriptName(name); - } catch (err) { - return reply.code(err.statusCode ?? 400).send({ error: err.message }); - } - + const rawName = decodeURIComponent( + /** @type {{ name: string }} */ (req.params).name, + ); const body = /** @type {{ content?: string, data?: unknown, context?: unknown, config?: unknown, owner?: string }} */ ( req.body ?? {} ); @@ -121,10 +267,18 @@ export default function scriptsPluginFactory(registry) { } const incomingContext = - body.context != null && typeof body.context === "object" && !Array.isArray(body.context) + body.context != null && + typeof body.context === "object" && + !Array.isArray(body.context) ? body.context : {}; + const resolved = resolveScriptRef(rawName); + const pluginDir = + resolved.kind === "plugin" && !resolved.error + ? resolved.pluginDir ?? null + : null; + const { log, logs } = createDryRunLogger(); const started = Date.now(); @@ -139,13 +293,22 @@ export default function scriptsPluginFactory(registry) { context: incomingContext, config, }; - const { fn, meta, metaError } = instantiateScriptSource(name, body.content, { - log, - workflowName: "dry-run", - owner, - }); + const { fn, meta, metaError } = instantiateScriptSource( + resolved.scriptRef || rawName, + body.content, + { + log, + workflowName: "dry-run", + owner, + pluginDir, + }, + ); const raw = await fn(ctx); - const result = normalizeStepResult(raw, incomingContext, name); + const result = normalizeStepResult( + raw, + incomingContext, + resolved.scriptRef || rawName, + ); return { status: "success", output: safeSerialize(result.output), @@ -158,7 +321,7 @@ export default function scriptsPluginFactory(registry) { metaError, }; } catch (err) { - const inspected = inspectScriptSource(name, body.content); + const inspected = inspectScriptSource(rawName, body.content); return { status: "failed", output: null, @@ -171,5 +334,148 @@ export default function scriptsPluginFactory(registry) { }; } }); + + // --- Plugins --- + + fastify.get("/plugins", async () => { + return { + appVersion: getAppVersion(), + plugins: listInstalledPlugins(), + warning: + "Installing plugins runs third-party code as the JerapahFlow OS user.", + }; + }); + + fastify.post( + "/plugins/create", + { onRequest: [fastify.requireAdmin] }, + async (req, reply) => { + const body = /** @type {{ id?: string, content?: string, description?: string }} */ ( + req.body ?? {} + ); + if (typeof body.id !== "string" || !body.id.trim()) { + return reply.code(400).send({ error: "id is required" }); + } + if (typeof body.content !== "string") { + return reply.code(400).send({ error: "content is required" }); + } + try { + const installed = createBlankPlugin(body.id.trim(), body.content, { + description: body.description, + }); + clearScriptCache(); + return reply.code(201).send({ + ...installed, + restartNeeded: true, + warning: + "Plugins run as the JerapahFlow process user. Review code before install.", + }); + } catch (err) { + return reply + .code(/** @type {any} */ (err).statusCode ?? 500) + .send({ error: err instanceof Error ? err.message : String(err) }); + } + }, + ); + + fastify.post( + "/plugins/install", + { onRequest: [fastify.requireAdmin] }, + async (req, reply) => { + const body = /** @type {{ + source?: string, + url?: string, + ref?: string, + path?: string, + exampleId?: string, + zipBase64?: string, + overwrite?: boolean, + }} */ (req.body ?? {}); + + try { + let installed; + if (body.source === "git") { + if (!body.url) { + return reply.code(400).send({ error: "url is required" }); + } + installed = await installPluginFromGit(body.url, { + ref: body.ref, + overwrite: Boolean(body.overwrite), + }); + } else if (body.source === "example") { + const id = body.exampleId || "get-current-time"; + installed = await installExamplePlugin(id, { + overwrite: Boolean(body.overwrite), + }); + } else if (body.source === "dir") { + if (!body.path) { + return reply.code(400).send({ error: "path is required" }); + } + // Only allow examples/ or existing staging under plugins for safety + const abs = path.resolve(body.path); + const allowed = + abs.startsWith(EXAMPLE_PLUGINS_DIR + path.sep) || + abs.startsWith(EXAMPLE_PLUGINS_DIR); + if (!allowed) { + return reply.code(403).send({ + error: "dir install only allowed under examples/plugins", + }); + } + installed = installPluginFromDirectory(abs, { + overwrite: Boolean(body.overwrite), + }); + } else if (body.source === "zip") { + if (!body.zipBase64) { + return reply.code(400).send({ error: "zipBase64 is required" }); + } + const buf = Buffer.from(body.zipBase64, "base64"); + installed = await installPluginFromZipBuffer(buf, { + overwrite: Boolean(body.overwrite), + }); + } else { + return reply.code(400).send({ + error: "source must be git | zip | example | dir", + }); + } + clearScriptCache(); + return reply.code(201).send({ + ...installed, + scriptRef: pluginScriptRef(installed.id), + restartNeeded: true, + warning: + "Plugins run as the JerapahFlow process user. Review code before install. Drain-restart workers to load new dependencies.", + }); + } catch (err) { + return reply + .code(/** @type {any} */ (err).statusCode ?? 500) + .send({ error: err instanceof Error ? err.message : String(err) }); + } + }, + ); + + fastify.delete( + "/plugins/:id", + { onRequest: [fastify.requireAdmin] }, + async (req, reply) => { + const { id } = /** @type {{ id: string }} */ (req.params); + try { + const scriptRef = pluginScriptRef(id); + if (registry.referencedScripts().has(scriptRef)) { + return reply + .code(409) + .send({ error: "plugin is referenced by a workflow" }); + } + uninstallPlugin(id); + clearScriptCache(); + return { ok: true, restartNeeded: true }; + } catch (err) { + return reply + .code(/** @type {any} */ (err).statusCode ?? 500) + .send({ error: err instanceof Error ? err.message : String(err) }); + } + }, + ); + + void getInstalledPlugin; }; } diff --git a/packages/server/test/plugins-smoke.js b/packages/server/test/plugins-smoke.js new file mode 100644 index 0000000..29776e0 --- /dev/null +++ b/packages/server/test/plugins-smoke.js @@ -0,0 +1,118 @@ +/** + * Smoke: core vs plugin scripts, fork, example install, resolve, run. + * + * Run: + * JFLOW_PLUGINS_DIR=packages/server/data/plugins-smoke-test \ + * JFLOW_DB_PATH=packages/server/data/plugins-smoke.db \ + * node packages/server/test/plugins-smoke.js + */ +import assert from "node:assert/strict"; +import fs from "fs"; +import { migrate, db } from "../db.js"; +import { getAppVersion, satisfiesRange } from "../app-version.js"; +import { + forkCoreScript, + resolveScriptRef, + uninstallPlugin, + listInstalledPlugins, + createBlankPlugin, +} from "../plugin-store.js"; +import { installExamplePlugin } from "../plugin-install.js"; +import { + runScript, + clearScriptCache, + inspectScriptSource, +} from "../script-sandbox.js"; +import { PLUGINS_DIR } from "../paths.js"; +import pino from "pino"; + +const silent = pino({ level: "silent" }); + +async function main() { + assert.equal(getAppVersion(), "0.1.0"); + assert.equal(satisfiesRange("0.1.0", ">=0.1.0 <1.0.0"), true); + assert.equal(satisfiesRange("1.0.0", ">=0.1.0 <1.0.0"), false); + assert.equal(satisfiesRange("0.2.0", ">=0.1.0 <1.0.0"), true); + + await migrate(); + + if (fs.existsSync(PLUGINS_DIR)) { + fs.rmSync(PLUGINS_DIR, { recursive: true, force: true }); + } + fs.mkdirSync(PLUGINS_DIR, { recursive: true }); + + const core = resolveScriptRef("fetch-http.js"); + assert.equal(core.kind, "core"); + assert.ok(core.filePath && fs.existsSync(core.filePath)); + assert.ok(resolveScriptRef("nope.js").error?.includes("not found")); + + const example = await installExamplePlugin("get-current-time", { + overwrite: true, + }); + assert.equal(example.id, "get-current-time"); + assert.equal(example.scriptRef, "plugin/get-current-time"); + + clearScriptCache(); + const pluginResolved = resolveScriptRef("plugin/get-current-time"); + assert.equal(pluginResolved.kind, "plugin"); + assert.ok(pluginResolved.filePath); + + const result = await runScript( + "plugin/get-current-time", + { data: null, context: {}, config: null }, + { log: silent, workflowName: "smoke", owner: "default" }, + ); + assert.ok(result?.output?.datetime); + + const forked = forkCoreScript("jsonata.js", "jsonata-smoke-fork"); + assert.equal(forked.scriptRef, "plugin/jsonata-smoke-fork"); + clearScriptCache(); + assert.equal(resolveScriptRef("plugin/jsonata-smoke-fork").kind, "plugin"); + + const blank = createBlankPlugin( + "blank-smoke", + `export default async function main(ctx) { return { output: { ok: true }, context: ctx.context ?? {} }; }`, + ); + assert.equal(blank.scriptRef, "plugin/blank-smoke"); + clearScriptCache(); + const blankRun = await runScript( + "plugin/blank-smoke", + { data: 1, context: {}, config: null }, + { log: silent, workflowName: "smoke", owner: "default" }, + ); + assert.equal(blankRun.output.ok, true); + + let hit = false; + try { + forkCoreScript("ntfy.js", "ntfy"); + } catch (err) { + hit = true; + assert.match(String(err.message), /collides/); + } + assert.equal(hit, true); + + const meta = inspectScriptSource( + "fetch-http.js", + fs.readFileSync(core.filePath, "utf8"), + ); + assert.ok(meta); + + assert.ok(listInstalledPlugins().some((p) => p.id === "get-current-time")); + + uninstallPlugin("jsonata-smoke-fork"); + uninstallPlugin("blank-smoke"); + uninstallPlugin("get-current-time"); + + console.log("plugins-smoke: ok"); + await db.destroy(); +} + +main().catch(async (err) => { + console.error(err); + try { + await db.destroy(); + } catch { + // ignore + } + process.exit(1); +}); diff --git a/packages/server/workflows/default/cron-example.yaml b/packages/server/workflows/default/cron-example.yaml index b829f36..f29501a 100644 --- a/packages/server/workflows/default/cron-example.yaml +++ b/packages/server/workflows/default/cron-example.yaml @@ -2,7 +2,7 @@ name: cron example description: | this workflow triggered by cron scripts: - - script: get-current-time.js + - script: plugin/get-current-time - set: expression: '{"message": context.datetime}' - script: ntfy.js diff --git a/packages/server/workflows/default/test.yaml b/packages/server/workflows/default/test.yaml index 4c1cb48..e8ec686 100644 --- a/packages/server/workflows/default/test.yaml +++ b/packages/server/workflows/default/test.yaml @@ -1,6 +1,6 @@ name: jsonata scripts: - - get-current-time.js + - plugin/get-current-time - script: jsonata.js config: expression: '{"message": data.datetime & " " & data.processId}' diff --git a/packages/server/workflows/default/time-and-comic-to-ntfy.yaml b/packages/server/workflows/default/time-and-comic-to-ntfy.yaml index 4268411..ab1156d 100644 --- a/packages/server/workflows/default/time-and-comic-to-ntfy.yaml +++ b/packages/server/workflows/default/time-and-comic-to-ntfy.yaml @@ -3,7 +3,7 @@ description: > Fan-in from two scripts (current time + monkeyuser comic), then send to ntfy. scripts: - id: time - script: get-current-time.js + script: plugin/get-current-time - id: comic script: fetch-html.js diff --git a/packages/server/workflows/default/time-to-ntfy-example.yaml b/packages/server/workflows/default/time-to-ntfy-example.yaml index df6144c..34404f9 100644 --- a/packages/server/workflows/default/time-to-ntfy-example.yaml +++ b/packages/server/workflows/default/time-to-ntfy-example.yaml @@ -2,7 +2,7 @@ name: time to ntfy example description: | this workflow will send a message to ntfy with the current time scripts: - - script: get-current-time.js + - script: plugin/get-current-time config: key: "" - script: ntfy.js diff --git a/packages/web/package.json b/packages/web/package.json index 0c484f9..64e7136 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -1,7 +1,7 @@ { "name": "@jerapah-flow/web", "private": true, - "version": "1.0.0", + "version": "0.1.0", "type": "module", "scripts": { "dev": "vite", diff --git a/packages/web/src/api/hooks.js b/packages/web/src/api/hooks.js index 2b80df5..d7fed6d 100644 --- a/packages/web/src/api/hooks.js +++ b/packages/web/src/api/hooks.js @@ -89,6 +89,47 @@ export function useSaveScript() { qc.invalidateQueries({ queryKey: ["scripts"] }); qc.invalidateQueries({ queryKey: ["scripts", vars.name] }); qc.invalidateQueries({ queryKey: ["dashboard"] }); + qc.invalidateQueries({ queryKey: ["ops-status"] }); + }, + }); +} + +export function useCreatePlugin() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: async ({ id, content, description }) => + (await api.post("/plugins/create", { id, content, description })).data, + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["scripts"] }); + qc.invalidateQueries({ queryKey: ["ops-status"] }); + }, + }); +} + +export function useForkScript() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: async ({ name, id, description }) => + ( + await api.post(`/scripts/${encodeURIComponent(name)}/fork`, { + id, + description, + }) + ).data, + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["scripts"] }); + qc.invalidateQueries({ queryKey: ["ops-status"] }); + }, + }); +} + +export function useInstallPlugin() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: async (body) => (await api.post("/plugins/install", body)).data, + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["scripts"] }); + qc.invalidateQueries({ queryKey: ["ops-status"] }); }, }); } diff --git a/packages/web/src/lib/workflow-doc.js b/packages/web/src/lib/workflow-doc.js index 647b48b..8c77c8e 100644 --- a/packages/web/src/lib/workflow-doc.js +++ b/packages/web/src/lib/workflow-doc.js @@ -5,7 +5,7 @@ const HTTP_METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS" export const NEW_WORKFLOW_YAML = `name: new workflow scripts: - - get-current-time.js + - plugin/get-current-time triggers: - type: HTTP method: POST diff --git a/packages/web/src/pages/ScriptEditPage.jsx b/packages/web/src/pages/ScriptEditPage.jsx index d0ee3fb..63bb4f1 100644 --- a/packages/web/src/pages/ScriptEditPage.jsx +++ b/packages/web/src/pages/ScriptEditPage.jsx @@ -1,39 +1,57 @@ import { useEffect, useState } from "react"; import { Link, useLocation, useNavigate, useParams } from "react-router-dom"; -import { LuArrowLeft, LuPlay, LuSave } from "react-icons/lu"; +import { LuArrowLeft, LuCopy, LuPlay, LuSave } from "react-icons/lu"; import { errorMessage } from "../api/client.js"; -import { useSaveScript, useScript } from "../api/hooks.js"; +import { + useCreatePlugin, + useForkScript, + useSaveScript, + useScript, +} from "../api/hooks.js"; import { CodeEditor } from "../components/CodeEditor.jsx"; import { ScriptIcon } from "../components/ScriptIcon.jsx"; import { ScriptMetaPanel } from "../components/ScriptMetaPanel.jsx"; -import { NEW_SCRIPT_TEMPLATE, normalizeScriptName } from "../lib/script.js"; +import { NEW_SCRIPT_TEMPLATE } from "../lib/script.js"; import { useNotifications } from "../notifications.jsx"; +function normalizePluginId(raw) { + return String(raw ?? "") + .trim() + .toLowerCase() + .replace(/\.js$/i, "") + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, ""); +} + export function ScriptNewPage() { const navigate = useNavigate(); const location = useLocation(); - const [name, setName] = useState(""); + const [id, setId] = useState(""); const [content, setContent] = useState( location.state?.content ?? NEW_SCRIPT_TEMPLATE, ); - const save = useSaveScript(); + const create = useCreatePlugin(); + const { notify } = useNotifications(); function onSave(e) { e.preventDefault(); - const file = normalizeScriptName(name); - if (!file) return; - save.mutate( - { name: file, content }, + const pluginId = normalizePluginId(id); + if (!pluginId) return; + create.mutate( + { id: pluginId, content }, { - onSuccess: () => navigate(`/scripts/${encodeURIComponent(file)}/edit`), + onSuccess: (data) => { + notify.success("Plugin created — drain-restart to load workers"); + navigate(`/scripts/${encodeURIComponent(data.scriptRef)}/edit`); + }, }, ); } function openDryRun() { - const file = normalizeScriptName(name); - if (!file) return; - navigate(`/scripts/${encodeURIComponent(file)}/dry-run`, { + const pluginId = normalizePluginId(id); + if (!pluginId) return; + navigate(`/scripts/${encodeURIComponent(`plugin/${pluginId}`)}/dry-run`, { state: { content }, }); } @@ -44,12 +62,12 @@ export function ScriptNewPage() { -

New script

+

New plugin

+
+ + User scripts are plugins (plugin/<id>). Core scripts + are read-only — fork them instead. Installing plugins runs code as the + JerapahFlow process user. + +
+
setName(e.target.value)} + className="input input-sm w-full max-w-md shrink-0 font-mono" + placeholder="my-script (becomes plugin/my-script)" + value={id} + onChange={(e) => setId(e.target.value)} required />
- {save.isError ? ( -

{errorMessage(save.error)}

+ {create.isError ? ( +

{errorMessage(create.error)}

) : null}
@@ -92,8 +118,12 @@ export function ScriptEditPage() { const { notify } = useNotifications(); const existing = useScript(name); const save = useSaveScript(); + const fork = useForkScript(); const [content, setContent] = useState(""); const [contentReady, setContentReady] = useState(false); + const [forkId, setForkId] = useState(""); + + const isCore = existing.data?.kind === "core" || existing.data?.editable === false; useEffect(() => { if (existing.isLoading) return; @@ -105,9 +135,10 @@ export function ScriptEditPage() { function onSave(e) { e.preventDefault(); + if (isCore) return; save.mutate( { name, content }, - { onSuccess: () => notify.success("Script saved") }, + { onSuccess: () => notify.success("Plugin saved") }, ); } @@ -117,6 +148,21 @@ export function ScriptEditPage() { }); } + function onFork(e) { + e.preventDefault(); + const id = normalizePluginId(forkId); + if (!id) return; + fork.mutate( + { name, id }, + { + onSuccess: (data) => { + notify.success("Forked to plugin — drain-restart recommended"); + navigate(`/scripts/${encodeURIComponent(data.scriptRef)}/edit`); + }, + }, + ); + } + if (existing.isLoading) { return (
@@ -145,24 +191,63 @@ export function ScriptEditPage() {

{name}

+ + {isCore ? "core" : "plugin"} +
- + {!isCore ? ( + + ) : null}
+ {isCore ? ( +
+ Core scripts are read-only. Fork to create an editable plugin copy. +
+ ) : null} + + {isCore ? ( +
+ setForkId(e.target.value)} + /> + + {fork.isError ? ( + {errorMessage(fork.error)} + ) : null} +
+ ) : null} +
- + {} : setContent} + height="100%" + readOnly={isCore} + />
diff --git a/packages/web/src/pages/ScriptsPage.jsx b/packages/web/src/pages/ScriptsPage.jsx index 83aa616..a53c358 100644 --- a/packages/web/src/pages/ScriptsPage.jsx +++ b/packages/web/src/pages/ScriptsPage.jsx @@ -1,11 +1,17 @@ import { useMemo, useState } from "react"; import { Link, Navigate, useNavigate, useSearchParams } from "react-router-dom"; -import { LuPencil, LuPlay, LuPlus, LuSearch, LuTrash2 } from "react-icons/lu"; +import { LuCopy, LuPencil, LuPlay, LuPlus, LuSearch, LuTrash2 } from "react-icons/lu"; import { errorMessage } from "../api/client.js"; -import { useDeleteScript, useScripts } from "../api/hooks.js"; +import { + useDeleteScript, + useForkScript, + useInstallPlugin, + useScripts, +} from "../api/hooks.js"; import { ScriptIcon } from "../components/ScriptIcon.jsx"; import { TagBadge } from "../components/TagBadge.jsx"; import { scriptTags } from "../lib/script.js"; +import { useNotifications } from "../notifications.jsx"; export function ScriptsPage() { const [params] = useSearchParams(); @@ -14,7 +20,12 @@ export function ScriptsPage() { const { data: scripts = [], isLoading } = useScripts(); const [confirmDelete, setConfirmDelete] = useState(null); const [query, setQuery] = useState(""); + const [forkFor, setForkFor] = useState(null); + const [forkId, setForkId] = useState(""); const del = useDeleteScript(); + const fork = useForkScript(); + const install = useInstallPlugin(); + const { notify } = useNotifications(); const visible = useMemo(() => { const term = query.trim().toLowerCase(); @@ -23,9 +34,11 @@ export function ScriptsPage() { const name = typeof s === "string" ? s : s.name ?? ""; const description = typeof s === "string" ? "" : s.meta?.description ?? ""; const tags = typeof s === "string" ? [] : scriptTags(s.meta); + const kind = typeof s === "string" ? "" : s.kind ?? ""; return ( name.toLowerCase().includes(term) || description.toLowerCase().includes(term) || + kind.toLowerCase().includes(term) || tags.some((t) => t.toLowerCase().includes(term)) ); }); @@ -50,13 +63,33 @@ export function ScriptsPage() { onChange={(e) => setQuery(e.target.value)} /> + - Add + Add plugin
+ {install.isError ? ( +

{errorMessage(install.error)}

+ ) : null} + {isLoading ? ( ) : scripts.length === 0 ? ( @@ -71,6 +104,8 @@ export function ScriptsPage() { const metaError = typeof s === "string" ? null : s.metaError; const hasIcon = typeof s === "string" ? undefined : s.hasIcon; const tags = typeof s === "string" ? [] : scriptTags(s.meta); + const kind = typeof s === "string" ? "core" : s.kind ?? "core"; + const isCore = kind === "core"; return (
+
+ + {kind} + +

{metaError ? ( {metaError} @@ -114,25 +156,47 @@ export function ScriptsPage() { type="button" className="btn btn-ghost btn-xs" title="Dry run" - onClick={() => navigate(`/scripts/${encodeURIComponent(name)}/dry-run`)} + onClick={() => + navigate(`/scripts/${encodeURIComponent(name)}/dry-run`) + } > - - - - + {isCore ? ( + + ) : ( + + + + )} + {!isCore ? ( + + ) : null}

@@ -141,20 +205,79 @@ export function ScriptsPage() { )} - {confirmDelete ? ( + {forkFor ? (
-

Delete {confirmDelete}?

- {del.isError ? ( -

{errorMessage(del.error)}

+

Fork {forkFor}

+

+ Creates plugin/<id> from this core script. +

+ setForkId(e.target.value)} + /> + {fork.isError ? ( +

{errorMessage(fork.error)}

) : null}
- +
+
+
+ +
+ + ) : null} + + {confirmDelete ? ( + +
+

Delete {confirmDelete}?

+

This uninstalls the plugin.

+ {del.isError ? ( +

{errorMessage(del.error)}

+ ) : null} +
+ +