diff --git a/.gitignore b/.gitignore index a71135d..e2debb1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ node_modules/ .pnpm-store/ +# Instance data (SQLite, live workflows, control-state, backups) data/ +# Process logs logs/ *.db *.db-* @@ -10,7 +12,7 @@ packages/web/dist # Personal/local scripts and workflows (not for the repo) debug-*.js -# Legacy live workflow tree (migrated to packages/server/data/workflows/) +# Legacy live workflow trees packages/server/workflows/ # Plugin install staging and per-plugin deps diff --git a/AGENTS.md b/AGENTS.md index bb7d59a..3002e5e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,14 +4,14 @@ This file tells agents how to add a **user plugin**. Do not put personal or site ## Workflows (instance data) -Live workflows are **not** product source. They live under `packages/server/data/workflows//` (gitignored; override with `JFLOW_WORKFLOWS_DIR`). +Live workflows are **not** product source. They live under `data/workflows//` (gitignored; override with `JFLOW_WORKFLOWS_DIR`). | Kind | In git? | Path | |---|---|---| -| Live / personal YAML | No | `packages/server/data/workflows//` | +| Live / personal YAML | No | `data/workflows//` | | Example presets | Yes | `examples/workflows/*.yaml` (copy into editor only; runner does not load them) | -Do **not** add personal YAML under `packages/server/`, `examples/workflows/`, or `packages/server/data/workflows/`. Prefer owner `local`. Example presets must use **core** scripts only (no `plugin/…` that requires install). +Do **not** add personal YAML under `packages/server/`, `examples/workflows/`, or `data/workflows/`. Prefer owner `local`. Example presets must use **core** scripts only (no `plugin/…` that requires install). ## Where things live diff --git a/README.md b/README.md index e985f0e..13a663e 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ pnpm --dir packages/server reset-admin -- --username admin --password 'your-pass | Kind | Loaded by runner? | Location | |---|---|---| -| **Live workflows** | Yes | `packages/server/data/workflows//` (gitignored) | +| **Live workflows** | Yes | `data/workflows//` (gitignored) | | **Example presets** | No | `examples/workflows/*.yaml` — offered when creating a new workflow | - Live YAML is **instance data**, same as SQLite and secrets — not product source. New resources use owner `local` (owner remains in storage/URLs for a possible future multi-tenant mode; the UI hides it). @@ -63,7 +63,8 @@ pnpm --dir packages/server reset-admin -- --username admin --password 'your-pass - Override the live store in tests with `JFLOW_WORKFLOWS_DIR`. ```bash -# Smoke +# Smoke (isolated under packages/server/data — not the live instance tree) +JFLOW_DATA_DIR=packages/server/data \ JFLOW_PLUGINS_DIR=packages/server/data/plugins-smoke-test \ JFLOW_DB_PATH=packages/server/data/plugins-smoke.db \ node packages/server/test/plugins-smoke.js @@ -144,7 +145,7 @@ Admin UI route **Ops** (`/ops`) talks to the control process. | Drain restart | Pause → wait active=0 → stop children → migrate → recreate → resume | | Force restart | Same without waiting (interrupts active runs; orphans marked `worker_lost`) | -Desired state is stored in `packages/server/data/control-state.json` (generation, worker count, restart-needed). Plugin installs (later) bump generation and set restart-needed; you apply with Drain restart. +Desired state is stored in `data/control-state.json` (generation, worker count, restart-needed). Plugin installs (later) bump generation and set restart-needed; you apply with Drain restart. ## Environment @@ -152,8 +153,10 @@ Desired state is stored in `packages/server/data/control-state.json` (generation |---|---|---| | `JFLOW_JWT_SECRET` | `jflow-dev-secret` (dev only) | **Required in production**. | | `JFLOW_SECRETS_KEY` | `jflow-dev-secrets-key` (dev only) | Master key for named secrets. **Required in production**. Changing it makes existing secrets unreadable. 64 hex chars are used as a raw AES-256 key; any other string is derived with scrypt. | -| `JFLOW_DB_PATH` | `packages/server/data/jerapah-flow.db` | SQLite file. | -| `JFLOW_WORKFLOWS_DIR` | `packages/server/data/workflows` | Live workflow YAML (instance data). | +| `JFLOW_DATA_DIR` | `data/` | Instance data root (SQLite, workflows, control-state, backups, trash). Falls back to `packages/server/data` if that tree still has the db or workflows. | +| `JFLOW_DB_PATH` | `data/jerapah-flow.db` | SQLite file. | +| `JFLOW_WORKFLOWS_DIR` | `data/workflows` | Live workflow YAML (instance data). | +| `JFLOW_LOGS_DIR` | `logs/` | Rolling process logs. | | `REDIS_URL` | `redis://127.0.0.1:6379` | Redis for BullMQ workflow queue. **Required** — the server will not start if Redis is unreachable. | | `REDIS_PASS` | — | Optional Redis AUTH password (sent via ioredis `password`). Prefer this over embedding credentials in `REDIS_URL` so logs stay clean. | | `JFLOW_QUEUE_NAME` | `jerapah-workflows` | BullMQ queue name. | diff --git a/packages/server/app-version.js b/packages/server/app-version.js index ae46a37..c5d0628 100644 --- a/packages/server/app-version.js +++ b/packages/server/app-version.js @@ -1,8 +1,8 @@ import fs from "fs"; import path from "path"; -import { SERVER_ROOT } from "./paths.js"; +import { REPO_ROOT } from "./paths.js"; -const ROOT_PKG = path.resolve(SERVER_ROOT, "../../package.json"); +const ROOT_PKG = path.join(REPO_ROOT, "package.json"); /** * JerapahFlow app version from the monorepo root package.json. diff --git a/packages/server/package.json b/packages/server/package.json index 97f4ea1..97367a8 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -13,8 +13,8 @@ "start:control": "node control.js", "start:web": "node web-server.js", "migrate": "node -e \"import('./db.js').then((m) => m.migrate().then(() => process.exit(0)))\"", - "test:plugins": "JFLOW_PLUGINS_DIR=./data/plugins-smoke-test JFLOW_DB_PATH=./data/plugins-smoke.db node test/plugins-smoke.js", - "test:workflow-history": "JFLOW_WORKFLOWS_DIR=./data/workflow-history-smoke JFLOW_DB_PATH=./data/workflow-history-smoke.db node test/workflow-history-smoke.js", + "test:plugins": "JFLOW_DATA_DIR=./data JFLOW_PLUGINS_DIR=./data/plugins-smoke-test JFLOW_DB_PATH=./data/plugins-smoke.db node test/plugins-smoke.js", + "test:workflow-history": "JFLOW_DATA_DIR=./data JFLOW_WORKFLOWS_DIR=./data/workflow-history-smoke JFLOW_DB_PATH=./data/workflow-history-smoke.db node test/workflow-history-smoke.js", "reset-admin": "node reset-admin.js", "test:profiles": "node test/profiles-smoke.js", "test:set-dry-run": "node test/set-dry-run-smoke.js", diff --git a/packages/server/paths.js b/packages/server/paths.js index e991bec..9b2dcf9 100644 --- a/packages/server/paths.js +++ b/packages/server/paths.js @@ -1,25 +1,49 @@ +import fs from "fs"; import path from "path"; import { fileURLToPath } from "url"; export const SERVER_ROOT = path.dirname(fileURLToPath(import.meta.url)); +export const REPO_ROOT = path.resolve(SERVER_ROOT, "../.."); export const SCRIPTS_DIR = path.join(SERVER_ROOT, "scripts"); -export const DATA_DIR = path.join(SERVER_ROOT, "data"); + +/** Prefer `preferred` unless only `legacy` already has files. */ +function existingDir(preferred, legacy, probe) { + const has = (dir) => + probe ? probe(dir) : fs.existsSync(dir); + if (has(preferred) || !has(legacy)) return preferred; + return legacy; +} + +function hasInstanceData(dir) { + return ( + fs.existsSync(path.join(dir, "jerapah-flow.db")) || + fs.existsSync(path.join(dir, "workflows")) + ); +} + +/** Instance data (SQLite, live workflows, control-state). Not product source. */ +export const DATA_DIR = path.resolve( + process.env.JFLOW_DATA_DIR ?? + existingDir( + path.join(REPO_ROOT, "data"), + path.join(SERVER_ROOT, "data"), + hasInstanceData, + ), +); /** Live instance workflows (not shipped in git). Override for tests. */ -export const WORKFLOWS_DIR = - process.env.JFLOW_WORKFLOWS_DIR ?? path.join(DATA_DIR, "workflows"); +export const WORKFLOWS_DIR = path.resolve( + process.env.JFLOW_WORKFLOWS_DIR ?? path.join(DATA_DIR, "workflows"), +); /** User plugins (repo-root /plugins, outside the pnpm workspace). */ -export const PLUGINS_DIR = - process.env.JFLOW_PLUGINS_DIR ?? - path.resolve(SERVER_ROOT, "../../plugins"); +export const PLUGINS_DIR = path.resolve( + process.env.JFLOW_PLUGINS_DIR ?? path.join(REPO_ROOT, "plugins"), +); /** Example plugin sources shipped with the repo. */ -export const EXAMPLE_PLUGINS_DIR = path.resolve( - SERVER_ROOT, - "../../examples/plugins", -); +export const EXAMPLE_PLUGINS_DIR = path.join(REPO_ROOT, "examples/plugins"); /** Example workflow YAML presets (not loaded by the runner). */ -export const EXAMPLE_WORKFLOWS_DIR = path.resolve( - SERVER_ROOT, - "../../examples/workflows", +export const EXAMPLE_WORKFLOWS_DIR = path.join(REPO_ROOT, "examples/workflows"); +export const LOGS_DIR = path.resolve( + process.env.JFLOW_LOGS_DIR ?? + existingDir(path.join(REPO_ROOT, "logs"), path.join(SERVER_ROOT, "logs")), ); -export const LOGS_DIR = path.join(SERVER_ROOT, "logs"); -export const WEB_DIST = path.resolve(SERVER_ROOT, "../web/dist"); +export const WEB_DIST = path.join(REPO_ROOT, "packages/web/dist"); diff --git a/packages/server/pm2-bridge.js b/packages/server/pm2-bridge.js index d7b2850..e59fb44 100644 --- a/packages/server/pm2-bridge.js +++ b/packages/server/pm2-bridge.js @@ -1,8 +1,6 @@ import path from "path"; import pm2 from "pm2"; -import { SERVER_ROOT } from "./paths.js"; - -const REPO_ROOT = path.resolve(SERVER_ROOT, "../.."); +import { REPO_ROOT, SERVER_ROOT } from "./paths.js"; export const PM2_HTTP_NAME = "jflow-http"; export const PM2_WORKER_NAME = "jflow-worker"; diff --git a/packages/server/test/plugins-smoke.js b/packages/server/test/plugins-smoke.js index 29776e0..c0fc7ba 100644 --- a/packages/server/test/plugins-smoke.js +++ b/packages/server/test/plugins-smoke.js @@ -2,6 +2,7 @@ * Smoke: core vs plugin scripts, fork, example install, resolve, run. * * Run: + * JFLOW_DATA_DIR=packages/server/data \ * JFLOW_PLUGINS_DIR=packages/server/data/plugins-smoke-test \ * JFLOW_DB_PATH=packages/server/data/plugins-smoke.db \ * node packages/server/test/plugins-smoke.js