chore(paths): refactor data directory structure and update references

- Updated paths for instance data, workflows, and logs to use a unified `data/` directory.
- Adjusted related documentation in AGENTS.md and README.md to reflect the new data structure.
- Refactored path handling in server files to ensure consistency and clarity in data management.
- Enhanced test scripts to align with the new directory structure for improved organization.

.gitignore

@@ -1,6 +1,8 @@
 node_modules/
 .pnpm-store/
+# Instance data (SQLite, live workflows, control-state, backups)
 data/
+# Process logs
 logs/
 *.db
 *.db-*
@@ -10,7 +12,7 @@ packages/web/dist
 # Personal/local scripts and workflows (not for the repo)
 debug-*.js
 
-# Legacy live workflow tree (migrated to packages/server/data/workflows/)
+# Legacy live workflow trees
 packages/server/workflows/
 
 # Plugin install staging and per-plugin deps

AGENTS.md

@@ -4,14 +4,14 @@ This file tells agents how to add a **user plugin**. Do not put personal or site
 
 ## Workflows (instance data)
 
-Live workflows are **not** product source. They live under `packages/server/data/workflows/<owner>/` (gitignored; override with `JFLOW_WORKFLOWS_DIR`).
+Live workflows are **not** product source. They live under `data/workflows/<owner>/` (gitignored; override with `JFLOW_WORKFLOWS_DIR`).
 
 | Kind | In git? | Path |
 |---|---|---|
-| Live / personal YAML | No | `packages/server/data/workflows/<owner>/` |
+| Live / personal YAML | No | `data/workflows/<owner>/` |
 | Example presets | Yes | `examples/workflows/*.yaml` (copy into editor only; runner does not load them) |
 
-Do **not** add personal YAML under `packages/server/`, `examples/workflows/`, or `packages/server/data/workflows/`. Prefer owner `local`. Example presets must use **core** scripts only (no `plugin/…` that requires install).
+Do **not** add personal YAML under `packages/server/`, `examples/workflows/`, or `data/workflows/`. Prefer owner `local`. Example presets must use **core** scripts only (no `plugin/…` that requires install).
 
 ## Where things live
 

README.md

@@ -55,7 +55,7 @@ pnpm --dir packages/server reset-admin -- --username admin --password 'your-pass
 
 | Kind | Loaded by runner? | Location |
 |---|---|---|
-| **Live workflows** | Yes | `packages/server/data/workflows/<owner>/` (gitignored) |
+| **Live workflows** | Yes | `data/workflows/<owner>/` (gitignored) |
 | **Example presets** | No | `examples/workflows/*.yaml` — offered when creating a new workflow |
 
 - Live YAML is **instance data**, same as SQLite and secrets — not product source. New resources use owner `local` (owner remains in storage/URLs for a possible future multi-tenant mode; the UI hides it).
@@ -63,7 +63,8 @@ pnpm --dir packages/server reset-admin -- --username admin --password 'your-pass
 - Override the live store in tests with `JFLOW_WORKFLOWS_DIR`.
 
 ```bash
-# Smoke
+# Smoke (isolated under packages/server/data — not the live instance tree)
+JFLOW_DATA_DIR=packages/server/data \
 JFLOW_PLUGINS_DIR=packages/server/data/plugins-smoke-test \
 JFLOW_DB_PATH=packages/server/data/plugins-smoke.db \
 node packages/server/test/plugins-smoke.js
@@ -144,7 +145,7 @@ Admin UI route **Ops** (`/ops`) talks to the control process.
 | Drain restart | Pause → wait active=0 → stop children → migrate → recreate → resume |
 | Force restart | Same without waiting (interrupts active runs; orphans marked `worker_lost`) |
 
-Desired state is stored in `packages/server/data/control-state.json` (generation, worker count, restart-needed). Plugin installs (later) bump generation and set restart-needed; you apply with Drain restart.
+Desired state is stored in `data/control-state.json` (generation, worker count, restart-needed). Plugin installs (later) bump generation and set restart-needed; you apply with Drain restart.
 
 ## Environment
 
@@ -152,8 +153,10 @@ Desired state is stored in `packages/server/data/control-state.json` (generation
 |---|---|---|
 | `JFLOW_JWT_SECRET` | `jflow-dev-secret` (dev only) | **Required in production**. |
 | `JFLOW_SECRETS_KEY` | `jflow-dev-secrets-key` (dev only) | Master key for named secrets. **Required in production**. Changing it makes existing secrets unreadable. 64 hex chars are used as a raw AES-256 key; any other string is derived with scrypt. |
-| `JFLOW_DB_PATH` | `packages/server/data/jerapah-flow.db` | SQLite file. |
+| `JFLOW_DATA_DIR` | `data/` | Instance data root (SQLite, workflows, control-state, backups, trash). Falls back to `packages/server/data` if that tree still has the db or workflows. |
-| `JFLOW_WORKFLOWS_DIR` | `packages/server/data/workflows` | Live workflow YAML (instance data). |
+| `JFLOW_DB_PATH` | `data/jerapah-flow.db` | SQLite file. |
+| `JFLOW_WORKFLOWS_DIR` | `data/workflows` | Live workflow YAML (instance data). |
+| `JFLOW_LOGS_DIR` | `logs/` | Rolling process logs. |
 | `REDIS_URL` | `redis://127.0.0.1:6379` | Redis for BullMQ workflow queue. **Required** — the server will not start if Redis is unreachable. |
 | `REDIS_PASS` | — | Optional Redis AUTH password (sent via ioredis `password`). Prefer this over embedding credentials in `REDIS_URL` so logs stay clean. |
 | `JFLOW_QUEUE_NAME` | `jerapah-workflows` | BullMQ queue name. |

packages/server/app-version.js

@@ -1,8 +1,8 @@
 import fs from "fs";
 import path from "path";
-import { SERVER_ROOT } from "./paths.js";
+import { REPO_ROOT } from "./paths.js";
 
-const ROOT_PKG = path.resolve(SERVER_ROOT, "../../package.json");
+const ROOT_PKG = path.join(REPO_ROOT, "package.json");
 
 /**
  * JerapahFlow app version from the monorepo root package.json.

packages/server/package.json

@@ -13,8 +13,8 @@
     "start:control": "node control.js",
     "start:web": "node web-server.js",
     "migrate": "node -e \"import('./db.js').then((m) => m.migrate().then(() => process.exit(0)))\"",
-    "test:plugins": "JFLOW_PLUGINS_DIR=./data/plugins-smoke-test JFLOW_DB_PATH=./data/plugins-smoke.db node test/plugins-smoke.js",
+    "test:plugins": "JFLOW_DATA_DIR=./data JFLOW_PLUGINS_DIR=./data/plugins-smoke-test JFLOW_DB_PATH=./data/plugins-smoke.db node test/plugins-smoke.js",
-    "test:workflow-history": "JFLOW_WORKFLOWS_DIR=./data/workflow-history-smoke JFLOW_DB_PATH=./data/workflow-history-smoke.db node test/workflow-history-smoke.js",
+    "test:workflow-history": "JFLOW_DATA_DIR=./data JFLOW_WORKFLOWS_DIR=./data/workflow-history-smoke JFLOW_DB_PATH=./data/workflow-history-smoke.db node test/workflow-history-smoke.js",
     "reset-admin": "node reset-admin.js",
     "test:profiles": "node test/profiles-smoke.js",
     "test:set-dry-run": "node test/set-dry-run-smoke.js",

packages/server/paths.js

@@ -1,25 +1,49 @@
+import fs from "fs";
 import path from "path";
 import { fileURLToPath } from "url";
 
 export const SERVER_ROOT = path.dirname(fileURLToPath(import.meta.url));
+export const REPO_ROOT = path.resolve(SERVER_ROOT, "../..");
 export const SCRIPTS_DIR = path.join(SERVER_ROOT, "scripts");
-export const DATA_DIR = path.join(SERVER_ROOT, "data");
+
+/** Prefer `preferred` unless only `legacy` already has files. */
+function existingDir(preferred, legacy, probe) {
+  const has = (dir) =>
+    probe ? probe(dir) : fs.existsSync(dir);
+  if (has(preferred) || !has(legacy)) return preferred;
+  return legacy;
+}
+
+function hasInstanceData(dir) {
+  return (
+    fs.existsSync(path.join(dir, "jerapah-flow.db")) ||
+    fs.existsSync(path.join(dir, "workflows"))
+  );
+}
+
+/** Instance data (SQLite, live workflows, control-state). Not product source. */
+export const DATA_DIR = path.resolve(
+  process.env.JFLOW_DATA_DIR ??
+    existingDir(
+      path.join(REPO_ROOT, "data"),
+      path.join(SERVER_ROOT, "data"),
+      hasInstanceData,
+    ),
+);
 /** Live instance workflows (not shipped in git). Override for tests. */
-export const WORKFLOWS_DIR =
+export const WORKFLOWS_DIR = path.resolve(
-  process.env.JFLOW_WORKFLOWS_DIR ?? path.join(DATA_DIR, "workflows");
+  process.env.JFLOW_WORKFLOWS_DIR ?? path.join(DATA_DIR, "workflows"),
+);
 /** User plugins (repo-root /plugins, outside the pnpm workspace). */
-export const PLUGINS_DIR =
+export const PLUGINS_DIR = path.resolve(
-  process.env.JFLOW_PLUGINS_DIR ??
+  process.env.JFLOW_PLUGINS_DIR ?? path.join(REPO_ROOT, "plugins"),
-  path.resolve(SERVER_ROOT, "../../plugins");
+);
 /** Example plugin sources shipped with the repo. */
-export const EXAMPLE_PLUGINS_DIR = path.resolve(
+export const EXAMPLE_PLUGINS_DIR = path.join(REPO_ROOT, "examples/plugins");
-  SERVER_ROOT,
-  "../../examples/plugins",
-);
 /** Example workflow YAML presets (not loaded by the runner). */
-export const EXAMPLE_WORKFLOWS_DIR = path.resolve(
+export const EXAMPLE_WORKFLOWS_DIR = path.join(REPO_ROOT, "examples/workflows");
-  SERVER_ROOT,
+export const LOGS_DIR = path.resolve(
-  "../../examples/workflows",
+  process.env.JFLOW_LOGS_DIR ??
+    existingDir(path.join(REPO_ROOT, "logs"), path.join(SERVER_ROOT, "logs")),
 );
-export const LOGS_DIR = path.join(SERVER_ROOT, "logs");
+export const WEB_DIST = path.join(REPO_ROOT, "packages/web/dist");
-export const WEB_DIST = path.resolve(SERVER_ROOT, "../web/dist");

packages/server/pm2-bridge.js

@@ -1,8 +1,6 @@
 import path from "path";
 import pm2 from "pm2";
-import { SERVER_ROOT } from "./paths.js";
+import { REPO_ROOT, SERVER_ROOT } from "./paths.js";
-
-const REPO_ROOT = path.resolve(SERVER_ROOT, "../..");
 
 export const PM2_HTTP_NAME = "jflow-http";
 export const PM2_WORKER_NAME = "jflow-worker";

packages/server/test/plugins-smoke.js

@@ -2,6 +2,7 @@
  * Smoke: core vs plugin scripts, fork, example install, resolve, run.
  *
  * Run:
+ *   JFLOW_DATA_DIR=packages/server/data \
  *   JFLOW_PLUGINS_DIR=packages/server/data/plugins-smoke-test \
  *   JFLOW_DB_PATH=packages/server/data/plugins-smoke.db \
  *   node packages/server/test/plugins-smoke.js