feat(ops): restart individual PM2 children and enrich manage UI
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Cursor
@@ -0,0 +1,43 @@
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
function loadEnv(file) {
|
||||
const out = {};
|
||||
if (!fs.existsSync(file)) return out;
|
||||
for (const line of fs.readFileSync(file, "utf8").split("\n")) {
|
||||
const t = line.trim();
|
||||
if (!t || t.startsWith("#")) continue;
|
||||
const i = t.indexOf("=");
|
||||
if (i === -1) continue;
|
||||
const key = t.slice(0, i).trim();
|
||||
let val = t.slice(i + 1).trim();
|
||||
if (
|
||||
(val.startsWith('"') && val.endsWith('"')) ||
|
||||
(val.startsWith("'") && val.endsWith("'"))
|
||||
) {
|
||||
val = val.slice(1, -1);
|
||||
}
|
||||
out[key] = val;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const root = __dirname;
|
||||
|
||||
module.exports = {
|
||||
apps: [
|
||||
{
|
||||
name: "jerapah-flow",
|
||||
cwd: root,
|
||||
script: "packages/server/runner.js",
|
||||
interpreter: "node",
|
||||
instances: 1,
|
||||
autorestart: true,
|
||||
max_restarts: 20,
|
||||
env: {
|
||||
NODE_ENV: "production",
|
||||
...loadEnv(path.join(root, ".env")),
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -35,6 +35,7 @@ import {
|
||||
PM2_HTTP_NAME,
|
||||
PM2_WORKER_NAME,
|
||||
recreateChildren,
|
||||
restartPm2Process,
|
||||
stopPm2App,
|
||||
} from "./pm2-bridge.js";
|
||||
|
||||
@@ -149,6 +150,43 @@ async function waitUntilIdle(timeoutMs, lockToken) {
|
||||
return { ok: active === 0, active };
|
||||
}
|
||||
|
||||
/** @param {unknown} raw */
|
||||
function parsePmId(raw) {
|
||||
if (raw == null || raw === "") return null;
|
||||
const id = Math.floor(Number(raw));
|
||||
if (!Number.isFinite(id) || id < 0) return Number.NaN;
|
||||
return id;
|
||||
}
|
||||
|
||||
async function handleProcessRestart(pmId, reply) {
|
||||
const lock = tryAcquireOpsLock(`process-restart:${process.pid}`);
|
||||
if (!lock.ok) {
|
||||
return reply.code(409).send({ error: lock.error, holder: lock.holder });
|
||||
}
|
||||
|
||||
try {
|
||||
const restarted = await restartPm2Process(pmId);
|
||||
return reply.send({ ok: true, ...restarted, status: await buildStatus() });
|
||||
} catch (err) {
|
||||
const code = err && typeof err === "object" ? err.code : undefined;
|
||||
if (code === "BAD_REQUEST") {
|
||||
return reply.code(400).send({ error: "pmId is required" });
|
||||
}
|
||||
if (code === "NOT_FOUND") {
|
||||
return reply.code(404).send({ error: "process not found" });
|
||||
}
|
||||
if (code === "FORBIDDEN") {
|
||||
return reply.code(403).send({ error: "process is not a JerapahFlow child" });
|
||||
}
|
||||
log.error({ err, pmId }, "process restart failed");
|
||||
return reply.code(500).send({
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
} finally {
|
||||
releaseOpsLock(lock.token);
|
||||
}
|
||||
}
|
||||
|
||||
async function buildStatus() {
|
||||
const state = readControlState();
|
||||
const children = await describeChildren();
|
||||
@@ -258,9 +296,18 @@ server.post(
|
||||
"/ops/restart",
|
||||
{ onRequest: [server.authenticate, server.requireAdmin] },
|
||||
async (req, reply) => {
|
||||
const body = /** @type {{ force?: boolean, timeoutMs?: number }} */ (
|
||||
const body = /** @type {{ force?: boolean, timeoutMs?: number, pmId?: number }} */ (
|
||||
req.body ?? {}
|
||||
);
|
||||
|
||||
const pmId = parsePmId(body.pmId);
|
||||
if (pmId != null) {
|
||||
if (Number.isNaN(pmId)) {
|
||||
return reply.code(400).send({ error: "pmId is required" });
|
||||
}
|
||||
return handleProcessRestart(pmId, reply);
|
||||
}
|
||||
|
||||
const force = Boolean(body.force);
|
||||
const timeoutMs = Math.min(
|
||||
Math.max(Number(body.timeoutMs) || DRAIN_DEFAULT_MS, 1_000),
|
||||
@@ -342,6 +389,19 @@ server.post(
|
||||
},
|
||||
);
|
||||
|
||||
server.post(
|
||||
"/ops/process/restart",
|
||||
{ onRequest: [server.authenticate, server.requireAdmin] },
|
||||
async (req, reply) => {
|
||||
const body = /** @type {{ pmId?: number }} */ (req.body ?? {});
|
||||
const pmId = parsePmId(body.pmId);
|
||||
if (pmId == null || Number.isNaN(pmId)) {
|
||||
return reply.code(400).send({ error: "pmId is required" });
|
||||
}
|
||||
return handleProcessRestart(pmId, reply);
|
||||
},
|
||||
);
|
||||
|
||||
server.post(
|
||||
"/ops/scale",
|
||||
{ onRequest: [server.authenticate, server.requireAdmin] },
|
||||
|
||||
@@ -130,6 +130,40 @@ export function restartPm2App(name) {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Restart a single PM2 process by id. Only jflow-http / jflow-worker.
|
||||
* @param {number} pmId
|
||||
* @returns {Promise<{ name: string, pmId: number }>}
|
||||
*/
|
||||
export async function restartPm2Process(pmId) {
|
||||
const id = Math.floor(Number(pmId));
|
||||
if (!Number.isFinite(id) || id < 0) {
|
||||
const err = new Error("invalid pmId");
|
||||
err.code = "BAD_REQUEST";
|
||||
throw err;
|
||||
}
|
||||
|
||||
const list = await listPm2();
|
||||
const proc = list.find((p) => Number(p.pm_id) === id);
|
||||
if (!proc) {
|
||||
const err = new Error("process not found");
|
||||
err.code = "NOT_FOUND";
|
||||
throw err;
|
||||
}
|
||||
if (proc.name !== PM2_HTTP_NAME && proc.name !== PM2_WORKER_NAME) {
|
||||
const err = new Error("process is not a JerapahFlow child");
|
||||
err.code = "FORBIDDEN";
|
||||
throw err;
|
||||
}
|
||||
await new Promise((resolve, reject) => {
|
||||
pm2.restart(id, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
return { name: proc.name, pmId: id };
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared env for child processes.
|
||||
* @param {{ generation: number }} opts
|
||||
@@ -251,19 +285,29 @@ export async function describeChildren() {
|
||||
|
||||
const mapOne = (p) => ({
|
||||
name: p.name,
|
||||
pmId: p.pm_id,
|
||||
pmId: Number(p.pm_id),
|
||||
status: p.pm2_env?.status ?? "unknown",
|
||||
pid: p.pid ?? null,
|
||||
restarts: p.pm2_env?.restart_time ?? 0,
|
||||
uptime: p.pm2_env?.pm_uptime ?? null,
|
||||
generation: Number(p.pm2_env?.JFLOW_CONFIG_GENERATION ?? 0) || null,
|
||||
memory: Number(p.monit?.memory) || 0,
|
||||
cpu: Number(p.monit?.cpu) || 0,
|
||||
});
|
||||
|
||||
const httpMapped = http.map(mapOne);
|
||||
const workersMapped = workers.map(mapOne);
|
||||
const all = [...httpMapped, ...workersMapped];
|
||||
|
||||
return {
|
||||
http: http.map(mapOne),
|
||||
workers: workers.map(mapOne),
|
||||
http: httpMapped,
|
||||
workers: workersMapped,
|
||||
httpOnline: http.some((p) => p.pm2_env?.status === "online"),
|
||||
workerOnlineCount: workers.filter((p) => p.pm2_env?.status === "online").length,
|
||||
totals: {
|
||||
memory: all.reduce((sum, p) => sum + p.memory, 0),
|
||||
cpu: all.reduce((sum, p) => sum + p.cpu, 0),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -126,7 +126,11 @@ export async function startApp(opts = {}) {
|
||||
}
|
||||
});
|
||||
|
||||
const registry = createRegistry(server, { queue: workflowQueue });
|
||||
const registry = createRegistry(server, {
|
||||
queue: workflowQueue,
|
||||
// Cron + HTTP triggers enqueue jobs; only the API process may own them.
|
||||
enableTriggers: runApi,
|
||||
});
|
||||
registry.registerWorkflows();
|
||||
if (runApi) {
|
||||
registry.registerHttpTriggers();
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
import { useState } from "react";
|
||||
import { errorMessage } from "../api/client.js";
|
||||
import { HeartbeatsToolbar } from "./HeartbeatsToolbar.jsx";
|
||||
import { useNotifications } from "../notifications.jsx";
|
||||
import { formatBytes, formatCpu, formatDuration } from "../lib/format.js";
|
||||
|
||||
const MAX_WORKERS = 32;
|
||||
|
||||
export function HeartbeatsAndPm2Card({
|
||||
data,
|
||||
children,
|
||||
restart,
|
||||
processRestart,
|
||||
workerCount,
|
||||
desiredWorkers,
|
||||
onDecreaseWorkers,
|
||||
onIncreaseWorkers,
|
||||
onScale,
|
||||
scalePending = false,
|
||||
onReload,
|
||||
reloadPending = false,
|
||||
}) {
|
||||
const { notify } = useNotifications();
|
||||
const [forceRestart, setForceRestart] = useState(false);
|
||||
const [pendingPmId, setPendingPmId] = useState(null);
|
||||
|
||||
async function onRestartAll() {
|
||||
const label = forceRestart ? "Force restart" : "Restart";
|
||||
if (
|
||||
forceRestart &&
|
||||
!window.confirm("Force restart will interrupt active jobs. Continue?")
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await restart.mutateAsync({ force: forceRestart });
|
||||
notify.success(`${label} ok`);
|
||||
} catch (e) {
|
||||
notify.error(errorMessage(e, `${label} failed`));
|
||||
}
|
||||
}
|
||||
|
||||
async function onProcessRestart(pmId) {
|
||||
const id = Number(pmId);
|
||||
setPendingPmId(id);
|
||||
try {
|
||||
await processRestart.mutateAsync({ pmId: id });
|
||||
notify.success("Process restart ok");
|
||||
} catch (e) {
|
||||
notify.error(errorMessage(e, "Process restart failed"));
|
||||
} finally {
|
||||
setPendingPmId(null);
|
||||
}
|
||||
}
|
||||
|
||||
const heartbeats = data?.heartbeats ?? [];
|
||||
const httpChildren = children?.http ?? [];
|
||||
const workerChildren = children?.workers ?? [];
|
||||
const pm2Children = [...httpChildren, ...workerChildren];
|
||||
const heartbeatByPid = new Map();
|
||||
for (const h of heartbeats) {
|
||||
if (!heartbeatByPid.has(h.pid)) heartbeatByPid.set(h.pid, h);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="card bg-base-100 border border-base-300 shadow-sm">
|
||||
<div className="card-body gap-4">
|
||||
<h2 className="card-title text-lg">Heartbeats</h2>
|
||||
|
||||
<HeartbeatsToolbar
|
||||
forceRestart={forceRestart}
|
||||
onForceRestartChange={setForceRestart}
|
||||
onRestartAll={onRestartAll}
|
||||
restartPending={restart.isPending}
|
||||
workerCount={workerCount}
|
||||
desiredWorkers={desiredWorkers}
|
||||
onDecreaseWorkers={onDecreaseWorkers}
|
||||
onIncreaseWorkers={onIncreaseWorkers}
|
||||
onScale={onScale}
|
||||
scalePending={scalePending}
|
||||
onReload={onReload}
|
||||
reloadPending={reloadPending}
|
||||
maxWorkers={MAX_WORKERS}
|
||||
/>
|
||||
|
||||
<div className="overflow-x-auto">
|
||||
<table className="table table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Role</th>
|
||||
<th>Name</th>
|
||||
<th>PM ID</th>
|
||||
<th>Status</th>
|
||||
<th>PID</th>
|
||||
<th>Generation</th>
|
||||
<th>Uptime</th>
|
||||
<th>Memory</th>
|
||||
<th>CPU</th>
|
||||
<th>Restarts</th>
|
||||
<th />
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{pm2Children.map((p) => {
|
||||
const pid = p.pid ?? null;
|
||||
const hb = pid != null ? heartbeatByPid.get(pid) : undefined;
|
||||
const role =
|
||||
hb?.role ??
|
||||
(p.name === "jflow-http"
|
||||
? "http"
|
||||
: p.name === "jflow-worker"
|
||||
? "worker"
|
||||
: p.name ?? "unknown");
|
||||
const generation = hb?.generation ?? p.generation ?? "—";
|
||||
const uptimeMs =
|
||||
p.uptime != null
|
||||
? Math.max(0, Date.now() - Number(p.uptime))
|
||||
: hb
|
||||
? Math.max(0, Date.now() - hb.ts)
|
||||
: null;
|
||||
|
||||
return (
|
||||
<tr key={p.pmId}>
|
||||
<td>{role}</td>
|
||||
<td className="font-mono">{p.name}</td>
|
||||
<td className="font-mono">{p.pmId}</td>
|
||||
<td>{p.status}</td>
|
||||
<td className="font-mono">{pid ?? "—"}</td>
|
||||
<td className="font-mono">{generation}</td>
|
||||
<td className="font-mono" title={uptimeMs != null ? `${uptimeMs}ms` : undefined}>
|
||||
{formatDuration(uptimeMs)}
|
||||
</td>
|
||||
<td className="font-mono">{formatBytes(p.memory)}</td>
|
||||
<td className="font-mono">{formatCpu(p.cpu)}</td>
|
||||
<td className="font-mono">{p.restarts ?? 0}</td>
|
||||
<td className="text-end">
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-xs"
|
||||
disabled={
|
||||
processRestart.isPending && pendingPmId === p.pmId
|
||||
}
|
||||
onClick={() => onProcessRestart(p.pmId)}
|
||||
>
|
||||
Restart
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
{!pm2Children.length ? (
|
||||
<tr>
|
||||
<td colSpan={11} className="opacity-60">
|
||||
No PM2 children
|
||||
</td>
|
||||
</tr>
|
||||
) : null}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import { LuMinus, LuPlus } from "react-icons/lu";
|
||||
|
||||
export function HeartbeatsToolbar({
|
||||
forceRestart,
|
||||
onForceRestartChange,
|
||||
onRestartAll,
|
||||
restartPending = false,
|
||||
workerCount,
|
||||
desiredWorkers,
|
||||
onDecreaseWorkers,
|
||||
onIncreaseWorkers,
|
||||
onScale,
|
||||
scalePending = false,
|
||||
onReload,
|
||||
reloadPending = false,
|
||||
minWorkers = 0,
|
||||
maxWorkers = 32,
|
||||
}) {
|
||||
return (
|
||||
<div className="flex flex-col gap-2">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<label className="label cursor-pointer gap-2 px-0">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="checkbox checkbox-sm"
|
||||
checked={forceRestart}
|
||||
onChange={(e) => onForceRestartChange(e.target.checked)}
|
||||
/>
|
||||
<span className="label-text">Force</span>
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
className={`btn btn-sm ${forceRestart ? "btn-error" : "btn-primary"}`}
|
||||
disabled={restartPending}
|
||||
onClick={onRestartAll}
|
||||
>
|
||||
Restart all
|
||||
</button>
|
||||
|
||||
<div className="divider divider-horizontal mx-1" />
|
||||
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-circle btn-sm"
|
||||
aria-label="Decrease workers"
|
||||
disabled={workerCount <= minWorkers}
|
||||
onClick={onDecreaseWorkers}
|
||||
>
|
||||
<LuMinus className="size-4" />
|
||||
</button>
|
||||
<span className="min-w-20 text-center text-sm tabular-nums">
|
||||
{workerCount} worker{workerCount === 1 ? "" : "s"}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-circle btn-sm"
|
||||
aria-label="Increase workers"
|
||||
disabled={workerCount >= maxWorkers}
|
||||
onClick={onIncreaseWorkers}
|
||||
>
|
||||
<LuPlus className="size-4" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm"
|
||||
disabled={scalePending || workerCount === desiredWorkers}
|
||||
onClick={onScale}
|
||||
>
|
||||
Scale
|
||||
</button>
|
||||
|
||||
<div className="divider divider-horizontal mx-1" />
|
||||
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm"
|
||||
disabled={reloadPending}
|
||||
onClick={onReload}
|
||||
>
|
||||
Reload workflows
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="divider my-0" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { LuCircleCheck, LuCircleX } from "react-icons/lu";
|
||||
|
||||
export function HttpProcessCard({
|
||||
httpOnline,
|
||||
onStart,
|
||||
onStop,
|
||||
startPending = false,
|
||||
stopPending = false,
|
||||
}) {
|
||||
return (
|
||||
<div className="card bg-base-100 border border-base-300 shadow-sm">
|
||||
<div className="card-body gap-4">
|
||||
<h2 className="card-title text-lg">HTTP process</h2>
|
||||
<div className="flex items-center gap-2">
|
||||
{httpOnline ? (
|
||||
<LuCircleCheck className="size-6 text-success" aria-hidden />
|
||||
) : (
|
||||
<LuCircleX className="size-6 text-error" aria-hidden />
|
||||
)}
|
||||
<span>{httpOnline ? "Running" : "Stopped"}</span>
|
||||
</div>
|
||||
<div className="card-actions justify-start px-0">
|
||||
{httpOnline ? (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-warning"
|
||||
disabled={stopPending}
|
||||
onClick={onStop}
|
||||
>
|
||||
Stop HTTP
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-primary"
|
||||
disabled={startPending}
|
||||
onClick={onStart}
|
||||
>
|
||||
Start HTTP
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { formatBytes, formatCpu } from "../lib/format.js";
|
||||
|
||||
export function ProcessResourcesCard({ children }) {
|
||||
const totals = children?.totals ?? { memory: 0, cpu: 0 };
|
||||
|
||||
return (
|
||||
<div className="card bg-base-100 border border-base-300 shadow-sm">
|
||||
<div className="card-body gap-4">
|
||||
<h2 className="card-title text-lg">Resources</h2>
|
||||
<div className="space-y-1 text-sm tabular-nums">
|
||||
<p>{formatBytes(totals.memory)} memory total</p>
|
||||
<p>{formatCpu(totals.cpu)} CPU total</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { LuCircleCheck, LuCirclePause } from "react-icons/lu";
|
||||
|
||||
export function QueueCard({
|
||||
paused = false,
|
||||
workersOnline = 0,
|
||||
queuedJobs = 0,
|
||||
activeJobs = 0,
|
||||
onPause,
|
||||
onResume,
|
||||
pausePending = false,
|
||||
resumePending = false,
|
||||
}) {
|
||||
return (
|
||||
<div className="card bg-base-100 border border-base-300 shadow-sm">
|
||||
<div className="card-body gap-4">
|
||||
<h2 className="card-title text-lg">Queue</h2>
|
||||
<div className="flex items-center gap-2">
|
||||
{paused ? (
|
||||
<LuCirclePause className="size-6 text-warning" aria-hidden />
|
||||
) : (
|
||||
<LuCircleCheck className="size-6 text-success" aria-hidden />
|
||||
)}
|
||||
<span>
|
||||
{paused
|
||||
? "Paused"
|
||||
: `Running (${workersOnline} worker${workersOnline === 1 ? "" : "s"})`}
|
||||
</span>
|
||||
</div>
|
||||
<div className="space-y-1 text-sm tabular-nums">
|
||||
<p>
|
||||
{activeJobs} running job{activeJobs === 1 ? "" : "s"}
|
||||
</p>
|
||||
<p>
|
||||
{queuedJobs} queued job{queuedJobs === 1 ? "" : "s"}
|
||||
</p>
|
||||
</div>
|
||||
<div className="card-actions justify-start px-0">
|
||||
{paused ? (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-primary"
|
||||
disabled={resumePending}
|
||||
onClick={onResume}
|
||||
>
|
||||
Resume workers
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-warning"
|
||||
disabled={pausePending}
|
||||
onClick={onPause}
|
||||
>
|
||||
Pause workers
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -3,12 +3,28 @@
|
||||
themes: light --default, dark --prefersdark;
|
||||
}
|
||||
|
||||
@property --value {
|
||||
syntax: "<number>";
|
||||
inherits: true;
|
||||
initial-value: 0;
|
||||
}
|
||||
|
||||
html,
|
||||
body,
|
||||
#root {
|
||||
min-height: 100%;
|
||||
}
|
||||
|
||||
.radial-progress {
|
||||
transition:
|
||||
--value 0.7s ease-out,
|
||||
--radialprogress 0.7s ease-out;
|
||||
}
|
||||
|
||||
.radial-progress::after {
|
||||
transition: transform 0.7s ease-out;
|
||||
}
|
||||
|
||||
.mermaid-clickable [id^="s"] {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
export function formatBytes(bytes) {
|
||||
const n = Number(bytes);
|
||||
if (!Number.isFinite(n)) return "—";
|
||||
if (n === 0) return "0 KB";
|
||||
if (n < 1024 * 1024) return `${Math.round(n / 1024)} KB`;
|
||||
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
export function formatCpu(cpu) {
|
||||
const n = Number(cpu);
|
||||
if (!Number.isFinite(n)) return "—";
|
||||
return `${n.toFixed(1)}%`;
|
||||
}
|
||||
|
||||
/** Compact duration from milliseconds (e.g. `850ms`, `12s`, `5m 3s`, `2h 15m`, `3d 4h`). */
|
||||
export function formatDuration(ms) {
|
||||
const n = Number(ms);
|
||||
if (!Number.isFinite(n) || n < 0) return "—";
|
||||
if (n < 1000) return `${Math.round(n)}ms`;
|
||||
const sec = Math.floor(n / 1000);
|
||||
if (sec < 60) return `${sec}s`;
|
||||
const min = Math.floor(sec / 60);
|
||||
const s = sec % 60;
|
||||
if (min < 60) return s ? `${min}m ${s}s` : `${min}m`;
|
||||
const hr = Math.floor(min / 60);
|
||||
const m = min % 60;
|
||||
if (hr < 48) return m ? `${hr}h ${m}m` : `${hr}h`;
|
||||
const days = Math.floor(hr / 24);
|
||||
const h = hr % 24;
|
||||
return h ? `${days}d ${h}h` : `${days}d`;
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
import { useState } from "react";
|
||||
import { useEffect, useState } from "react";
|
||||
import {
|
||||
useOpsBumpGeneration,
|
||||
useOpsHttpStart,
|
||||
useOpsHttpStop,
|
||||
useOpsPause,
|
||||
useOpsProcessRestart,
|
||||
useOpsReload,
|
||||
useOpsRestart,
|
||||
useOpsResume,
|
||||
@@ -11,11 +11,20 @@ import {
|
||||
useOpsStatus,
|
||||
} from "../api/hooks.js";
|
||||
import { errorMessage } from "../api/client.js";
|
||||
import { HttpProcessCard } from "../components/HttpProcessCard.jsx";
|
||||
import { QueueCard } from "../components/QueueCard.jsx";
|
||||
import { ProcessResourcesCard } from "../components/ProcessResourcesCard.jsx";
|
||||
import { HeartbeatsAndPm2Card } from "../components/HeartbeatsAndPm2Card.jsx";
|
||||
import { useNotifications } from "../notifications.jsx";
|
||||
|
||||
const MAX_WORKERS = 32;
|
||||
|
||||
function Stat({ label, value }) {
|
||||
return (
|
||||
<div className="flex flex-col gap-0.5">
|
||||
<span className="text-xs uppercase tracking-wide opacity-60">{label}</span>
|
||||
<span className="text-xs uppercase tracking-wide opacity-60">
|
||||
{label}
|
||||
</span>
|
||||
<span className="font-mono text-sm">{value}</span>
|
||||
</div>
|
||||
);
|
||||
@@ -30,10 +39,10 @@ export function OpsPage() {
|
||||
const scale = useOpsScale();
|
||||
const httpStart = useOpsHttpStart();
|
||||
const httpStop = useOpsHttpStop();
|
||||
const bump = useOpsBumpGeneration();
|
||||
const [workerCount, setWorkerCount] = useState("");
|
||||
const [msg, setMsg] = useState(null);
|
||||
const [err, setErr] = useState(null);
|
||||
const processRestart = useOpsProcessRestart();
|
||||
const { notify } = useNotifications();
|
||||
const [workerCount, setWorkerCount] = useState(1);
|
||||
const [workerInit, setWorkerInit] = useState(false);
|
||||
|
||||
const unavailable =
|
||||
status.isError &&
|
||||
@@ -41,15 +50,34 @@ export function OpsPage() {
|
||||
status.error?.response?.status === 404 ||
|
||||
status.error?.message?.includes("Network"));
|
||||
|
||||
const data = status.data;
|
||||
const desired = data?.desired;
|
||||
const queue = data?.queue;
|
||||
const children = data?.children;
|
||||
const httpOnline = Boolean(children?.httpOnline);
|
||||
|
||||
useEffect(() => {
|
||||
if (!workerInit && desired?.workers != null) {
|
||||
setWorkerCount(desired.workers);
|
||||
setWorkerInit(true);
|
||||
}
|
||||
}, [desired?.workers, workerInit]);
|
||||
|
||||
function run(label, mutateAsync, args) {
|
||||
setMsg(null);
|
||||
setErr(null);
|
||||
mutateAsync(args)
|
||||
.then((data) => {
|
||||
setMsg(`${label} ok`);
|
||||
return data;
|
||||
const call = args === undefined ? mutateAsync() : mutateAsync(args);
|
||||
call
|
||||
.then((result) => {
|
||||
notify.success(`${label} ok`);
|
||||
return result;
|
||||
})
|
||||
.catch((e) => setErr(errorMessage(e, `${label} failed`)));
|
||||
.catch((e) => notify.error(errorMessage(e, `${label} failed`)));
|
||||
}
|
||||
|
||||
function scaleWorkers() {
|
||||
run("Scale", scale.mutateAsync, {
|
||||
workers: workerCount,
|
||||
force: false,
|
||||
});
|
||||
}
|
||||
|
||||
if (unavailable) {
|
||||
@@ -68,11 +96,6 @@ export function OpsPage() {
|
||||
);
|
||||
}
|
||||
|
||||
const data = status.data;
|
||||
const desired = data?.desired;
|
||||
const queue = data?.queue;
|
||||
const children = data?.children;
|
||||
|
||||
return (
|
||||
<div className="mx-auto space-y-6">
|
||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||
@@ -98,226 +121,50 @@ export function OpsPage() {
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{msg ? (
|
||||
<div className="alert alert-success text-sm">
|
||||
<span>{msg}</span>
|
||||
</div>
|
||||
) : null}
|
||||
{err ? (
|
||||
<div className="alert alert-error text-sm">
|
||||
<span>{err}</span>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{status.isLoading && !data ? (
|
||||
<span className="loading loading-spinner" />
|
||||
) : (
|
||||
<>
|
||||
<section className="grid gap-4 sm:grid-cols-2 lg:grid-cols-4">
|
||||
<Stat label="HTTP desired" value={desired?.http ?? "—"} />
|
||||
<Stat label="Workers desired" value={desired?.workers ?? "—"} />
|
||||
<Stat label="Generation" value={desired?.generation ?? "—"} />
|
||||
<Stat
|
||||
label="Queue"
|
||||
value={queue?.paused ? "paused" : "running"}
|
||||
<div className="grid gap-6 md:grid-cols-3">
|
||||
<HttpProcessCard
|
||||
httpOnline={httpOnline}
|
||||
startPending={httpStart.isPending}
|
||||
stopPending={httpStop.isPending}
|
||||
onStart={() => run("HTTP start", httpStart.mutateAsync)}
|
||||
onStop={() => run("HTTP stop", httpStop.mutateAsync)}
|
||||
/>
|
||||
<Stat
|
||||
label="Active jobs"
|
||||
value={queue?.counts?.active ?? "—"}
|
||||
/>
|
||||
<Stat
|
||||
label="Waiting jobs"
|
||||
value={queue?.counts?.waiting ?? "—"}
|
||||
/>
|
||||
<Stat
|
||||
label="HTTP online"
|
||||
value={children?.httpOnline ? "yes" : "no"}
|
||||
/>
|
||||
<Stat
|
||||
label="Workers online"
|
||||
value={children?.workerOnlineCount ?? "—"}
|
||||
/>
|
||||
</section>
|
||||
|
||||
<section className="space-y-2">
|
||||
<h2 className="text-lg font-medium">Queue</h2>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<button
|
||||
type="button"
|
||||
className={`btn btn-sm ${queue?.paused ? "btn-disabled" : ""}`}
|
||||
disabled={pause.isPending || Boolean(queue?.paused)}
|
||||
onClick={() => run("Pause", pause.mutateAsync)}
|
||||
>
|
||||
Pause workers
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`btn btn-sm ${!queue?.paused ? "btn-disabled" : ""}`}
|
||||
disabled={resume.isPending || !queue?.paused}
|
||||
onClick={() => run("Resume", resume.mutateAsync)}
|
||||
>
|
||||
Resume workers
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm"
|
||||
disabled={reload.isPending}
|
||||
onClick={() => run("Reload", reload.mutateAsync)}
|
||||
>
|
||||
Reload workflows
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="space-y-2">
|
||||
<h2 className="text-lg font-medium">HTTP process</h2>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-primary"
|
||||
disabled={httpStart.isPending}
|
||||
onClick={() => run("HTTP start", httpStart.mutateAsync)}
|
||||
>
|
||||
Start HTTP
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-warning"
|
||||
disabled={httpStop.isPending}
|
||||
onClick={() => run("HTTP stop", httpStop.mutateAsync)}
|
||||
>
|
||||
Stop HTTP
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="space-y-2">
|
||||
<h2 className="text-lg font-medium">Workers</h2>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<input
|
||||
type="number"
|
||||
min={0}
|
||||
max={32}
|
||||
className="input input-bordered input-sm w-24"
|
||||
placeholder={String(desired?.workers ?? 1)}
|
||||
value={workerCount}
|
||||
onChange={(e) => setWorkerCount(e.target.value)}
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm"
|
||||
disabled={scale.isPending || workerCount === ""}
|
||||
onClick={() =>
|
||||
run("Scale", scale.mutateAsync, {
|
||||
workers: Number(workerCount),
|
||||
force: false,
|
||||
})
|
||||
<QueueCard
|
||||
paused={Boolean(queue?.paused)}
|
||||
workersOnline={Number(children?.workerOnlineCount) || 0}
|
||||
activeJobs={Number(queue?.counts?.active) || 0}
|
||||
queuedJobs={
|
||||
(Number(queue?.counts?.waiting) || 0) +
|
||||
(Number(queue?.counts?.delayed) || 0)
|
||||
}
|
||||
>
|
||||
Scale (drain)
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-warning"
|
||||
disabled={scale.isPending || workerCount === ""}
|
||||
onClick={() =>
|
||||
run("Scale force", scale.mutateAsync, {
|
||||
workers: Number(workerCount),
|
||||
force: true,
|
||||
})
|
||||
}
|
||||
>
|
||||
Scale (force)
|
||||
</button>
|
||||
pausePending={pause.isPending}
|
||||
resumePending={resume.isPending}
|
||||
onPause={() => run("Pause", pause.mutateAsync)}
|
||||
onResume={() => run("Resume", resume.mutateAsync)}
|
||||
/>
|
||||
<ProcessResourcesCard children={children} />
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="space-y-2">
|
||||
<h2 className="text-lg font-medium">Restart HTTP + workers</h2>
|
||||
<p className="text-sm opacity-70">
|
||||
Pauses the queue, waits until active jobs are 0, migrates DB, then
|
||||
recreates PM2 processes. Force skips the wait (may interrupt runs).
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-primary"
|
||||
disabled={restart.isPending}
|
||||
onClick={() => run("Restart", restart.mutateAsync, { force: false })}
|
||||
>
|
||||
Drain restart
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-error"
|
||||
disabled={restart.isPending}
|
||||
onClick={() => {
|
||||
if (
|
||||
!window.confirm(
|
||||
"Force restart will interrupt active jobs. Continue?",
|
||||
)
|
||||
) {
|
||||
return;
|
||||
<HeartbeatsAndPm2Card
|
||||
data={data}
|
||||
children={children}
|
||||
restart={restart}
|
||||
processRestart={processRestart}
|
||||
workerCount={workerCount}
|
||||
desiredWorkers={Number(desired?.workers) || 0}
|
||||
onDecreaseWorkers={() => setWorkerCount((n) => Math.max(0, n - 1))}
|
||||
onIncreaseWorkers={() =>
|
||||
setWorkerCount((n) => Math.min(MAX_WORKERS, n + 1))
|
||||
}
|
||||
run("Force restart", restart.mutateAsync, { force: true });
|
||||
}}
|
||||
>
|
||||
Force restart
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-sm btn-ghost"
|
||||
disabled={bump.isPending}
|
||||
onClick={() =>
|
||||
run("Bump generation", bump.mutateAsync, "manual test flag")
|
||||
}
|
||||
>
|
||||
Flag restart-needed
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="space-y-2">
|
||||
<h2 className="text-lg font-medium">Heartbeats</h2>
|
||||
<div className="overflow-x-auto">
|
||||
<table className="table table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Role</th>
|
||||
<th>PID</th>
|
||||
<th>Generation</th>
|
||||
<th>Age (ms)</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(data?.heartbeats ?? []).map((h) => (
|
||||
<tr key={h.field}>
|
||||
<td>{h.role}</td>
|
||||
<td className="font-mono">{h.pid}</td>
|
||||
<td className="font-mono">{h.generation}</td>
|
||||
<td className="font-mono">
|
||||
{Math.max(0, Date.now() - h.ts)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
{!data?.heartbeats?.length ? (
|
||||
<tr>
|
||||
<td colSpan={4} className="opacity-60">
|
||||
No live heartbeats yet
|
||||
</td>
|
||||
</tr>
|
||||
) : null}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="space-y-2">
|
||||
<h2 className="text-lg font-medium">PM2 children</h2>
|
||||
<pre className="bg-base-200 max-h-64 overflow-auto rounded-lg p-3 text-xs">
|
||||
{JSON.stringify(children ?? {}, null, 2)}
|
||||
</pre>
|
||||
</section>
|
||||
onScale={scaleWorkers}
|
||||
scalePending={scale.isPending}
|
||||
onReload={() => run("Reload", reload.mutateAsync)}
|
||||
reloadPending={reload.isPending}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user