feat(runner): queue workflow runs with BullMQ (phase A)
Enqueue HTTP, cron, and manual runs via Redis/BullMQ; workers execute asynchronously with configurable concurrency. Triggers return 202 and clients poll GET /api/runs/:id for progress (queued → running → done). Co-authored-by: Nasyarobby Putra <nasyarobby@gmail.com>
This commit is contained in:
Cursor Agent
+150
-95
@@ -22,6 +22,12 @@ import httpPagesPlugin from "./src/api/http-pages.js";
|
||||
import httpAuthsPlugin from "./src/api/http-auths.js";
|
||||
import { WEB_DIST } from "./paths.js";
|
||||
import { resolveSecretsKeyMaterial } from "./secrets.js";
|
||||
import {
|
||||
closeRedis,
|
||||
createWorkflowQueue,
|
||||
createWorkflowWorker,
|
||||
getRedisUrl,
|
||||
} from "./workflow-queue.js";
|
||||
|
||||
await migrate();
|
||||
enableLogPersistence();
|
||||
@@ -42,6 +48,20 @@ try {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const role = (process.env.JFLOW_ROLE || "all").toLowerCase();
|
||||
const runApi = role === "all" || role === "api";
|
||||
const runWorker = role === "all" || role === "worker";
|
||||
|
||||
log.info({ redis: getRedisUrl(), role }, "starting jerapah-flow");
|
||||
|
||||
const workflowQueue = createWorkflowQueue();
|
||||
try {
|
||||
await workflowQueue.waitUntilReady();
|
||||
} catch (err) {
|
||||
log.error({ err, redis: getRedisUrl() }, "failed to connect to Redis");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const server = fastify({ loggerInstance: log });
|
||||
|
||||
await server.register(cookie);
|
||||
@@ -71,98 +91,129 @@ server.decorate("requireAdmin", async function requireAdmin(req, reply) {
|
||||
}
|
||||
});
|
||||
|
||||
const registry = createRegistry(server);
|
||||
const registry = createRegistry(server, { queue: workflowQueue });
|
||||
registry.registerWorkflows();
|
||||
registry.registerHttpTriggers();
|
||||
registry.registerCronTriggers();
|
||||
registry.registerPruneJob();
|
||||
if (runApi) {
|
||||
registry.registerHttpTriggers();
|
||||
registry.registerCronTriggers();
|
||||
registry.registerPruneJob();
|
||||
}
|
||||
|
||||
await server.register(
|
||||
async (api) => {
|
||||
api.addHook("onRequest", async (req, reply) => {
|
||||
const raw = (req.url || "").split("?")[0];
|
||||
const stripped = raw.replace(/^\/api/, "") || "/";
|
||||
const routeUrl = req.routeOptions?.url || stripped;
|
||||
const open =
|
||||
OPEN_API_ROUTES.has(`${req.method} ${routeUrl}`) ||
|
||||
OPEN_API_ROUTES.has(`${req.method} ${stripped}`);
|
||||
if (open) return;
|
||||
await server.authenticate(req, reply);
|
||||
});
|
||||
await api.register(authPlugin);
|
||||
await api.register(usersPlugin);
|
||||
await api.register(secretsPlugin);
|
||||
await api.register(variablesPlugin);
|
||||
await api.register(kvPlugin);
|
||||
await api.register(httpPagesPlugin);
|
||||
await api.register(httpAuthsPlugin);
|
||||
await api.register(scriptsPluginFactory(registry));
|
||||
await api.register(workflowsPluginFactory(registry));
|
||||
await api.register(runsPlugin);
|
||||
await api.register(dashboardPluginFactory(registry));
|
||||
},
|
||||
{ prefix: "/api" },
|
||||
);
|
||||
|
||||
server.post(
|
||||
"/admin/workflows/reregister",
|
||||
{ onRequest: [server.authenticate] },
|
||||
async (_req, reply) => {
|
||||
registry.reregister();
|
||||
return reply.send({ message: "Workflows refreshed" });
|
||||
},
|
||||
);
|
||||
|
||||
server.get(
|
||||
"/admin/runs",
|
||||
{ onRequest: [server.authenticate] },
|
||||
async (req, reply) => {
|
||||
const q = /** @type {Record<string, string | undefined>} */ (req.query);
|
||||
const limit = q.limit ? Number(q.limit) : undefined;
|
||||
const runs = await store.listRuns({
|
||||
owner: q.owner,
|
||||
workflow: q.workflow,
|
||||
status: q.status,
|
||||
limit: Number.isFinite(limit) ? limit : undefined,
|
||||
before: q.before,
|
||||
});
|
||||
return reply.send({ runs });
|
||||
},
|
||||
);
|
||||
|
||||
server.get(
|
||||
"/admin/runs/:id",
|
||||
{ onRequest: [server.authenticate] },
|
||||
async (req, reply) => {
|
||||
const { id } = /** @type {{ id: string }} */ (req.params);
|
||||
const run = await store.getRun(id);
|
||||
if (!run) {
|
||||
return reply.code(404).send({ error: "run not found" });
|
||||
/** @type {import("bullmq").Worker | null} */
|
||||
let workflowWorker = null;
|
||||
if (runWorker) {
|
||||
workflowWorker = createWorkflowWorker(async (job) => {
|
||||
const data = /** @type {{ runId?: string, key?: string, depth?: number }} */ (
|
||||
job.data ?? {}
|
||||
);
|
||||
if (typeof data.runId !== "string" || typeof data.key !== "string") {
|
||||
throw new Error("invalid workflow job payload");
|
||||
}
|
||||
return reply.send(run);
|
||||
},
|
||||
);
|
||||
|
||||
if (fs.existsSync(WEB_DIST)) {
|
||||
await server.register(fastifyStatic, {
|
||||
root: WEB_DIST,
|
||||
wildcard: false,
|
||||
});
|
||||
server.setNotFoundHandler((req, reply) => {
|
||||
const url = req.raw.url ?? "";
|
||||
if (
|
||||
url.startsWith("/api") ||
|
||||
url.startsWith("/u/") ||
|
||||
url.startsWith("/admin")
|
||||
) {
|
||||
return reply.code(404).send({ error: "not found" });
|
||||
const result = await registry.executeQueuedRun({
|
||||
runId: data.runId,
|
||||
key: data.key,
|
||||
depth: data.depth ?? 0,
|
||||
});
|
||||
if (result.status === "failed") {
|
||||
throw new Error(result.error || "workflow failed");
|
||||
}
|
||||
return reply.sendFile("index.html");
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
if (runApi) {
|
||||
await server.register(
|
||||
async (api) => {
|
||||
api.addHook("onRequest", async (req, reply) => {
|
||||
const raw = (req.url || "").split("?")[0];
|
||||
const stripped = raw.replace(/^\/api/, "") || "/";
|
||||
const routeUrl = req.routeOptions?.url || stripped;
|
||||
const open =
|
||||
OPEN_API_ROUTES.has(`${req.method} ${routeUrl}`) ||
|
||||
OPEN_API_ROUTES.has(`${req.method} ${stripped}`);
|
||||
if (open) return;
|
||||
await server.authenticate(req, reply);
|
||||
});
|
||||
await api.register(authPlugin);
|
||||
await api.register(usersPlugin);
|
||||
await api.register(secretsPlugin);
|
||||
await api.register(variablesPlugin);
|
||||
await api.register(kvPlugin);
|
||||
await api.register(httpPagesPlugin);
|
||||
await api.register(httpAuthsPlugin);
|
||||
await api.register(scriptsPluginFactory(registry));
|
||||
await api.register(workflowsPluginFactory(registry));
|
||||
await api.register(runsPlugin);
|
||||
await api.register(dashboardPluginFactory(registry));
|
||||
},
|
||||
{ prefix: "/api" },
|
||||
);
|
||||
|
||||
server.post(
|
||||
"/admin/workflows/reregister",
|
||||
{ onRequest: [server.authenticate] },
|
||||
async (_req, reply) => {
|
||||
registry.reregister();
|
||||
return reply.send({ message: "Workflows refreshed" });
|
||||
},
|
||||
);
|
||||
|
||||
server.get(
|
||||
"/admin/runs",
|
||||
{ onRequest: [server.authenticate] },
|
||||
async (req, reply) => {
|
||||
const q = /** @type {Record<string, string | undefined>} */ (req.query);
|
||||
const limit = q.limit ? Number(q.limit) : undefined;
|
||||
const runs = await store.listRuns({
|
||||
owner: q.owner,
|
||||
workflow: q.workflow,
|
||||
status: q.status,
|
||||
limit: Number.isFinite(limit) ? limit : undefined,
|
||||
before: q.before,
|
||||
});
|
||||
return reply.send({ runs });
|
||||
},
|
||||
);
|
||||
|
||||
server.get(
|
||||
"/admin/runs/:id",
|
||||
{ onRequest: [server.authenticate] },
|
||||
async (req, reply) => {
|
||||
const { id } = /** @type {{ id: string }} */ (req.params);
|
||||
const run = await store.getRun(id);
|
||||
if (!run) {
|
||||
return reply.code(404).send({ error: "run not found" });
|
||||
}
|
||||
return reply.send(run);
|
||||
},
|
||||
);
|
||||
|
||||
if (fs.existsSync(WEB_DIST)) {
|
||||
await server.register(fastifyStatic, {
|
||||
root: WEB_DIST,
|
||||
wildcard: false,
|
||||
});
|
||||
server.setNotFoundHandler((req, reply) => {
|
||||
const url = req.raw.url ?? "";
|
||||
if (
|
||||
url.startsWith("/api") ||
|
||||
url.startsWith("/u/") ||
|
||||
url.startsWith("/admin")
|
||||
) {
|
||||
return reply.code(404).send({ error: "not found" });
|
||||
}
|
||||
return reply.sendFile("index.html");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function shutdown() {
|
||||
try {
|
||||
if (workflowWorker) {
|
||||
await workflowWorker.close();
|
||||
}
|
||||
await workflowQueue.close();
|
||||
await closeRedis();
|
||||
await flushLogs();
|
||||
await db.destroy();
|
||||
} catch (err) {
|
||||
@@ -176,15 +227,19 @@ process.on("SIGTERM", shutdown);
|
||||
|
||||
const port = Number(process.env.PORT ?? 9000);
|
||||
|
||||
server
|
||||
.listen({
|
||||
host: "0.0.0.0",
|
||||
port,
|
||||
})
|
||||
.then(() => {
|
||||
log.info(`Server is running on port ${port}`);
|
||||
})
|
||||
.catch((err) => {
|
||||
log.error({ err }, "failed to start server");
|
||||
process.exit(1);
|
||||
});
|
||||
if (runApi) {
|
||||
server
|
||||
.listen({
|
||||
host: "0.0.0.0",
|
||||
port,
|
||||
})
|
||||
.then(() => {
|
||||
log.info(`Server is running on port ${port}`);
|
||||
})
|
||||
.catch((err) => {
|
||||
log.error({ err }, "failed to start server");
|
||||
process.exit(1);
|
||||
});
|
||||
} else {
|
||||
log.info("worker-only mode; HTTP server not started");
|
||||
}
|
||||
Reference in New Issue
Block a user