import { Queue, Worker } from "bullmq"; import IORedis from "ioredis"; import { log } from "./logger.js"; const DEFAULT_REDIS_URL = "redis://127.0.0.1:6379"; const DEFAULT_QUEUE_NAME = "jerapah-workflows"; const DEFAULT_CONCURRENCY = 5; /** @type {IORedis | null} */ let sharedConnection = null; export function getRedisUrl() { return process.env.REDIS_URL || DEFAULT_REDIS_URL; } export function getQueueName() { return process.env.JFLOW_QUEUE_NAME || DEFAULT_QUEUE_NAME; } export function getWorkerConcurrency() { const raw = Number(process.env.JFLOW_WORKER_CONCURRENCY ?? DEFAULT_CONCURRENCY); if (!Number.isFinite(raw) || raw < 1) return DEFAULT_CONCURRENCY; return Math.floor(raw); } /** * BullMQ requires maxRetriesPerRequest: null for blocking commands. * @returns {IORedis} */ export function getSharedConnection() { if (sharedConnection) return sharedConnection; sharedConnection = new IORedis(getRedisUrl(), { maxRetriesPerRequest: null, enableReadyCheck: true, }); sharedConnection.on("error", (err) => { log.error({ err }, "redis connection error"); }); return sharedConnection; } /** * @returns {Queue} */ export function createWorkflowQueue() { return new Queue(getQueueName(), { connection: getSharedConnection(), defaultJobOptions: { removeOnComplete: { count: 1000 }, removeOnFail: { count: 5000 }, attempts: 1, }, }); } /** * @param {(job: import("bullmq").Job) => Promise} processor * @returns {Worker} */ export function createWorkflowWorker(processor) { const concurrency = getWorkerConcurrency(); const worker = new Worker(getQueueName(), processor, { connection: getSharedConnection(), concurrency, }); worker.on("error", (err) => { log.error({ err }, "workflow worker error"); }); log.info({ concurrency, queue: getQueueName() }, "workflow worker started"); return worker; } /** * @param {Queue} queue * @param {{ * runId: string, * key: string, * depth?: number, * }} data */ export async function enqueueWorkflowJob(queue, data) { const job = await queue.add( "run", { runId: data.runId, key: data.key, depth: data.depth ?? 0, }, { jobId: data.runId, }, ); return job; } /** * @param {IORedis | null} [connection] */ export async function closeRedis(connection = sharedConnection) { if (!connection) return; if (connection === sharedConnection) sharedConnection = null; await connection.quit().catch(() => connection.disconnect()); }