feat(workflows): add auto-disable feature for workflows on consecutive failures
Deploy to Raspberry Pi / deploy (push) Canceled after 0s
Deploy to Raspberry Pi / deploy (push) Canceled after 0s
- Introduced `disableOnConsecutiveFailures` option in workflow triggers to automatically disable workflows after reaching a specified failure threshold. - Updated related functions to handle the new feature, including persistence of the disabled state and reloading of registries. - Enhanced UI components to support the new option, allowing users to toggle the auto-disable feature in the workflow configuration.
This commit is contained in:
@@ -36,7 +36,9 @@ import {
|
||||
resolveFailureTriggerConfig,
|
||||
} from "./trigger-failure.js";
|
||||
import { enqueueWorkflowJob } from "./workflow-queue.js";
|
||||
import { ensureInitialRevision } from "./workflow-history.js";
|
||||
import { ensureInitialRevision, recordRevision } from "./workflow-history.js";
|
||||
import { workflowIdFromFile } from "./workflow-normalize.js";
|
||||
import { publishReload } from "./control-bus.js";
|
||||
|
||||
/**
|
||||
* @typedef {{ owner: string, file: string, workflow: any }} WorkflowEntry
|
||||
@@ -652,6 +654,45 @@ export function createRegistry(server, opts = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist `enabled: false` for a workflow and reload registries across processes.
|
||||
* @param {string} owner
|
||||
* @param {string} file
|
||||
* @param {string} key
|
||||
*/
|
||||
async function disableWorkflowForConsecutiveFailures(owner, file, key) {
|
||||
const content = fsStore.readWorkflowYaml(owner, file);
|
||||
if (content == null) {
|
||||
throw new Error(`workflow file missing for ${key}`);
|
||||
}
|
||||
const doc = yaml.parseDocument(content);
|
||||
if (doc.errors?.length) {
|
||||
throw new Error(doc.errors[0]?.message ?? "invalid yaml");
|
||||
}
|
||||
const parsed = doc.toJSON();
|
||||
if (parsed?.enabled === false) {
|
||||
log.debug({ workflow: key }, "workflow already disabled");
|
||||
return;
|
||||
}
|
||||
doc.set("enabled", false);
|
||||
const nextContent = String(doc);
|
||||
fsStore.writeWorkflowYaml(owner, file, nextContent);
|
||||
await recordRevision({
|
||||
workflowId: workflowIdFromFile(file),
|
||||
owner,
|
||||
file,
|
||||
content: nextContent,
|
||||
reason: "disable-on-consecutive-failures",
|
||||
});
|
||||
reregister();
|
||||
try {
|
||||
await publishReload({ type: "workflows" });
|
||||
} catch {
|
||||
// Redis may be briefly unavailable; local reload already applied.
|
||||
}
|
||||
log.warn({ workflow: key }, "disabled workflow after consecutive failures");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{
|
||||
* key: string,
|
||||
@@ -688,6 +729,17 @@ export function createRegistry(server, opts = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (failureConfig.disableOnConsecutiveFailures) {
|
||||
const entry = workflows.get(opts.key);
|
||||
if (entry) {
|
||||
await disableWorkflowForConsecutiveFailures(entry.owner, entry.file, opts.key);
|
||||
} else {
|
||||
log.warn({ workflow: opts.key }, "cannot disable missing workflow entry");
|
||||
}
|
||||
}
|
||||
|
||||
if (!failureConfig.workflowName) return;
|
||||
|
||||
const destKey = resolveWorkflowTriggerKey(opts.owner, failureConfig.workflowName);
|
||||
const alertData = buildFailureAlertData({
|
||||
sourceKey: opts.key,
|
||||
|
||||
Reference in New Issue
Block a user