From 04c0d4b80171a02676f1bac68ef0a376663a2eb8 Mon Sep 17 00:00:00 2001 From: Nasyarobby Putra Date: Tue, 18 Aug 2026 11:14:46 +0700 Subject: [PATCH] feat(failures): implement consecutive failure tracking and dashboard integration - Added a new function to list consecutive failure streaks for workflows, allowing tracking of workflows that have failed multiple times in a row. - Integrated the consecutive failure data into the dashboard, displaying streaks and counts for better visibility of workflow health. - Created a dedicated FailuresPage to present detailed information about consecutive failures, enhancing user experience and monitoring capabilities. - Updated API endpoints and hooks to support the new failure tracking features, ensuring seamless data retrieval and display. This enhancement improves the ability to monitor and respond to workflow failures, contributing to overall system reliability. --- packages/server/src/api/dashboard.js | 9 +- packages/server/src/api/runs.js | 9 ++ packages/server/store.js | 92 ++++++++++++++ .../default/detect-example-changes.yaml | 1 + packages/web/src/App.jsx | 2 + packages/web/src/api/hooks.js | 11 ++ packages/web/src/pages/EventsPage.jsx | 2 +- packages/web/src/pages/FailuresPage.jsx | 63 +++++++++ packages/web/src/pages/HomePage.jsx | 120 +++++++++++------- 9 files changed, 262 insertions(+), 47 deletions(-) create mode 100644 packages/web/src/pages/FailuresPage.jsx diff --git a/packages/server/src/api/dashboard.js b/packages/server/src/api/dashboard.js index fe3831e..584a930 100644 --- a/packages/server/src/api/dashboard.js +++ b/packages/server/src/api/dashboard.js @@ -63,9 +63,10 @@ export default function dashboardPluginFactory(registry) { } } - const [running, failed, recent] = await Promise.all([ + const [running, streaks, failedEvents, recent] = await Promise.all([ store.listRuns({ status: "running", limit: 10 }), - store.listRuns({ status: "failed", limit: 20 }), + store.listConsecutiveFailureStreaks({ minCount: 4, limit: 10 }), + store.listRuns({ status: "failed", limit: 5 }), store.listRuns({ limit: 10 }), ]); @@ -76,9 +77,11 @@ export default function dashboardPluginFactory(registry) { brokenCount, running, needsAttention: { - failed, + consecutiveFailures: streaks.items, + consecutiveFailureCount: streaks.total, brokenWorkflows, }, + failedEvents, recent, }; }); diff --git a/packages/server/src/api/runs.js b/packages/server/src/api/runs.js index 570e707..1aeb8ba 100644 --- a/packages/server/src/api/runs.js +++ b/packages/server/src/api/runs.js @@ -17,6 +17,15 @@ export default async function runsPlugin(fastify) { return { runs }; }); + fastify.get("/consecutive-failures", async (req) => { + const q = /** @type {Record} */ (req.query ?? {}); + const limit = q.limit ? Number(q.limit) : undefined; + return store.listConsecutiveFailureStreaks({ + minCount: 4, + limit: Number.isFinite(limit) ? limit : 200, + }); + }); + fastify.get("/runs/:id", async (req, reply) => { const { id } = /** @type {{ id: string }} */ (req.params); const run = await store.getRun(id); diff --git a/packages/server/store.js b/packages/server/store.js index d8510dd..ea62c1f 100644 --- a/packages/server/store.js +++ b/packages/server/store.js @@ -278,6 +278,98 @@ export async function countConsecutiveFailures(workflow, triggerType, triggerDet return count; } +const CONSECUTIVE_FAILURE_WINDOW = 5000; +const STREAK_LAST_RUN_FIELDS = [ + "id", + "owner", + "workflow", + "workflow_name", + "trigger_type", + "trigger_detail", + "status", + "started_at", + "finished_at", + "duration_ms", + "error", +]; + +/** + * Workflow+trigger groups currently in a trailing failure streak. + * + * @param {{ + * minCount?: number, + * limit?: number, + * }} [opts] + * @returns {Promise<{ + * items: Array<{ + * consecutiveFailures: number, + * workflow: string, + * workflow_name: string | null, + * owner: string, + * trigger_type: string, + * trigger_detail: string | null, + * lastRun: { + * id: string, + * owner: string, + * workflow: string, + * workflow_name: string | null, + * trigger_type: string, + * trigger_detail: string | null, + * status: string, + * started_at: string, + * finished_at: string | null, + * duration_ms: number | null, + * error: string | null, + * }, + * }>, + * total: number, + * }>} + */ +export async function listConsecutiveFailureStreaks(opts = {}) { + const minCount = Math.max(opts.minCount ?? 4, 1); + const limit = Math.min(Math.max(opts.limit ?? 100, 1), 200); + + const rows = await db("workflow_runs") + .select(STREAK_LAST_RUN_FIELDS) + .whereIn("status", ["success", "failed"]) + .orderBy("started_at", "desc") + .limit(CONSECUTIVE_FAILURE_WINDOW); + + /** @type {Map} */ + const groups = new Map(); + for (const row of rows) { + const key = `${row.workflow}\0${row.trigger_type}\0${row.trigger_detail ?? ""}`; + let group = groups.get(key); + if (!group) { + group = { count: 0, done: false, lastRun: row }; + groups.set(key, group); + } + if (group.done) continue; + if (row.status === "failed") group.count += 1; + else group.done = true; + } + + const streaks = [...groups.values()] + .filter((g) => g.lastRun.status === "failed" && g.count >= minCount) + .sort((a, b) => { + if (b.count !== a.count) return b.count - a.count; + return String(b.lastRun.started_at).localeCompare(String(a.lastRun.started_at)); + }); + + return { + total: streaks.length, + items: streaks.slice(0, limit).map((g) => ({ + consecutiveFailures: g.count, + workflow: g.lastRun.workflow, + workflow_name: g.lastRun.workflow_name, + owner: g.lastRun.owner, + trigger_type: g.lastRun.trigger_type, + trigger_detail: g.lastRun.trigger_detail, + lastRun: g.lastRun, + })), + }; +} + /** * @returns {Promise>} */ diff --git a/packages/server/workflows/default/detect-example-changes.yaml b/packages/server/workflows/default/detect-example-changes.yaml index be96813..326a7c7 100644 --- a/packages/server/workflows/default/detect-example-changes.yaml +++ b/packages/server/workflows/default/detect-example-changes.yaml @@ -20,3 +20,4 @@ triggers: schedule: "* * * * *" onConsecutiveFailures: 3 onFailureWorkflow: dev-zte-sms +enabled: false diff --git a/packages/web/src/App.jsx b/packages/web/src/App.jsx index 6bf8baf..e90e00f 100644 --- a/packages/web/src/App.jsx +++ b/packages/web/src/App.jsx @@ -12,6 +12,7 @@ import { WorkflowsPage } from "./pages/WorkflowsPage.jsx"; import { WorkflowEditPage, WorkflowNewPage } from "./pages/WorkflowEditPage.jsx"; import { EventsPage } from "./pages/EventsPage.jsx"; import { EventDetailPage } from "./pages/EventDetailPage.jsx"; +import { FailuresPage } from "./pages/FailuresPage.jsx"; import { KvPage } from "./pages/KvPage.jsx"; import { AuthProfilesPage } from "./pages/AuthProfilesPage.jsx"; import { ResponsesPage } from "./pages/ResponsesPage.jsx"; @@ -61,6 +62,7 @@ export function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/packages/web/src/api/hooks.js b/packages/web/src/api/hooks.js index e4387f2..7c48e82 100644 --- a/packages/web/src/api/hooks.js +++ b/packages/web/src/api/hooks.js @@ -266,6 +266,17 @@ export function useRuns(filters = {}) { }); } +export function useConsecutiveFailures(limit) { + return useQuery({ + queryKey: ["consecutive-failures", limit ?? "all"], + queryFn: async () => { + const params = {}; + if (limit) params.limit = limit; + return (await api.get("/consecutive-failures", { params })).data; + }, + }); +} + export function useRun(id) { return useQuery({ queryKey: ["runs", id], diff --git a/packages/web/src/pages/EventsPage.jsx b/packages/web/src/pages/EventsPage.jsx index 7df9f34..4e867ce 100644 --- a/packages/web/src/pages/EventsPage.jsx +++ b/packages/web/src/pages/EventsPage.jsx @@ -21,7 +21,7 @@ export function EventsPage() { return (
-

Events

+

{status === "failed" ? "Failed events" : "Events"}

+

Failures

+

+ Workflows that have failed 4 or more times in a row for the same trigger. +

+ {isLoading ? ( + + ) : error ? ( +

Failed to load consecutive failures

+ ) : items.length === 0 ? ( +

None

+ ) : ( +
+ + + + + + + + + + + + {items.map((s) => ( + + + + + + + + ))} + +
StreakWorkflowTriggerLast errorLast failed
+ {s.consecutiveFailures} + + + {s.workflow_name || s.workflow} + + + {s.trigger_type} + {s.trigger_detail ? ` · ${s.trigger_detail}` : ""} + + {s.lastRun.error || "—"} + {formatTime(s.lastRun.started_at)}
+
+ )} +
+ ); +} diff --git a/packages/web/src/pages/HomePage.jsx b/packages/web/src/pages/HomePage.jsx index 6b515ac..1631626 100644 --- a/packages/web/src/pages/HomePage.jsx +++ b/packages/web/src/pages/HomePage.jsx @@ -1,10 +1,5 @@ import { Link } from "react-router-dom"; -import { - LuActivity, - LuCode, - LuGitBranch, - LuTriangleAlert, -} from "react-icons/lu"; +import { LuCode, LuGitBranch, LuTriangleAlert } from "react-icons/lu"; import { useDashboard } from "../api/hooks.js"; import { formatTime, StatusBadge } from "../lib/format.jsx"; @@ -18,8 +13,10 @@ export function HomePage() { return

Failed to load dashboard

; } - const failed = data.needsAttention?.failed ?? []; + const streaks = data.needsAttention?.consecutiveFailures ?? []; + const streakCount = data.needsAttention?.consecutiveFailureCount ?? streaks.length; const broken = data.needsAttention?.brokenWorkflows ?? []; + const failedEvents = data.failedEvents ?? []; return (
@@ -39,52 +36,89 @@ export function HomePage() {
Scripts
{data.scriptCount}
-
-
- -
-
Running
-
{data.running?.length ?? 0}
-
Attention
-
- {failed.length + broken.length} -
+
{streakCount + broken.length}
-
-

Running

- -
+
+
+
+

Needs attention

+ + Show all + +
+ {broken.length > 0 ? ( +
    + {broken.map((w) => ( +
  • + + {w.key}: {w.loadError} + +
    • + ))} +
+ ) : null} + +
-
-

Needs attention

- {broken.length > 0 ? ( -
    - {broken.map((w) => ( -
  • - - {w.key}: {w.loadError} +
    +
    +

    Failed events

    + + View all + +
    + +
    + +
    +

    Recent

    + +
    +
+ + ); +} + +function StreakList({ streaks, empty }) { + if (!streaks?.length) { + return empty ?

{empty}

: null; + } + return ( +
+ + + + + + + + + + {streaks.map((s) => ( + + + + + + ))} + +
StreakWorkflowLast failed
+ {s.consecutiveFailures} + + + {s.workflow_name || s.workflow} - - ))} - - ) : null} - - - -
-

Recent

- -
+ 		{formatTime(s.lastRun.started_at)}
); }