feat(failures): implement consecutive failure tracking and dashboard integration

- Added a new function to list consecutive failure streaks for workflows, allowing tracking of workflows that have failed multiple times in a row.
- Integrated the consecutive failure data into the dashboard, displaying streaks and counts for better visibility of workflow health.
- Created a dedicated FailuresPage to present detailed information about consecutive failures, enhancing user experience and monitoring capabilities.
- Updated API endpoints and hooks to support the new failure tracking features, ensuring seamless data retrieval and display.

This enhancement improves the ability to monitor and respond to workflow failures, contributing to overall system reliability.

packages/server/src/api/dashboard.js

@@ -63,9 +63,10 @@ export default function dashboardPluginFactory(registry) {
         }
       }
 
-      const [running, failed, recent] = await Promise.all([
+      const [running, streaks, failedEvents, recent] = await Promise.all([
         store.listRuns({ status: "running", limit: 10 }),
-        store.listRuns({ status: "failed", limit: 20 }),
+        store.listConsecutiveFailureStreaks({ minCount: 4, limit: 10 }),
+        store.listRuns({ status: "failed", limit: 5 }),
         store.listRuns({ limit: 10 }),
       ]);
 
@@ -76,9 +77,11 @@ export default function dashboardPluginFactory(registry) {
         brokenCount,
         running,
         needsAttention: {
-          failed,
+          consecutiveFailures: streaks.items,
+          consecutiveFailureCount: streaks.total,
           brokenWorkflows,
         },
+        failedEvents,
         recent,
       };
     });

packages/server/src/api/runs.js

@@ -17,6 +17,15 @@ export default async function runsPlugin(fastify) {
     return { runs };
   });
 
+  fastify.get("/consecutive-failures", async (req) => {
+    const q = /** @type {Record<string, string | undefined>} */ (req.query ?? {});
+    const limit = q.limit ? Number(q.limit) : undefined;
+    return store.listConsecutiveFailureStreaks({
+      minCount: 4,
+      limit: Number.isFinite(limit) ? limit : 200,
+    });
+  });
+
   fastify.get("/runs/:id", async (req, reply) => {
     const { id } = /** @type {{ id: string }} */ (req.params);
     const run = await store.getRun(id);

packages/server/store.js

@@ -278,6 +278,98 @@ export async function countConsecutiveFailures(workflow, triggerType, triggerDet
   return count;
 }
 
+const CONSECUTIVE_FAILURE_WINDOW = 5000;
+const STREAK_LAST_RUN_FIELDS = [
+  "id",
+  "owner",
+  "workflow",
+  "workflow_name",
+  "trigger_type",
+  "trigger_detail",
+  "status",
+  "started_at",
+  "finished_at",
+  "duration_ms",
+  "error",
+];
+
+/**
+ * Workflow+trigger groups currently in a trailing failure streak.
+ *
+ * @param {{
+ *   minCount?: number,
+ *   limit?: number,
+ * }} [opts]
+ * @returns {Promise<{
+ *   items: Array<{
+ *     consecutiveFailures: number,
+ *     workflow: string,
+ *     workflow_name: string | null,
+ *     owner: string,
+ *     trigger_type: string,
+ *     trigger_detail: string | null,
+ *     lastRun: {
+ *       id: string,
+ *       owner: string,
+ *       workflow: string,
+ *       workflow_name: string | null,
+ *       trigger_type: string,
+ *       trigger_detail: string | null,
+ *       status: string,
+ *       started_at: string,
+ *       finished_at: string | null,
+ *       duration_ms: number | null,
+ *       error: string | null,
+ *     },
+ *   }>,
+ *   total: number,
+ * }>}
+ */
+export async function listConsecutiveFailureStreaks(opts = {}) {
+  const minCount = Math.max(opts.minCount ?? 4, 1);
+  const limit = Math.min(Math.max(opts.limit ?? 100, 1), 200);
+
+  const rows = await db("workflow_runs")
+    .select(STREAK_LAST_RUN_FIELDS)
+    .whereIn("status", ["success", "failed"])
+    .orderBy("started_at", "desc")
+    .limit(CONSECUTIVE_FAILURE_WINDOW);
+
+  /** @type {Map<string, { count: number, done: boolean, lastRun: (typeof rows)[number] }>} */
+  const groups = new Map();
+  for (const row of rows) {
+    const key = `${row.workflow}\0${row.trigger_type}\0${row.trigger_detail ?? ""}`;
+    let group = groups.get(key);
+    if (!group) {
+      group = { count: 0, done: false, lastRun: row };
+      groups.set(key, group);
+    }
+    if (group.done) continue;
+    if (row.status === "failed") group.count += 1;
+    else group.done = true;
+  }
+
+  const streaks = [...groups.values()]
+    .filter((g) => g.lastRun.status === "failed" && g.count >= minCount)
+    .sort((a, b) => {
+      if (b.count !== a.count) return b.count - a.count;
+      return String(b.lastRun.started_at).localeCompare(String(a.lastRun.started_at));
+    });
+
+  return {
+    total: streaks.length,
+    items: streaks.slice(0, limit).map((g) => ({
+      consecutiveFailures: g.count,
+      workflow: g.lastRun.workflow,
+      workflow_name: g.lastRun.workflow_name,
+      owner: g.lastRun.owner,
+      trigger_type: g.lastRun.trigger_type,
+      trigger_detail: g.lastRun.trigger_detail,
+      lastRun: g.lastRun,
+    })),
+  };
+}
+
 /**
  * @returns {Promise<Record<string, { invocationCount: number, lastInvokedAt: string | null, lastStatus: string | null }>>}
  */

packages/server/workflows/default/detect-example-changes.yaml

@@ -20,3 +20,4 @@ triggers:
     schedule: "* * * * *"
     onConsecutiveFailures: 3
     onFailureWorkflow: dev-zte-sms
+enabled: false

packages/web/src/App.jsx

@@ -12,6 +12,7 @@ import { WorkflowsPage } from "./pages/WorkflowsPage.jsx";
 import { WorkflowEditPage, WorkflowNewPage } from "./pages/WorkflowEditPage.jsx";
 import { EventsPage } from "./pages/EventsPage.jsx";
 import { EventDetailPage } from "./pages/EventDetailPage.jsx";
+import { FailuresPage } from "./pages/FailuresPage.jsx";
 import { KvPage } from "./pages/KvPage.jsx";
 import { AuthProfilesPage } from "./pages/AuthProfilesPage.jsx";
 import { ResponsesPage } from "./pages/ResponsesPage.jsx";
@@ -61,6 +62,7 @@ export function App() {
         <Route path="/workflows/:owner/:file/edit" element={<WorkflowEditPage />} />
         <Route path="/events" element={<EventsPage />} />
         <Route path="/events/:id" element={<EventDetailPage />} />
+        <Route path="/failures" element={<FailuresPage />} />
         <Route path="/kv" element={<KvPage />} />
         <Route path="/variables" element={<VariablesPage />} />
         <Route path="/auth" element={<AuthProfilesPage />} />

packages/web/src/api/hooks.js

@@ -266,6 +266,17 @@ export function useRuns(filters = {}) {
   });
 }
 
+export function useConsecutiveFailures(limit) {
+  return useQuery({
+    queryKey: ["consecutive-failures", limit ?? "all"],
+    queryFn: async () => {
+      const params = {};
+      if (limit) params.limit = limit;
+      return (await api.get("/consecutive-failures", { params })).data;
+    },
+  });
+}
+
 export function useRun(id) {
   return useQuery({
     queryKey: ["runs", id],

packages/web/src/pages/EventsPage.jsx

@@ -21,7 +21,7 @@ export function EventsPage() {
 
   return (
     <div className="space-y-4">
-      <h1 className="text-xl font-semibold">Events</h1>
+      <h1 className="text-xl font-semibold">{status === "failed" ? "Failed events" : "Events"}</h1>
       <div className="flex flex-col sm:flex-row gap-2">
         <input
           className="input input-sm w-full sm:max-w-sm"

packages/web/src/pages/FailuresPage.jsx

@@ -0,0 +1,63 @@
+import { Link } from "react-router-dom";
+import { useConsecutiveFailures } from "../api/hooks.js";
+import { formatTime } from "../lib/format.jsx";
+
+export function FailuresPage() {
+  const { data, isLoading, error } = useConsecutiveFailures();
+  const items = data?.items ?? [];
+
+  return (
+    <div className="space-y-4">
+      <h1 className="text-xl font-semibold">Failures</h1>
+      <p className="opacity-70 text-sm">
+        Workflows that have failed 4 or more times in a row for the same trigger.
+      </p>
+      {isLoading ? (
+        <span className="loading loading-spinner" />
+      ) : error ? (
+        <p className="text-error">Failed to load consecutive failures</p>
+      ) : items.length === 0 ? (
+        <p className="opacity-60 text-sm">None</p>
+      ) : (
+        <div className="overflow-x-auto">
+          <table className="table table-sm">
+            <thead>
+              <tr>
+                <th>Streak</th>
+                <th>Workflow</th>
+                <th>Trigger</th>
+                <th>Last error</th>
+                <th>Last failed</th>
+              </tr>
+            </thead>
+            <tbody>
+              {items.map((s) => (
+                <tr
+                  key={`${s.workflow}\0${s.trigger_type}\0${s.trigger_detail ?? ""}`}
+                  className="hover"
+                >
+                  <td>
+                    <span className="badge badge-error badge-sm">{s.consecutiveFailures}</span>
+                  </td>
+                  <td>
+                    <Link className="link" to={`/events/${s.lastRun.id}`}>
+                      {s.workflow_name || s.workflow}
+                    </Link>
+                  </td>
+                  <td className="text-xs">
+                    {s.trigger_type}
+                    {s.trigger_detail ? ` · ${s.trigger_detail}` : ""}
+                  </td>
+                  <td className="max-w-md truncate text-xs" title={s.lastRun.error ?? ""}>
+                    {s.lastRun.error || "—"}
+                  </td>
+                  <td className="whitespace-nowrap">{formatTime(s.lastRun.started_at)}</td>
+                </tr>
+              ))}
+            </tbody>
+          </table>
+        </div>
+      )}
+    </div>
+  );
+}

packages/web/src/pages/HomePage.jsx

@@ -1,10 +1,5 @@
 import { Link } from "react-router-dom";
-import {
+import { LuCode, LuGitBranch, LuTriangleAlert } from "react-icons/lu";
-  LuActivity,
-  LuCode,
-  LuGitBranch,
-  LuTriangleAlert,
-} from "react-icons/lu";
 import { useDashboard } from "../api/hooks.js";
 import { formatTime, StatusBadge } from "../lib/format.jsx";
 
@@ -18,8 +13,10 @@ export function HomePage() {
     return <p className="text-error">Failed to load dashboard</p>;
   }
 
-  const failed = data.needsAttention?.failed ?? [];
+  const streaks = data.needsAttention?.consecutiveFailures ?? [];
+  const streakCount = data.needsAttention?.consecutiveFailureCount ?? streaks.length;
   const broken = data.needsAttention?.brokenWorkflows ?? [];
+  const failedEvents = data.failedEvents ?? [];
 
   return (
     <div className="space-y-6">
@@ -39,52 +36,89 @@ export function HomePage() {
           <div className="stat-title">Scripts</div>
           <div className="stat-value text-2xl">{data.scriptCount}</div>
         </div>
-        <div className="stat">
-          <div className="stat-figure text-warning">
-            <LuActivity className="size-7" />
-          </div>
-          <div className="stat-title">Running</div>
-          <div className="stat-value text-2xl">{data.running?.length ?? 0}</div>
-        </div>
         <div className="stat">
           <div className="stat-figure text-error">
             <LuTriangleAlert className="size-7" />
           </div>
           <div className="stat-title">Attention</div>
-          <div className="stat-value text-2xl">
+          <div className="stat-value text-2xl">{streakCount + broken.length}</div>
-            {failed.length + broken.length}
-          </div>
         </div>
       </div>
 
-      <section>
+      <div className="grid grid-cols-1 gap-6 lg:grid-cols-3">
-        <h2 className="text-lg font-semibold mb-2">Running</h2>
+        <section className="min-w-0">
-        <RunList runs={data.running} empty="None" />
+          <div className="mb-2 flex items-baseline justify-between gap-2">
-      </section>
+            <h2 className="text-lg font-semibold">Needs attention</h2>
+            <Link className="link link-hover text-sm" to="/failures">
+              Show all
+            </Link>
+          </div>
+          {broken.length > 0 ? (
+            <ul className="mb-3 space-y-1">
+              {broken.map((w) => (
+                <li key={w.key}>
+                  <Link
+                    className="link link-error"
+                    to={`/workflows/${encodeURIComponent(w.owner)}/${encodeURIComponent(w.file)}/edit`}
+                  >
+                    {w.key}: {w.loadError}
+                  </Link>
+                </li>
+              ))}
+            </ul>
+          ) : null}
+          <StreakList streaks={streaks} empty={broken.length ? "" : "None"} />
+        </section>
 
-      <section>
+        <section className="min-w-0">
-        <h2 className="text-lg font-semibold mb-2">Needs attention</h2>
+          <div className="mb-2 flex items-baseline justify-between gap-2">
-        {broken.length > 0 ? (
+            <h2 className="text-lg font-semibold">Failed events</h2>
-          <ul className="mb-3 space-y-1">
+            <Link className="link link-hover text-sm" to="/events?status=failed">
-            {broken.map((w) => (
+              View all
-              <li key={w.key}>
+            </Link>
-                <Link
+          </div>
-                  className="link link-error"
+          <RunList runs={failedEvents} empty="None" />
-                  to={`/workflows/${encodeURIComponent(w.owner)}/${encodeURIComponent(w.file)}/edit`}
+        </section>
-                >
+
-                  {w.key}: {w.loadError}
+        <section className="min-w-0">
+          <h2 className="text-lg font-semibold mb-2">Recent</h2>
+          <RunList runs={data.recent} empty="None" />
+        </section>
+      </div>
+    </div>
+  );
+}
+
+function StreakList({ streaks, empty }) {
+  if (!streaks?.length) {
+    return empty ? <p className="opacity-60 text-sm">{empty}</p> : null;
+  }
+  return (
+    <div className="overflow-x-auto">
+      <table className="table table-sm">
+        <thead>
+          <tr>
+            <th>Streak</th>
+            <th>Workflow</th>
+            <th>Last failed</th>
+          </tr>
+        </thead>
+        <tbody>
+          {streaks.map((s) => (
+            <tr key={`${s.workflow}\0${s.trigger_type}\0${s.trigger_detail ?? ""}`} className="hover">
+              <td>
+                <span className="badge badge-error badge-sm">{s.consecutiveFailures}</span>
+              </td>
+              <td>
+                <Link className="link" to={`/events/${s.lastRun.id}`}>
+                  {s.workflow_name || s.workflow}
                 </Link>
-              </li>
+              </td>
-            ))}
+              <td className="whitespace-nowrap">{formatTime(s.lastRun.started_at)}</td>
-          </ul>
+            </tr>
-        ) : null}
+          ))}
-        <RunList runs={failed} empty={broken.length ? "" : "None"} />
+        </tbody>
-      </section>
+      </table>
-
-      <section>
-        <h2 className="text-lg font-semibold mb-2">Recent</h2>
-        <RunList runs={data.recent} empty="None" />
-      </section>
     </div>
   );
 }