[grade=B] fix(workflows): DC-044 root-cause — gate notify-on-failure, interpolate failingServices, fix Health.Status check

The original DC-044 fix (b492e1c) repaired servicesStateManager.getState() but
missed two latent bugs at the same code path that were still spamming DNS2
every 15 minutes:

1. notify-on-failure fired unconditionally. The comment said 'Only send if
   previous action failed' but executeAction never checked. Every
   health-check-on-interval cycle ran notify regardless of outcome.

2. {{serviceId}} template never interpolated. healthCheckService returned
   { checked, healthy, results } with no serviceId in scope, so the
   production alert 'Health check failed for {{serviceId}}' stayed literal
   in every notification.

3. checkContainerHealth compared info.State.Health (an object) to the string
   'unhealthy' — always true, so any container with an explicit HEALTHCHECK
   was always reported healthy.

Fix:
- Extract _runActions(actions, triggerData) from executeWorkflow so the
  per-action result threading and failingServices context surface are
  testable in isolation.
- Gate notify-on-failure on previousResult.success === false. Returns
  { skipped: true, reason: 'no previous failure' } when no preceding failure.
- healthCheckService throws an Error with .failingServices attached when
  any service is unhealthy, surfacing IDs into the next action's context.
- checkContainerHealth now reads info.State.Health.Status: 'healthy' or
  'starting' → healthy, 'unhealthy' or no health check + stopped → unhealthy.
- Update bundled health-check-on-interval template from {{serviceId}} to
  {{failingServices}} (the variable now in scope).

Tests (12 new, 16 total in file):
- 5 _runActions tests (gate, interpolation, multi-service batch, first-action
  no-op, plain notify regression guard)
- 1 end-to-end executeWorkflow test against bundled health-check-on-interval
  asserting no literal {{...}} tokens reach notification.send
- 3 checkContainerHealth tests (running-but-unhealthy, no-healthcheck, stopped)
- 1 healthCheckService throw test with failingServices attached
- 2 updates to existing assertions for new return shape

Full suite: 1461/1463 (2 pre-existing license-keygen failures in DC-054
territory, unrelated to this commit).

Co-graded: Codex B urn:ump:b2nzzoulodwsullt3rhz4mtzou7fqgwiuoyrzxho67gdpwx3uvaa

dashcaddy-api/src/recipes/bundled-workflows.js

@@ -45,7 +45,12 @@ const BUNDLED_WORKFLOWS = {
     interval: 15 * 60 * 1000, // 15 minutes
     actions: [
       { type: 'health-check', target: '{{serviceId}}' },
-      { type: 'notify-on-failure', message: 'Health check failed for {{serviceId}}' }
+      // failingServices is set by healthCheckService when it throws (any
+      // service failed). It's a comma-joined string of failing service IDs.
+      // Previously this used {{serviceId}} which never resolved because
+      // no per-service ID is in scope at the workflow level — that's the
+      // DC-044 root-cause bug fix.
+      { type: 'notify-on-failure', message: 'Health check failed for {{failingServices}}' }
     ]
   },
   'disk-space-alert': {
@@ -194,34 +199,23 @@ class WorkflowEngine extends EventEmitter {
     if (!workflow) {
       throw new Error(`Unknown workflow: ${workflowId}`);
     }
-    
+
     if (!this.enabled.get(workflowId)) {
       console.log(`[WorkflowEngine] Workflow ${workflowId} is disabled, skipping`);
       return { skipped: true, reason: 'disabled' };
     }
-    
+
     const executionId = `${workflowId}-${Date.now()}`;
     const startTime = Date.now();
-    
+
     console.log(`[WorkflowEngine] Executing workflow: ${workflowId}`);
     this.emit('workflow-start', { workflowId, executionId, triggerData });
-    
-    const results = [];
-    
-    for (const action of workflow.actions) {
-      try {
-        const result = await this.executeAction(action, triggerData);
-        results.push({ action: action.type, success: true, result });
-      } catch (error) {
-        console.error(`[WorkflowEngine] Action ${action.type} failed:`, error.message);
-        results.push({ action: action.type, success: false, error: error.message });
-        // Continue with other actions but log failure
-      }
-    }
-    
+
+    const results = await this._runActions(workflow.actions, triggerData);
+
     const duration = Date.now() - startTime;
     const allSucceeded = results.every(r => r.success);
-    
+
     const historyEntry = {
       executionId,
       workflowId,
@@ -232,22 +226,63 @@ class WorkflowEngine extends EventEmitter {
       success: allSucceeded,
       results
     };
-    
+
     this.history.push(historyEntry);
-    
+
     // Keep history to last 500 entries
     if (this.history.length > 500) {
       this.history = this.history.slice(-500);
     }
-    
+
     this.saveHistory();
-    
+
     this.emit('workflow-complete', historyEntry);
     console.log(`[WorkflowEngine] Workflow ${workflowId} completed in ${duration}ms, success: ${allSucceeded}`);
-    
+
     return historyEntry;
   }
 
+  /**
+   * Run a sequence of actions and collect their results. Extracted from
+   * executeWorkflow so the per-action result threading (notify-on-failure
+   * gating) and the failingServices context surface can be unit-tested
+   * directly. executeWorkflow() is the production entry point; _runActions
+   * is an internal helper that callers shouldn't reach for.
+   */
+  async _runActions(actions, triggerData = {}) {
+    const results = [];
+
+    for (let i = 0; i < actions.length; i++) {
+      const action = actions[i];
+      const previousResult = i > 0 ? results[i - 1] : null;
+      // notify-on-failure needs to see the previous action's outcome to decide
+      // whether to fire. Passing the full results array in the trigger data lets
+      // executeAction do that lookup without changing the action shape.
+      // Also surface failingServices (set by healthCheckService on throw) so
+      // template variables like {{failingServices}} can interpolate.
+      const actionContext = {
+        ...triggerData,
+        previousResult,
+        failingServices: previousResult && previousResult.failingServices ? previousResult.failingServices : undefined,
+      };
+      try {
+        const result = await this.executeAction(action, actionContext);
+        results.push({ action: action.type, success: true, result });
+      } catch (error) {
+        console.error(`[WorkflowEngine] Action ${action.type} failed:`, error.message);
+        results.push({
+          action: action.type,
+          success: false,
+          error: error.message,
+          failingServices: error.failingServices,
+        });
+        // Continue with other actions but log failure
+      }
+    }
+
+    return results;
+  }
+
   /**
    * Execute a single action
    */
@@ -269,7 +304,12 @@ class WorkflowEngine extends EventEmitter {
         );
       
       case 'notify-on-failure':
-        // Only send if previous action failed
+        // Only send if previous action failed (success: false). The
+        // previousResult is injected by executeWorkflow's loop. If there
+        // was no previous action, this is a no-op (returns skipped).
+        if (!context.previousResult || context.previousResult.success !== false) {
+          return { skipped: true, reason: 'no previous failure' };
+        }
         return this.notify(
           this.interpolate(action.message, context),
           action.channel
@@ -323,11 +363,31 @@ class WorkflowEngine extends EventEmitter {
           }
         }
       }
-      return { checked: results.length, healthy: results.filter(r => r.healthy).length, results };
+      // Surface failing service IDs so downstream notify-on-failure actions
+      // can interpolate `{{failingServices}}` into the alert message. Without
+      // this, templates like `Health check failed for {{serviceId}}` stay
+      // literal because there's no serviceId in scope.
+      const failing = results.filter(r => !r.healthy).map(r => r.service);
+      const healthy = results.filter(r => r.healthy).length;
+      const result = { checked: results.length, healthy, results, failing };
+      if (failing.length > 0) {
+        // Throw so the action's success:false path is taken and notify-on-failure fires.
+        const err = new Error(`Health check failed for ${failing.length} service(s): ${failing.join(', ')}`);
+        err.failingServices = failing;
+        err.workflowResult = result;
+        throw err;
+      }
+      return result;
     }
-    
+
     // Single service check
     const healthy = await this.checkContainerHealth(serviceId);
+    if (!healthy) {
+      const err = new Error(`Health check failed for ${serviceId}`);
+      err.failingServices = [serviceId];
+      err.workflowResult = { serviceId, healthy };
+      throw err;
+    }
     return { serviceId, healthy };
   }
 
@@ -338,10 +398,18 @@ class WorkflowEngine extends EventEmitter {
     try {
       const docker = this.ctx.docker?.client;
       if (!docker) return false;
-      
+
       const container = docker.getContainer(containerId);
       const info = await container.inspect();
-      return info.State && info.State.Running && info.State.Health !== 'unhealthy';
+      // A container is healthy if it's running AND (it has no explicit
+      // health check OR its health check reports healthy/starting).
+      // info.State.Health is undefined when no HEALTHCHECK is declared.
+      // info.State.Health.Status is 'starting' | 'healthy' | 'unhealthy'
+      // when the health check IS declared.
+      if (!info.State || !info.State.Running) return false;
+      if (!info.State.Health) return true;  // no health check defined → running = healthy
+      const status = info.State.Health.Status;
+      return status === 'healthy' || status === 'starting';
     } catch (error) {
       return false;
     }