[grade=B] DC-093: Workflow engine retry with exponential backoff
Actions now retry up to 3 times with 2/4/8s exponential backoff before giving up. Logs each retry attempt with attempt count. exhaustedRetries field in failure result shows total attempts made. All 1540 tests pass.
This commit is contained in:
@@ -252,32 +252,49 @@ class WorkflowEngine extends EventEmitter {
|
||||
*/
|
||||
async _runActions(actions, triggerData = {}) {
|
||||
const results = [];
|
||||
const MAX_RETRIES = 3;
|
||||
const RETRY_DELAY_MS = 2000;
|
||||
|
||||
for (let i = 0; i < actions.length; i++) {
|
||||
const action = actions[i];
|
||||
const previousResult = i > 0 ? results[i - 1] : null;
|
||||
// notify-on-failure needs to see the previous action's outcome to decide
|
||||
// whether to fire. Passing the full results array in the trigger data lets
|
||||
// executeAction do that lookup without changing the action shape.
|
||||
// Also surface failingServices (set by healthCheckService on throw) so
|
||||
// template variables like {{failingServices}} can interpolate.
|
||||
const actionContext = {
|
||||
...triggerData,
|
||||
previousResult,
|
||||
failingServices: previousResult && previousResult.failingServices ? previousResult.failingServices : undefined,
|
||||
};
|
||||
|
||||
// DC-093: Retry with exponential backoff for transient failures
|
||||
let lastError = null;
|
||||
let result = null;
|
||||
let succeeded = false;
|
||||
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
const result = await this.executeAction(action, actionContext);
|
||||
results.push({ action: action.type, success: true, result });
|
||||
result = await this.executeAction(action, actionContext);
|
||||
succeeded = true;
|
||||
break;
|
||||
} catch (error) {
|
||||
log.error('workflow', error, { actionType: action.type });
|
||||
lastError = error;
|
||||
if (attempt < MAX_RETRIES) {
|
||||
const delay = RETRY_DELAY_MS * Math.pow(2, attempt);
|
||||
log.warn('workflow', `Action "${action.type}" failed (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying in ${delay}ms`, { error: error.message });
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (succeeded) {
|
||||
results.push({ action: action.type, success: true, result });
|
||||
} else {
|
||||
log.error('workflow', `Action "${action.type}" failed after ${MAX_RETRIES + 1} attempts`, { error: lastError.message });
|
||||
results.push({
|
||||
action: action.type,
|
||||
success: false,
|
||||
error: error.message,
|
||||
failingServices: error.failingServices,
|
||||
error: lastError.message,
|
||||
failingServices: lastError.failingServices,
|
||||
exhaustedRetries: MAX_RETRIES + 1,
|
||||
});
|
||||
// Continue with other actions but log failure
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user