[grade=B] DC-093: Workflow engine retry with exponential backoff
Actions now retry up to 3 times with 2/4/8s exponential backoff before giving up. Logs each retry attempt with attempt count. exhaustedRetries field in failure result shows total attempts made. All 1540 tests pass.
This commit is contained in:
@@ -252,32 +252,49 @@ class WorkflowEngine extends EventEmitter {
|
|||||||
*/
|
*/
|
||||||
async _runActions(actions, triggerData = {}) {
|
async _runActions(actions, triggerData = {}) {
|
||||||
const results = [];
|
const results = [];
|
||||||
|
const MAX_RETRIES = 3;
|
||||||
|
const RETRY_DELAY_MS = 2000;
|
||||||
|
|
||||||
for (let i = 0; i < actions.length; i++) {
|
for (let i = 0; i < actions.length; i++) {
|
||||||
const action = actions[i];
|
const action = actions[i];
|
||||||
const previousResult = i > 0 ? results[i - 1] : null;
|
const previousResult = i > 0 ? results[i - 1] : null;
|
||||||
// notify-on-failure needs to see the previous action's outcome to decide
|
|
||||||
// whether to fire. Passing the full results array in the trigger data lets
|
|
||||||
// executeAction do that lookup without changing the action shape.
|
|
||||||
// Also surface failingServices (set by healthCheckService on throw) so
|
|
||||||
// template variables like {{failingServices}} can interpolate.
|
|
||||||
const actionContext = {
|
const actionContext = {
|
||||||
...triggerData,
|
...triggerData,
|
||||||
previousResult,
|
previousResult,
|
||||||
failingServices: previousResult && previousResult.failingServices ? previousResult.failingServices : undefined,
|
failingServices: previousResult && previousResult.failingServices ? previousResult.failingServices : undefined,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// DC-093: Retry with exponential backoff for transient failures
|
||||||
|
let lastError = null;
|
||||||
|
let result = null;
|
||||||
|
let succeeded = false;
|
||||||
|
|
||||||
|
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||||
try {
|
try {
|
||||||
const result = await this.executeAction(action, actionContext);
|
result = await this.executeAction(action, actionContext);
|
||||||
results.push({ action: action.type, success: true, result });
|
succeeded = true;
|
||||||
|
break;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
log.error('workflow', error, { actionType: action.type });
|
lastError = error;
|
||||||
|
if (attempt < MAX_RETRIES) {
|
||||||
|
const delay = RETRY_DELAY_MS * Math.pow(2, attempt);
|
||||||
|
log.warn('workflow', `Action "${action.type}" failed (attempt ${attempt + 1}/${MAX_RETRIES + 1}), retrying in ${delay}ms`, { error: error.message });
|
||||||
|
await new Promise(resolve => setTimeout(resolve, delay));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (succeeded) {
|
||||||
|
results.push({ action: action.type, success: true, result });
|
||||||
|
} else {
|
||||||
|
log.error('workflow', `Action "${action.type}" failed after ${MAX_RETRIES + 1} attempts`, { error: lastError.message });
|
||||||
results.push({
|
results.push({
|
||||||
action: action.type,
|
action: action.type,
|
||||||
success: false,
|
success: false,
|
||||||
error: error.message,
|
error: lastError.message,
|
||||||
failingServices: error.failingServices,
|
failingServices: lastError.failingServices,
|
||||||
|
exhaustedRetries: MAX_RETRIES + 1,
|
||||||
});
|
});
|
||||||
// Continue with other actions but log failure
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user