368 lines
14 KiB
JavaScript
368 lines
14 KiB
JavaScript
/**
|
|
* Smoke tests for auto-restart-manager.js
|
|
* Verifies the AutoRestartManager class:
|
|
* - Policy CRUD (set/get/list/remove)
|
|
* - handleContainerDown: cooldown, max-retries, restart attempt, failure
|
|
* - handleContainerUp: retry counter reset
|
|
* - _handleStatusCheck: healthy→unhealthy and unhealthy→healthy transitions
|
|
* - _resolveContainerId: lookup precedence
|
|
*/
|
|
|
|
const EventEmitter = require('events');
|
|
const { AutoRestartManager, DEFAULT_POLICY } = require('../auto-restart-manager');
|
|
|
|
jest.mock('../fs-helpers', () => ({
|
|
readJsonFile: jest.fn().mockResolvedValue({}),
|
|
writeJsonFile: jest.fn().mockResolvedValue(undefined),
|
|
}));
|
|
|
|
const fsHelpers = require('../fs-helpers');
|
|
|
|
function makeManager(overrides = {}) {
|
|
const servicesStateManager = {
|
|
read: jest.fn().mockResolvedValue([]),
|
|
...(overrides.servicesStateManager || {}),
|
|
};
|
|
|
|
const docker = {
|
|
client: {
|
|
getContainer: jest.fn(),
|
|
...(overrides.dockerClient || {}),
|
|
},
|
|
};
|
|
|
|
const healthChecker = new EventEmitter();
|
|
if (overrides.healthChecker) {
|
|
Object.assign(healthChecker, overrides.healthChecker);
|
|
}
|
|
|
|
const notification = {
|
|
send: jest.fn().mockResolvedValue({ success: true }),
|
|
...(overrides.notification || {}),
|
|
};
|
|
|
|
const ctx = {
|
|
docker,
|
|
healthChecker,
|
|
notification,
|
|
servicesStateManager,
|
|
SERVICES_FILE: '/tmp/dc-test/services.json',
|
|
log: { info: jest.fn(), error: jest.fn(), warn: jest.fn(), debug: jest.fn() },
|
|
logError: jest.fn(),
|
|
};
|
|
|
|
const manager = new AutoRestartManager(ctx);
|
|
return { manager, ctx, docker, healthChecker, notification, servicesStateManager };
|
|
}
|
|
|
|
describe('AutoRestartManager', () => {
|
|
beforeEach(() => {
|
|
jest.clearAllMocks();
|
|
fsHelpers.readJsonFile.mockResolvedValue({});
|
|
fsHelpers.writeJsonFile.mockResolvedValue(undefined);
|
|
});
|
|
|
|
describe('constants & construction', () => {
|
|
test('DEFAULT_POLICY has the documented fields and sensible defaults', () => {
|
|
expect(DEFAULT_POLICY).toEqual({
|
|
enabled: true,
|
|
maxRetries: 3,
|
|
retryIntervalMs: 5000,
|
|
windowMinutes: 10,
|
|
currentRetries: 0,
|
|
lastRestartAt: null,
|
|
cooldownUntil: null,
|
|
});
|
|
});
|
|
|
|
test('manager extends EventEmitter and stores ctx deps', () => {
|
|
const { manager, ctx } = makeManager();
|
|
expect(manager).toBeInstanceOf(EventEmitter);
|
|
expect(manager.docker).toBe(ctx.docker);
|
|
expect(manager.healthChecker).toBe(ctx.healthChecker);
|
|
expect(manager.notification).toBe(ctx.notification);
|
|
expect(manager.policies).toBeInstanceOf(Map);
|
|
});
|
|
});
|
|
|
|
describe('lifecycle', () => {
|
|
test('start() loads persisted policies from fs-helpers', async () => {
|
|
fsHelpers.readJsonFile.mockResolvedValue({
|
|
'svc-1': { enabled: false, maxRetries: 7 },
|
|
});
|
|
const { manager } = makeManager();
|
|
await manager.start();
|
|
expect(manager.policies.has('svc-1')).toBe(true);
|
|
const policy = manager.getPolicy('svc-1');
|
|
expect(policy.maxRetries).toBe(7);
|
|
expect(policy.enabled).toBe(false);
|
|
});
|
|
|
|
test('start() is idempotent (second call does nothing new)', async () => {
|
|
const { manager, healthChecker } = makeManager();
|
|
await manager.start();
|
|
const listenerCount = healthChecker.listenerCount('status-check');
|
|
await manager.start();
|
|
expect(healthChecker.listenerCount('status-check')).toBe(listenerCount);
|
|
});
|
|
|
|
test('stop() removes the status-check listener', async () => {
|
|
const { manager, healthChecker } = makeManager();
|
|
await manager.start();
|
|
expect(healthChecker.listenerCount('status-check')).toBe(1);
|
|
manager.stop();
|
|
expect(healthChecker.listenerCount('status-check')).toBe(0);
|
|
});
|
|
});
|
|
|
|
describe('policy CRUD', () => {
|
|
test('setPolicy throws on missing serviceId', async () => {
|
|
const { manager } = makeManager();
|
|
await expect(manager.setPolicy('', { enabled: true })).rejects.toThrow(/serviceId/);
|
|
await expect(manager.setPolicy(null, {})).rejects.toThrow(/serviceId/);
|
|
});
|
|
|
|
test('setPolicy merges fields with existing policy', async () => {
|
|
const { manager } = makeManager();
|
|
await manager.setPolicy('svc-1', { maxRetries: 5 });
|
|
await manager.setPolicy('svc-1', { enabled: false });
|
|
const policy = manager.getPolicy('svc-1');
|
|
expect(policy.maxRetries).toBe(5); // preserved from earlier
|
|
expect(policy.enabled).toBe(false); // updated by second call
|
|
});
|
|
|
|
test('setPolicy persists via fs-helpers.writeJsonFile', async () => {
|
|
const { manager } = makeManager();
|
|
await manager.setPolicy('svc-1', { maxRetries: 4 });
|
|
expect(fsHelpers.writeJsonFile).toHaveBeenCalled();
|
|
const [filePath, payload] = fsHelpers.writeJsonFile.mock.calls[0];
|
|
expect(filePath).toMatch(/auto-restart-policies\.json$/);
|
|
expect(payload['svc-1'].maxRetries).toBe(4);
|
|
});
|
|
|
|
test('getPolicy returns a copy, not the internal reference', async () => {
|
|
const { manager } = makeManager();
|
|
await manager.setPolicy('svc-1', { maxRetries: 2 });
|
|
const a = manager.getPolicy('svc-1');
|
|
a.maxRetries = 999;
|
|
const b = manager.getPolicy('svc-1');
|
|
expect(b.maxRetries).toBe(2);
|
|
});
|
|
|
|
test('getPolicy returns null for unknown service', () => {
|
|
const { manager } = makeManager();
|
|
expect(manager.getPolicy('does-not-exist')).toBeNull();
|
|
});
|
|
|
|
test('listPolicies returns array of all policies', async () => {
|
|
const { manager } = makeManager();
|
|
await manager.setPolicy('svc-1', { maxRetries: 1 });
|
|
await manager.setPolicy('svc-2', { maxRetries: 2 });
|
|
const list = manager.listPolicies();
|
|
expect(Array.isArray(list)).toBe(true);
|
|
expect(list).toHaveLength(2);
|
|
const ids = list.map(p => p.serviceId);
|
|
expect(ids).toEqual(expect.arrayContaining(['svc-1', 'svc-2']));
|
|
});
|
|
|
|
test('removePolicy returns true and deletes the policy', async () => {
|
|
const { manager } = makeManager();
|
|
await manager.setPolicy('svc-1', { maxRetries: 1 });
|
|
expect(await manager.removePolicy('svc-1')).toBe(true);
|
|
expect(manager.getPolicy('svc-1')).toBeNull();
|
|
});
|
|
|
|
test('removePolicy returns false for unknown service', async () => {
|
|
const { manager } = makeManager();
|
|
expect(await manager.removePolicy('does-not-exist')).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('handleContainerDown', () => {
|
|
test('returns ignored/no-policy when no policy exists', async () => {
|
|
const { manager } = makeManager();
|
|
const result = await manager.handleContainerDown('unknown', 'cid');
|
|
expect(result.action).toBe('ignored');
|
|
expect(result.reason).toBe('no-policy');
|
|
});
|
|
|
|
test('returns ignored/disabled when policy.enabled is false', async () => {
|
|
const { manager } = makeManager();
|
|
await manager.setPolicy('svc-1', { enabled: false });
|
|
const result = await manager.handleContainerDown('svc-1', 'cid');
|
|
expect(result.action).toBe('ignored');
|
|
expect(result.reason).toBe('disabled');
|
|
});
|
|
|
|
test('returns skipped/cooldown when cooldownUntil is in the future', async () => {
|
|
const { manager } = makeManager();
|
|
// setPolicy() intentionally guards runtime fields; we have to set
|
|
// cooldownUntil via the internal map to simulate an in-progress cooldown
|
|
await manager.setPolicy('svc-1', { maxRetries: 3 });
|
|
manager.policies.get('svc-1').cooldownUntil = Date.now() + 60_000;
|
|
const result = await manager.handleContainerDown('svc-1', 'cid');
|
|
expect(result.action).toBe('skipped');
|
|
expect(result.reason).toBe('cooldown');
|
|
});
|
|
|
|
test('increments currentRetries and calls docker.start on a successful restart', async () => {
|
|
const { manager, docker } = makeManager();
|
|
docker.client.getContainer.mockReturnValue({
|
|
start: jest.fn().mockResolvedValue(undefined),
|
|
});
|
|
await manager.setPolicy('svc-1', { maxRetries: 3, retryIntervalMs: 0 });
|
|
|
|
const onAttempt = jest.fn();
|
|
const onSuccess = jest.fn();
|
|
manager.on('auto-restart-attempt', onAttempt);
|
|
manager.on('auto-restart-success', onSuccess);
|
|
|
|
const result = await manager.handleContainerDown('svc-1', 'cid-abc');
|
|
expect(result.action).toBe('restarted');
|
|
expect(result.attempt).toBe(1);
|
|
expect(result.serviceId).toBe('svc-1');
|
|
expect(docker.client.getContainer).toHaveBeenCalledWith('cid-abc');
|
|
expect(onAttempt).toHaveBeenCalledTimes(1);
|
|
expect(onSuccess).toHaveBeenCalledTimes(1);
|
|
expect(manager.getPolicy('svc-1').currentRetries).toBe(1);
|
|
});
|
|
|
|
test('emits auto-restart-failed and increments currentRetries when docker.start throws', async () => {
|
|
const { manager, docker } = makeManager();
|
|
docker.client.getContainer.mockReturnValue({
|
|
start: jest.fn().mockRejectedValue(new Error('docker daemon down')),
|
|
});
|
|
await manager.setPolicy('svc-1', { maxRetries: 3, retryIntervalMs: 0 });
|
|
|
|
const onFailed = jest.fn();
|
|
manager.on('auto-restart-failed', onFailed);
|
|
|
|
const result = await manager.handleContainerDown('svc-1', 'cid-abc');
|
|
expect(result.action).toBe('failed');
|
|
expect(result.error).toMatch(/docker daemon down/);
|
|
expect(onFailed).toHaveBeenCalledTimes(1);
|
|
expect(manager.getPolicy('svc-1').currentRetries).toBe(1);
|
|
});
|
|
|
|
test('emits auto-restart-max-reached and sets cooldown when maxRetries exceeded', async () => {
|
|
const { manager, docker } = makeManager();
|
|
docker.client.getContainer.mockReturnValue({
|
|
start: jest.fn().mockResolvedValue(undefined),
|
|
});
|
|
await manager.setPolicy('svc-1', { maxRetries: 2, retryIntervalMs: 0 });
|
|
|
|
const onMax = jest.fn();
|
|
manager.on('auto-restart-max-reached', onMax);
|
|
|
|
// First attempt: currentRetries=0 -> succeeds, increments to 1
|
|
await manager.handleContainerDown('svc-1', 'cid');
|
|
// Second: 1 -> succeeds, increments to 2
|
|
await manager.handleContainerDown('svc-1', 'cid');
|
|
// Third: 2 >= maxRetries(2) -> max-reached, currentRetries reset to 0
|
|
const result = await manager.handleContainerDown('svc-1', 'cid');
|
|
|
|
expect(result.action).toBe('max-reached');
|
|
expect(onMax).toHaveBeenCalledTimes(1);
|
|
const policy = manager.getPolicy('svc-1');
|
|
expect(policy.currentRetries).toBe(0);
|
|
expect(policy.cooldownUntil).toBeGreaterThan(Date.now());
|
|
});
|
|
});
|
|
|
|
describe('handleContainerUp', () => {
|
|
test('resets currentRetries and cooldownUntil when service is tracked', async () => {
|
|
const { manager } = makeManager();
|
|
await manager.setPolicy('svc-1', { currentRetries: 2, cooldownUntil: Date.now() + 10000 });
|
|
// Mutate via internal map (bypassing the setter guard)
|
|
manager.policies.get('svc-1').currentRetries = 2;
|
|
manager.policies.get('svc-1').cooldownUntil = Date.now() + 10000;
|
|
|
|
await manager.handleContainerUp('svc-1');
|
|
const policy = manager.getPolicy('svc-1');
|
|
expect(policy.currentRetries).toBe(0);
|
|
expect(policy.cooldownUntil).toBeNull();
|
|
});
|
|
|
|
test('is a no-op when service is not tracked', async () => {
|
|
const { manager } = makeManager();
|
|
await expect(manager.handleContainerUp('unknown')).resolves.toBeUndefined();
|
|
});
|
|
});
|
|
|
|
describe('_handleStatusCheck', () => {
|
|
test('triggers handleContainerDown on healthy→unhealthy transition', async () => {
|
|
const { manager, docker } = makeManager();
|
|
docker.client.getContainer.mockReturnValue({
|
|
start: jest.fn().mockResolvedValue(undefined),
|
|
});
|
|
await manager.setPolicy('svc-1', { maxRetries: 3, retryIntervalMs: 0 });
|
|
// Pre-set previous health
|
|
manager._previousHealth.set('svc-1', 'up');
|
|
|
|
const handleDownSpy = jest.spyOn(manager, 'handleContainerDown');
|
|
await manager._handleStatusCheck({
|
|
serviceId: 'svc-1',
|
|
status: 'down',
|
|
details: { containerId: 'cid-1' },
|
|
});
|
|
expect(handleDownSpy).toHaveBeenCalledWith('svc-1', 'cid-1');
|
|
});
|
|
|
|
test('triggers handleContainerUp on unhealthy→healthy transition', async () => {
|
|
const { manager } = makeManager();
|
|
await manager.setPolicy('svc-1', { maxRetries: 3, retryIntervalMs: 0 });
|
|
manager._previousHealth.set('svc-1', 'down');
|
|
|
|
const handleUpSpy = jest.spyOn(manager, 'handleContainerUp');
|
|
await manager._handleStatusCheck({ serviceId: 'svc-1', status: 'up' });
|
|
expect(handleUpSpy).toHaveBeenCalledWith('svc-1');
|
|
});
|
|
|
|
test('does nothing for services without a policy', async () => {
|
|
const { manager } = makeManager();
|
|
const handleDownSpy = jest.spyOn(manager, 'handleContainerDown');
|
|
const handleUpSpy = jest.spyOn(manager, 'handleContainerUp');
|
|
await manager._handleStatusCheck({ serviceId: 'untracked', status: 'down' });
|
|
expect(handleDownSpy).not.toHaveBeenCalled();
|
|
expect(handleUpSpy).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test('ignores status with no serviceId', async () => {
|
|
const { manager } = makeManager();
|
|
const handleDownSpy = jest.spyOn(manager, 'handleContainerDown');
|
|
await manager._handleStatusCheck({ status: 'down' });
|
|
expect(handleDownSpy).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe('_resolveContainerId', () => {
|
|
test('returns containerId from status.details when present', () => {
|
|
const { manager } = makeManager();
|
|
const cid = manager._resolveContainerId('svc-1', { details: { containerId: 'cid-details' } });
|
|
expect(cid).toBe('cid-details');
|
|
});
|
|
|
|
test('falls back to healthChecker.config.services[serviceId].containerId', () => {
|
|
const { manager, healthChecker } = makeManager();
|
|
healthChecker.config = { services: { 'svc-1': { containerId: 'cid-hc' } } };
|
|
const cid = manager._resolveContainerId('svc-1', { details: {} });
|
|
expect(cid).toBe('cid-hc');
|
|
});
|
|
|
|
test('falls back to servicesStateManager.read when sync list is returned', () => {
|
|
const { manager, servicesStateManager } = makeManager();
|
|
servicesStateManager.read.mockReturnValue([
|
|
{ id: 'svc-1', containerId: 'cid-state' },
|
|
]);
|
|
const cid = manager._resolveContainerId('svc-1', { details: {} });
|
|
expect(cid).toBe('cid-state');
|
|
});
|
|
|
|
test('returns null when no source has a containerId', () => {
|
|
const { manager } = makeManager();
|
|
const cid = manager._resolveContainerId('svc-unknown', { details: {} });
|
|
expect(cid).toBeNull();
|
|
});
|
|
});
|
|
});
|