/** * Smoke tests for auto-restart-manager.js * Verifies the AutoRestartManager class: * - Policy CRUD (set/get/list/remove) * - handleContainerDown: cooldown, max-retries, restart attempt, failure * - handleContainerUp: retry counter reset * - _handleStatusCheck: healthy→unhealthy and unhealthy→healthy transitions * - _resolveContainerId: lookup precedence */ const EventEmitter = require('events'); const { AutoRestartManager, DEFAULT_POLICY } = require('../src/managers/auto-restart-manager'); jest.mock('../src/utilities/fs-helpers', () => ({ readJsonFile: jest.fn().mockResolvedValue({}), writeJsonFile: jest.fn().mockResolvedValue(undefined), })); const fsHelpers = require('../src/utilities/fs-helpers'); function makeManager(overrides = {}) { const servicesStateManager = { read: jest.fn().mockResolvedValue([]), ...(overrides.servicesStateManager || {}), }; const docker = { client: { getContainer: jest.fn(), ...(overrides.dockerClient || {}), }, }; const healthChecker = new EventEmitter(); if (overrides.healthChecker) { Object.assign(healthChecker, overrides.healthChecker); } const notification = { send: jest.fn().mockResolvedValue({ success: true }), ...(overrides.notification || {}), }; const ctx = { docker, healthChecker, notification, servicesStateManager, SERVICES_FILE: '/tmp/dc-test/services.json', log: { info: jest.fn(), error: jest.fn(), warn: jest.fn(), debug: jest.fn() }, logError: jest.fn(), }; const manager = new AutoRestartManager(ctx); return { manager, ctx, docker, healthChecker, notification, servicesStateManager }; } describe('AutoRestartManager', () => { beforeEach(() => { jest.clearAllMocks(); fsHelpers.readJsonFile.mockResolvedValue({}); fsHelpers.writeJsonFile.mockResolvedValue(undefined); }); describe('constants & construction', () => { test('DEFAULT_POLICY has the documented fields and sensible defaults', () => { expect(DEFAULT_POLICY).toEqual({ enabled: true, maxRetries: 3, retryIntervalMs: 5000, windowMinutes: 10, currentRetries: 0, lastRestartAt: null, cooldownUntil: null, }); }); test('manager extends EventEmitter and stores ctx deps', () => { const { manager, ctx } = makeManager(); expect(manager).toBeInstanceOf(EventEmitter); expect(manager.docker).toBe(ctx.docker); expect(manager.healthChecker).toBe(ctx.healthChecker); expect(manager.notification).toBe(ctx.notification); expect(manager.policies).toBeInstanceOf(Map); }); }); describe('lifecycle', () => { test('start() loads persisted policies from fs-helpers', async () => { fsHelpers.readJsonFile.mockResolvedValue({ 'svc-1': { enabled: false, maxRetries: 7 }, }); const { manager } = makeManager(); await manager.start(); expect(manager.policies.has('svc-1')).toBe(true); const policy = manager.getPolicy('svc-1'); expect(policy.maxRetries).toBe(7); expect(policy.enabled).toBe(false); }); test('start() is idempotent (second call does nothing new)', async () => { const { manager, healthChecker } = makeManager(); await manager.start(); const listenerCount = healthChecker.listenerCount('status-check'); await manager.start(); expect(healthChecker.listenerCount('status-check')).toBe(listenerCount); }); test('stop() removes the status-check listener', async () => { const { manager, healthChecker } = makeManager(); await manager.start(); expect(healthChecker.listenerCount('status-check')).toBe(1); manager.stop(); expect(healthChecker.listenerCount('status-check')).toBe(0); }); }); describe('policy CRUD', () => { test('setPolicy throws on missing serviceId', async () => { const { manager } = makeManager(); await expect(manager.setPolicy('', { enabled: true })).rejects.toThrow(/serviceId/); await expect(manager.setPolicy(null, {})).rejects.toThrow(/serviceId/); }); test('setPolicy merges fields with existing policy', async () => { const { manager } = makeManager(); await manager.setPolicy('svc-1', { maxRetries: 5 }); await manager.setPolicy('svc-1', { enabled: false }); const policy = manager.getPolicy('svc-1'); expect(policy.maxRetries).toBe(5); // preserved from earlier expect(policy.enabled).toBe(false); // updated by second call }); test('setPolicy persists via fs-helpers.writeJsonFile', async () => { const { manager } = makeManager(); await manager.setPolicy('svc-1', { maxRetries: 4 }); expect(fsHelpers.writeJsonFile).toHaveBeenCalled(); const [filePath, payload] = fsHelpers.writeJsonFile.mock.calls[0]; expect(filePath).toMatch(/auto-restart-policies\.json$/); expect(payload['svc-1'].maxRetries).toBe(4); }); test('getPolicy returns a copy, not the internal reference', async () => { const { manager } = makeManager(); await manager.setPolicy('svc-1', { maxRetries: 2 }); const a = manager.getPolicy('svc-1'); a.maxRetries = 999; const b = manager.getPolicy('svc-1'); expect(b.maxRetries).toBe(2); }); test('getPolicy returns null for unknown service', () => { const { manager } = makeManager(); expect(manager.getPolicy('does-not-exist')).toBeNull(); }); test('listPolicies returns array of all policies', async () => { const { manager } = makeManager(); await manager.setPolicy('svc-1', { maxRetries: 1 }); await manager.setPolicy('svc-2', { maxRetries: 2 }); const list = manager.listPolicies(); expect(Array.isArray(list)).toBe(true); expect(list).toHaveLength(2); const ids = list.map(p => p.serviceId); expect(ids).toEqual(expect.arrayContaining(['svc-1', 'svc-2'])); }); test('removePolicy returns true and deletes the policy', async () => { const { manager } = makeManager(); await manager.setPolicy('svc-1', { maxRetries: 1 }); expect(await manager.removePolicy('svc-1')).toBe(true); expect(manager.getPolicy('svc-1')).toBeNull(); }); test('removePolicy returns false for unknown service', async () => { const { manager } = makeManager(); expect(await manager.removePolicy('does-not-exist')).toBe(false); }); }); describe('handleContainerDown', () => { test('returns ignored/no-policy when no policy exists', async () => { const { manager } = makeManager(); const result = await manager.handleContainerDown('unknown', 'cid'); expect(result.action).toBe('ignored'); expect(result.reason).toBe('no-policy'); }); test('returns ignored/disabled when policy.enabled is false', async () => { const { manager } = makeManager(); await manager.setPolicy('svc-1', { enabled: false }); const result = await manager.handleContainerDown('svc-1', 'cid'); expect(result.action).toBe('ignored'); expect(result.reason).toBe('disabled'); }); test('returns skipped/cooldown when cooldownUntil is in the future', async () => { const { manager } = makeManager(); // setPolicy() intentionally guards runtime fields; we have to set // cooldownUntil via the internal map to simulate an in-progress cooldown await manager.setPolicy('svc-1', { maxRetries: 3 }); manager.policies.get('svc-1').cooldownUntil = Date.now() + 60_000; const result = await manager.handleContainerDown('svc-1', 'cid'); expect(result.action).toBe('skipped'); expect(result.reason).toBe('cooldown'); }); test('increments currentRetries and calls docker.start on a successful restart', async () => { const { manager, docker } = makeManager(); docker.client.getContainer.mockReturnValue({ start: jest.fn().mockResolvedValue(undefined), }); await manager.setPolicy('svc-1', { maxRetries: 3, retryIntervalMs: 0 }); const onAttempt = jest.fn(); const onSuccess = jest.fn(); manager.on('auto-restart-attempt', onAttempt); manager.on('auto-restart-success', onSuccess); const result = await manager.handleContainerDown('svc-1', 'cid-abc'); expect(result.action).toBe('restarted'); expect(result.attempt).toBe(1); expect(result.serviceId).toBe('svc-1'); expect(docker.client.getContainer).toHaveBeenCalledWith('cid-abc'); expect(onAttempt).toHaveBeenCalledTimes(1); expect(onSuccess).toHaveBeenCalledTimes(1); expect(manager.getPolicy('svc-1').currentRetries).toBe(1); }); test('emits auto-restart-failed and increments currentRetries when docker.start throws', async () => { const { manager, docker } = makeManager(); docker.client.getContainer.mockReturnValue({ start: jest.fn().mockRejectedValue(new Error('docker daemon down')), }); await manager.setPolicy('svc-1', { maxRetries: 3, retryIntervalMs: 0 }); const onFailed = jest.fn(); manager.on('auto-restart-failed', onFailed); const result = await manager.handleContainerDown('svc-1', 'cid-abc'); expect(result.action).toBe('failed'); expect(result.error).toMatch(/docker daemon down/); expect(onFailed).toHaveBeenCalledTimes(1); expect(manager.getPolicy('svc-1').currentRetries).toBe(1); }); test('emits auto-restart-max-reached and sets cooldown when maxRetries exceeded', async () => { const { manager, docker } = makeManager(); docker.client.getContainer.mockReturnValue({ start: jest.fn().mockResolvedValue(undefined), }); await manager.setPolicy('svc-1', { maxRetries: 2, retryIntervalMs: 0 }); const onMax = jest.fn(); manager.on('auto-restart-max-reached', onMax); // First attempt: currentRetries=0 -> succeeds, increments to 1 await manager.handleContainerDown('svc-1', 'cid'); // Second: 1 -> succeeds, increments to 2 await manager.handleContainerDown('svc-1', 'cid'); // Third: 2 >= maxRetries(2) -> max-reached, currentRetries reset to 0 const result = await manager.handleContainerDown('svc-1', 'cid'); expect(result.action).toBe('max-reached'); expect(onMax).toHaveBeenCalledTimes(1); const policy = manager.getPolicy('svc-1'); expect(policy.currentRetries).toBe(0); expect(policy.cooldownUntil).toBeGreaterThan(Date.now()); }); }); describe('handleContainerUp', () => { test('resets currentRetries and cooldownUntil when service is tracked', async () => { const { manager } = makeManager(); await manager.setPolicy('svc-1', { currentRetries: 2, cooldownUntil: Date.now() + 10000 }); // Mutate via internal map (bypassing the setter guard) manager.policies.get('svc-1').currentRetries = 2; manager.policies.get('svc-1').cooldownUntil = Date.now() + 10000; await manager.handleContainerUp('svc-1'); const policy = manager.getPolicy('svc-1'); expect(policy.currentRetries).toBe(0); expect(policy.cooldownUntil).toBeNull(); }); test('is a no-op when service is not tracked', async () => { const { manager } = makeManager(); await expect(manager.handleContainerUp('unknown')).resolves.toBeUndefined(); }); }); describe('_handleStatusCheck', () => { test('triggers handleContainerDown on healthy→unhealthy transition', async () => { const { manager, docker } = makeManager(); docker.client.getContainer.mockReturnValue({ start: jest.fn().mockResolvedValue(undefined), }); await manager.setPolicy('svc-1', { maxRetries: 3, retryIntervalMs: 0 }); // Pre-set previous health manager._previousHealth.set('svc-1', 'up'); const handleDownSpy = jest.spyOn(manager, 'handleContainerDown'); await manager._handleStatusCheck({ serviceId: 'svc-1', status: 'down', details: { containerId: 'cid-1' }, }); expect(handleDownSpy).toHaveBeenCalledWith('svc-1', 'cid-1'); }); test('triggers handleContainerUp on unhealthy→healthy transition', async () => { const { manager } = makeManager(); await manager.setPolicy('svc-1', { maxRetries: 3, retryIntervalMs: 0 }); manager._previousHealth.set('svc-1', 'down'); const handleUpSpy = jest.spyOn(manager, 'handleContainerUp'); await manager._handleStatusCheck({ serviceId: 'svc-1', status: 'up' }); expect(handleUpSpy).toHaveBeenCalledWith('svc-1'); }); test('does nothing for services without a policy', async () => { const { manager } = makeManager(); const handleDownSpy = jest.spyOn(manager, 'handleContainerDown'); const handleUpSpy = jest.spyOn(manager, 'handleContainerUp'); await manager._handleStatusCheck({ serviceId: 'untracked', status: 'down' }); expect(handleDownSpy).not.toHaveBeenCalled(); expect(handleUpSpy).not.toHaveBeenCalled(); }); test('ignores status with no serviceId', async () => { const { manager } = makeManager(); const handleDownSpy = jest.spyOn(manager, 'handleContainerDown'); await manager._handleStatusCheck({ status: 'down' }); expect(handleDownSpy).not.toHaveBeenCalled(); }); }); describe('_resolveContainerId', () => { test('returns containerId from status.details when present', async () => { const { manager } = makeManager(); const cid = await manager._resolveContainerId('svc-1', { details: { containerId: 'cid-details' } }); expect(cid).toBe('cid-details'); }); test('falls back to healthChecker.config.services[serviceId].containerId', async () => { const { manager, healthChecker } = makeManager(); healthChecker.config = { services: { 'svc-1': { containerId: 'cid-hc' } } }; const cid = await manager._resolveContainerId('svc-1', { details: {} }); expect(cid).toBe('cid-hc'); }); test('DC-060: awaits async servicesStateManager.read() and resolves containerId', async () => { // Regression test for the auto-restart silently no-op bug: // _resolveContainerId used to fire servicesStateManager.read() via // .then(...) and discard the result. Callers gated on the return // value, so a healthy→unhealthy transition whose only containerId // source was the async state manager never triggered handleContainerDown. const { manager, servicesStateManager } = makeManager(); servicesStateManager.read.mockResolvedValue([ { id: 'svc-1', containerId: 'cid-state' }, ]); const cid = await manager._resolveContainerId('svc-1', { details: {} }); expect(cid).toBe('cid-state'); }); test('returns null when no source has a containerId', async () => { const { manager } = makeManager(); const cid = await manager._resolveContainerId('svc-unknown', { details: {} }); expect(cid).toBeNull(); }); test('swallows servicesStateManager.read() rejection', async () => { const { manager, servicesStateManager } = makeManager(); servicesStateManager.read.mockRejectedValue(new Error('disk gone')); const cid = await manager._resolveContainerId('svc-1', { details: {} }); expect(cid).toBeNull(); }); }); describe('DC-060: healthy→unhealthy transitions trigger restart via async lookup', () => { test('handleContainerDown is invoked with containerId from async state-manager lookup', async () => { // End-to-end: containerId comes ONLY from servicesStateManager.read() // (the production path for services.json-backed deployments). const { manager, docker, servicesStateManager } = makeManager(); docker.client.getContainer.mockReturnValue({ start: jest.fn().mockResolvedValue(undefined), }); await manager.setPolicy('svc-1', { maxRetries: 3, retryIntervalMs: 0 }); manager._previousHealth.set('svc-1', 'up'); servicesStateManager.read.mockResolvedValue([ { id: 'svc-1', containerId: 'cid-from-state' }, ]); const handleDownSpy = jest.spyOn(manager, 'handleContainerDown'); await manager._handleStatusCheck({ serviceId: 'svc-1', status: 'down' }); expect(handleDownSpy).toHaveBeenCalledWith('svc-1', 'cid-from-state'); }); test('handleContainerDown is NOT invoked when async lookup returns no containerId', async () => { const { manager, servicesStateManager } = makeManager(); await manager.setPolicy('svc-1', { maxRetries: 3, retryIntervalMs: 0 }); manager._previousHealth.set('svc-1', 'up'); servicesStateManager.read.mockResolvedValue([ { id: 'svc-1' /* no containerId */ }, ]); const handleDownSpy = jest.spyOn(manager, 'handleContainerDown'); await manager._handleStatusCheck({ serviceId: 'svc-1', status: 'down' }); expect(handleDownSpy).not.toHaveBeenCalled(); }); }); });