DC-005: Fix all 138 broken test paths after src/ refactor
After the DC-005 module reorganization (41 files moved into src/ subdirs),
138 test suites failed because the refactor script's path-rewrite logic
missed three categories:
1. Files inside src/ doing 'require("./src/...")' — should be 'require("../...")'
2. Files in src/X/Y/ doing 'require("../../../src/...")' — should be 'require("../../...")'
3. Test files in __tests__/ with leftover 'require("../../../src/...")' paths
Root cause: the original refactor script ran before all files were moved,
so it computed relative paths against stale filesystem state.
Result:
- 30/30 test suites pass
- 879/879 tests pass (was: 18/30 suites, 614/687 tests)
Also fixed:
- routes/apps/restore.js: wrong responses import path
- routes/*/*.js: '../../src/utilities/X' → '../src/utilities/X' (depth 2 routes)
This commit is contained in:
Hermes
@@ -0,0 +1,302 @@
|
||||
/**
|
||||
* Authentication Manager for DashCaddy
|
||||
* Handles JWT tokens and API key generation/validation
|
||||
* Provides defense-in-depth alongside Caddy forward_auth
|
||||
*/
|
||||
|
||||
const jwt = require('jsonwebtoken');
|
||||
const crypto = require('crypto');
|
||||
const credentialManager = require('./credential-manager');
|
||||
const cryptoUtils = require('../security/crypto-utils');
|
||||
|
||||
// JWT signing secret - derived from encryption key for consistency
|
||||
const JWT_SECRET = cryptoUtils.loadOrCreateKey();
|
||||
|
||||
// Namespace for API keys in credential manager
|
||||
const API_KEY_NAMESPACE = 'auth.apikey';
|
||||
const API_KEY_METADATA_NAMESPACE = 'auth.metadata';
|
||||
|
||||
class AuthManager {
|
||||
constructor() {
|
||||
this.keyMetadataCache = new Map(); // Cache for API key metadata
|
||||
console.log('[AuthManager] Initialized');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate JWT token
|
||||
* @param {Object} payload - Token payload (must include sub: userId)
|
||||
* @param {string} expiresIn - Expiration time (default: '24h')
|
||||
* @returns {Promise<string>} JWT token
|
||||
*/
|
||||
async generateJWT(payload, expiresIn = '24h') {
|
||||
try {
|
||||
if (!payload.sub) {
|
||||
throw new Error('JWT payload must include "sub" (subject/userId)');
|
||||
}
|
||||
|
||||
const token = jwt.sign(
|
||||
{
|
||||
...payload,
|
||||
iat: Math.floor(Date.now() / 1000),
|
||||
scope: payload.scope || ['read', 'write']
|
||||
},
|
||||
JWT_SECRET,
|
||||
{ expiresIn }
|
||||
);
|
||||
|
||||
console.log(`[AuthManager] Generated JWT for user: ${payload.sub}, expires in: ${expiresIn}`);
|
||||
return token;
|
||||
} catch (error) {
|
||||
console.error('[AuthManager] JWT generation failed:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify JWT token
|
||||
* @param {string} token - JWT token to verify
|
||||
* @returns {Promise<Object|null>} Decoded payload or null if invalid
|
||||
*/
|
||||
async verifyJWT(token) {
|
||||
try {
|
||||
const decoded = jwt.verify(token, JWT_SECRET);
|
||||
return {
|
||||
userId: decoded.sub,
|
||||
scope: decoded.scope || [],
|
||||
iat: decoded.iat,
|
||||
exp: decoded.exp
|
||||
};
|
||||
} catch (error) {
|
||||
if (error.name === 'TokenExpiredError') {
|
||||
console.log('[AuthManager] JWT token expired');
|
||||
} else if (error.name === 'JsonWebTokenError') {
|
||||
console.log('[AuthManager] JWT token invalid:', error.message);
|
||||
} else {
|
||||
console.error('[AuthManager] JWT verification failed:', error.message);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate API key
|
||||
* @param {string} name - Human-readable name for the key
|
||||
* @param {Array<string>} scopes - Permission scopes (default: ['read', 'write'])
|
||||
* @returns {Promise<Object>} { key, id, name, scopes, createdAt }
|
||||
*/
|
||||
async generateAPIKey(name, scopes = ['read', 'write']) {
|
||||
try {
|
||||
if (!name || typeof name !== 'string') {
|
||||
throw new Error('API key name is required');
|
||||
}
|
||||
|
||||
// Generate secure random key (32 bytes = 64 hex chars)
|
||||
const keyId = crypto.randomBytes(16).toString('hex');
|
||||
const keySecret = crypto.randomBytes(32).toString('hex');
|
||||
const apiKey = `dk_${keyId}_${keySecret}`; // dk = DashCaddy Key
|
||||
|
||||
// Store key hash (not the key itself) in credential manager
|
||||
const keyHash = crypto.createHash('sha256').update(apiKey).digest('hex');
|
||||
const credentialKey = `${API_KEY_NAMESPACE}.${keyId}`;
|
||||
|
||||
await credentialManager.store(credentialKey, keyHash);
|
||||
|
||||
// Store metadata separately (non-sensitive)
|
||||
const metadata = {
|
||||
id: keyId,
|
||||
name,
|
||||
scopes,
|
||||
createdAt: new Date().toISOString(),
|
||||
lastUsed: null
|
||||
};
|
||||
|
||||
const metadataKey = `${API_KEY_METADATA_NAMESPACE}.${keyId}`;
|
||||
await credentialManager.store(metadataKey, JSON.stringify(metadata));
|
||||
|
||||
// Cache metadata
|
||||
this.keyMetadataCache.set(keyId, metadata);
|
||||
|
||||
console.log(`[AuthManager] Generated API key: ${name} (${keyId})`);
|
||||
|
||||
return {
|
||||
key: apiKey,
|
||||
id: keyId,
|
||||
name,
|
||||
scopes,
|
||||
createdAt: metadata.createdAt
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('[AuthManager] API key generation failed:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify API key
|
||||
* @param {string} key - API key to verify
|
||||
* @returns {Promise<Object|null>} { keyId, scopes, name } or null if invalid
|
||||
*/
|
||||
async verifyAPIKey(key) {
|
||||
try {
|
||||
// Parse key format: dk_<keyId>_<secret>
|
||||
if (!key || !key.startsWith('dk_')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const parts = key.split('_');
|
||||
if (parts.length !== 3) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const keyId = parts[1];
|
||||
const credentialKey = `${API_KEY_NAMESPACE}.${keyId}`;
|
||||
|
||||
// Retrieve stored hash
|
||||
const storedHash = await credentialManager.retrieve(credentialKey);
|
||||
if (!storedHash) {
|
||||
console.log(`[AuthManager] API key not found: ${keyId}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Verify key matches stored hash
|
||||
const providedHash = crypto.createHash('sha256').update(key).digest('hex');
|
||||
if (!crypto.timingSafeEqual(Buffer.from(storedHash), Buffer.from(providedHash))) {
|
||||
console.log(`[AuthManager] API key hash mismatch: ${keyId}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Get metadata
|
||||
const metadata = await this.getKeyMetadata(keyId);
|
||||
if (!metadata) {
|
||||
console.log(`[AuthManager] API key metadata not found: ${keyId}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Update last used timestamp (non-blocking)
|
||||
this.updateLastUsed(keyId, metadata).catch(err =>
|
||||
console.error(`[AuthManager] Failed to update lastUsed for ${keyId}:`, err.message)
|
||||
);
|
||||
|
||||
console.log(`[AuthManager] API key verified: ${metadata.name} (${keyId})`);
|
||||
|
||||
return {
|
||||
keyId,
|
||||
scopes: metadata.scopes || [],
|
||||
name: metadata.name
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('[AuthManager] API key verification failed:', error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Revoke API key
|
||||
* @param {string} keyId - Key ID to revoke
|
||||
* @returns {Promise<boolean>} Success status
|
||||
*/
|
||||
async revokeAPIKey(keyId) {
|
||||
try {
|
||||
const credentialKey = `${API_KEY_NAMESPACE}.${keyId}`;
|
||||
const metadataKey = `${API_KEY_METADATA_NAMESPACE}.${keyId}`;
|
||||
|
||||
await credentialManager.delete(credentialKey);
|
||||
await credentialManager.delete(metadataKey);
|
||||
|
||||
this.keyMetadataCache.delete(keyId);
|
||||
|
||||
console.log(`[AuthManager] Revoked API key: ${keyId}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error(`[AuthManager] Failed to revoke API key ${keyId}:`, error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List all API keys (returns metadata, not actual keys)
|
||||
* @returns {Promise<Array<Object>>} Array of API key metadata
|
||||
*/
|
||||
async listAPIKeys() {
|
||||
try {
|
||||
const allKeys = await credentialManager.list();
|
||||
const metadataKeys = allKeys.filter(k => k.startsWith(API_KEY_METADATA_NAMESPACE));
|
||||
|
||||
const keys = [];
|
||||
for (const metaKey of metadataKeys) {
|
||||
const keyId = metaKey.replace(`${API_KEY_METADATA_NAMESPACE}.`, '');
|
||||
const metadata = await this.getKeyMetadata(keyId);
|
||||
if (metadata) {
|
||||
keys.push(metadata);
|
||||
}
|
||||
}
|
||||
|
||||
return keys;
|
||||
} catch (error) {
|
||||
console.error('[AuthManager] Failed to list API keys:', error.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata for a specific API key
|
||||
* @param {string} keyId - Key ID
|
||||
* @returns {Promise<Object|null>} Metadata or null
|
||||
*/
|
||||
async getKeyMetadata(keyId) {
|
||||
try {
|
||||
// Check cache first
|
||||
if (this.keyMetadataCache.has(keyId)) {
|
||||
return this.keyMetadataCache.get(keyId);
|
||||
}
|
||||
|
||||
const metadataKey = `${API_KEY_METADATA_NAMESPACE}.${keyId}`;
|
||||
const metadataJson = await credentialManager.retrieve(metadataKey);
|
||||
|
||||
if (!metadataJson) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const metadata = JSON.parse(metadataJson);
|
||||
this.keyMetadataCache.set(keyId, metadata);
|
||||
|
||||
return metadata;
|
||||
} catch (error) {
|
||||
console.error(`[AuthManager] Failed to get metadata for ${keyId}:`, error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update last used timestamp for API key
|
||||
* @param {string} keyId - Key ID
|
||||
* @param {Object} metadata - Current metadata
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async updateLastUsed(keyId, metadata) {
|
||||
try {
|
||||
const updatedMetadata = {
|
||||
...metadata,
|
||||
lastUsed: new Date().toISOString()
|
||||
};
|
||||
|
||||
const metadataKey = `${API_KEY_METADATA_NAMESPACE}.${keyId}`;
|
||||
await credentialManager.store(metadataKey, JSON.stringify(updatedMetadata));
|
||||
|
||||
this.keyMetadataCache.set(keyId, updatedMetadata);
|
||||
} catch (error) {
|
||||
console.error(`[AuthManager] Failed to update lastUsed for ${keyId}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear metadata cache (useful for testing or cache invalidation)
|
||||
*/
|
||||
clearCache() {
|
||||
this.keyMetadataCache.clear();
|
||||
console.log('[AuthManager] Cache cleared');
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
module.exports = new AuthManager();
|
||||
@@ -0,0 +1,503 @@
|
||||
/**
|
||||
* Auto-Restart Manager - Per-container restart policies with retry tracking
|
||||
*
|
||||
* When a container goes down, attempts automatic restart up to N times
|
||||
* (configurable per-service). Sends notifications on each attempt and
|
||||
* when max retries are exceeded. Integrates with HealthChecker events.
|
||||
*
|
||||
* @module auto-restart-manager
|
||||
*/
|
||||
|
||||
const EventEmitter = require('events');
|
||||
const path = require('path');
|
||||
const { readJsonFile, writeJsonFile } = require('../utilities/fs-helpers');
|
||||
|
||||
/**
|
||||
* Default policy values applied when a new policy is created.
|
||||
* @readonly
|
||||
*/
|
||||
const DEFAULT_POLICY = {
|
||||
enabled: true,
|
||||
maxRetries: 3,
|
||||
retryIntervalMs: 5000,
|
||||
windowMinutes: 10,
|
||||
currentRetries: 0,
|
||||
lastRestartAt: null,
|
||||
cooldownUntil: null,
|
||||
};
|
||||
|
||||
/**
|
||||
* Manages automatic container restart policies and execution.
|
||||
*
|
||||
* @extends EventEmitter
|
||||
*
|
||||
* @fires AutoRestartManager#auto-restart-attempt
|
||||
* @fires AutoRestartManager#auto-restart-success
|
||||
* @fires AutoRestartManager#auto-restart-failed
|
||||
* @fires AutoRestartManager#auto-restart-max-reached
|
||||
*/
|
||||
class AutoRestartManager extends EventEmitter {
|
||||
/**
|
||||
* @param {Object} ctx - Shared application context
|
||||
* @param {Object} ctx.docker - Docker client wrapper ({ client: Dockerode })
|
||||
* @param {Object} ctx.healthChecker - HealthChecker singleton
|
||||
* @param {Object} ctx.notification - NotificationManager instance
|
||||
* @param {Object} ctx.log - Logger instance
|
||||
* @param {Function} ctx.logError - Error logging function
|
||||
* @param {string} ctx.SERVICES_FILE - Path to services.json (used to derive data dir)
|
||||
*/
|
||||
constructor(ctx) {
|
||||
super();
|
||||
this.ctx = ctx;
|
||||
this.log = ctx.log || console;
|
||||
this.logError = ctx.logError || ((_ctx, err) => console.error(err));
|
||||
this.docker = ctx.docker;
|
||||
this.healthChecker = ctx.healthChecker;
|
||||
this.notification = ctx.notification;
|
||||
|
||||
/** @type {Map<string, Object>} serviceId -> policy */
|
||||
this.policies = new Map();
|
||||
|
||||
/** Path to the JSON file that persists policies */
|
||||
this.policiesFile = path.join(path.dirname(ctx.SERVICES_FILE), 'auto-restart-policies.json');
|
||||
|
||||
/** Track previous health status per service for transition detection */
|
||||
this._previousHealth = new Map();
|
||||
|
||||
/** Bound handlers so we can remove them on stop() */
|
||||
this._onStatusCheck = this._handleStatusCheck.bind(this);
|
||||
this._started = false;
|
||||
}
|
||||
|
||||
// ─── Lifecycle ────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Load persisted policies, then wire into HealthChecker events.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async start() {
|
||||
if (this._started) return;
|
||||
|
||||
// Load persisted policies from disk
|
||||
try {
|
||||
const data = await readJsonFile(this.policiesFile, {});
|
||||
for (const [serviceId, policy] of Object.entries(data)) {
|
||||
this.policies.set(serviceId, { ...DEFAULT_POLICY, ...policy });
|
||||
}
|
||||
this.log.info('auto-restart', 'Policies loaded', { count: this.policies.size });
|
||||
} catch (err) {
|
||||
this.log.error('auto-restart', 'Failed to load policies', { error: err.message });
|
||||
}
|
||||
|
||||
// Listen to health checker status transitions
|
||||
if (this.healthChecker) {
|
||||
this.healthChecker.on('status-check', this._onStatusCheck);
|
||||
}
|
||||
|
||||
this._started = true;
|
||||
this.log.info('auto-restart', 'Manager started');
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove event listeners and stop processing health events.
|
||||
*/
|
||||
stop() {
|
||||
if (!this._started) return;
|
||||
|
||||
if (this.healthChecker) {
|
||||
this.healthChecker.removeListener('status-check', this._onStatusCheck);
|
||||
}
|
||||
|
||||
this._started = false;
|
||||
this.log.info('auto-restart', 'Manager stopped');
|
||||
}
|
||||
|
||||
// ─── Policy CRUD ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Create or update a restart policy for a service.
|
||||
*
|
||||
* @param {string} serviceId - Unique service identifier
|
||||
* @param {Object} policy - Partial policy fields to merge
|
||||
* @param {boolean} [policy.enabled=true]
|
||||
* @param {number} [policy.maxRetries=3]
|
||||
* @param {number} [policy.retryIntervalMs=5000]
|
||||
* @param {number} [policy.windowMinutes=10]
|
||||
* @returns {Promise<Object>} The resulting policy
|
||||
* @throws {Error} If serviceId is invalid
|
||||
*/
|
||||
async setPolicy(serviceId, policy) {
|
||||
if (!serviceId || typeof serviceId !== 'string') {
|
||||
throw new Error('serviceId is required');
|
||||
}
|
||||
|
||||
const existing = this.policies.get(serviceId) || { ...DEFAULT_POLICY, serviceId };
|
||||
|
||||
const merged = {
|
||||
...existing,
|
||||
...policy,
|
||||
serviceId,
|
||||
// Never allow caller to override runtime counters directly
|
||||
currentRetries: existing.currentRetries || 0,
|
||||
lastRestartAt: existing.lastRestartAt,
|
||||
cooldownUntil: existing.cooldownUntil,
|
||||
};
|
||||
|
||||
this.policies.set(serviceId, merged);
|
||||
await this._savePolicies();
|
||||
|
||||
this.log.info('auto-restart', 'Policy set', { serviceId, enabled: merged.enabled });
|
||||
return { ...merged };
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the policy for a service.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Object|null} Policy object or null if none exists
|
||||
*/
|
||||
getPolicy(serviceId) {
|
||||
const policy = this.policies.get(serviceId);
|
||||
return policy ? { ...policy } : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return all policies as an array.
|
||||
* @returns {Object[]}
|
||||
*/
|
||||
listPolicies() {
|
||||
return Array.from(this.policies.values()).map(p => ({ ...p }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a service's restart policy.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<boolean>} true if a policy was removed
|
||||
*/
|
||||
async removePolicy(serviceId) {
|
||||
if (!this.policies.has(serviceId)) return false;
|
||||
|
||||
this.policies.delete(serviceId);
|
||||
await this._savePolicies();
|
||||
|
||||
this.log.info('auto-restart', 'Policy removed', { serviceId });
|
||||
return true;
|
||||
}
|
||||
|
||||
// ─── Core Restart Logic ──────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Called when a container is detected as down.
|
||||
*
|
||||
* Checks policy, cooldown, and retry count, then either attempts a
|
||||
* Docker restart or notifies that max retries were exceeded.
|
||||
*
|
||||
* @param {string} serviceId - Service identifier
|
||||
* @param {string} containerId - Docker container ID to restart
|
||||
* @returns {Promise<Object>} Result of the operation
|
||||
*/
|
||||
async handleContainerDown(serviceId, containerId) {
|
||||
const policy = this.policies.get(serviceId);
|
||||
if (!policy) {
|
||||
return { action: 'ignored', reason: 'no-policy' };
|
||||
}
|
||||
|
||||
if (!policy.enabled) {
|
||||
return { action: 'ignored', reason: 'disabled' };
|
||||
}
|
||||
|
||||
// Check cooldown window
|
||||
const now = Date.now();
|
||||
if (policy.cooldownUntil && now < policy.cooldownUntil) {
|
||||
this.log.info('auto-restart', 'Skipping — cooldown active', {
|
||||
serviceId,
|
||||
cooldownUntil: new Date(policy.cooldownUntil).toISOString(),
|
||||
});
|
||||
return { action: 'skipped', reason: 'cooldown' };
|
||||
}
|
||||
|
||||
// Max retries exceeded — notify and enter cooldown
|
||||
if (policy.currentRetries >= policy.maxRetries) {
|
||||
const cooldownMs = policy.windowMinutes * 60 * 1000;
|
||||
policy.cooldownUntil = now + cooldownMs;
|
||||
policy.currentRetries = 0; // Reset so next window can try again
|
||||
await this._savePolicies();
|
||||
|
||||
const eventData = {
|
||||
serviceId,
|
||||
containerId,
|
||||
maxRetries: policy.maxRetries,
|
||||
cooldownUntil: policy.cooldownUntil,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-max-reached
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-max-reached', eventData);
|
||||
|
||||
// Send notification
|
||||
try {
|
||||
await this._notify('auto-restart', {
|
||||
containerName: serviceId,
|
||||
message: `⛔ Max auto-restart retries (${policy.maxRetries}) exceeded for "${serviceId}". Cooldown until ${new Date(policy.cooldownUntil).toISOString()}.`,
|
||||
...eventData,
|
||||
});
|
||||
} catch (notifErr) {
|
||||
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||
}
|
||||
|
||||
return { action: 'max-reached', ...eventData };
|
||||
}
|
||||
|
||||
// Wait for the configured retry interval before attempting
|
||||
if (policy.retryIntervalMs > 0 && policy.lastRestartAt) {
|
||||
const elapsed = now - new Date(policy.lastRestartAt).getTime();
|
||||
if (elapsed < policy.retryIntervalMs) {
|
||||
const waitMs = policy.retryIntervalMs - elapsed;
|
||||
this.log.info('auto-restart', 'Waiting for retry interval', { serviceId, waitMs });
|
||||
await new Promise(resolve => setTimeout(resolve, waitMs));
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt restart
|
||||
policy.currentRetries += 1;
|
||||
const attemptNum = policy.currentRetries;
|
||||
const maxRetries = policy.maxRetries;
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-attempt
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-attempt', {
|
||||
serviceId,
|
||||
containerId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
|
||||
try {
|
||||
if (!this.docker?.client) {
|
||||
throw new Error('Docker client not available');
|
||||
}
|
||||
|
||||
const container = this.docker.client.getContainer(containerId);
|
||||
await container.start();
|
||||
|
||||
policy.lastRestartAt = new Date().toISOString();
|
||||
await this._savePolicies();
|
||||
|
||||
const successData = {
|
||||
serviceId,
|
||||
containerId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-success
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-success', successData);
|
||||
|
||||
// Notify
|
||||
try {
|
||||
await this._notify('auto-restart', {
|
||||
containerName: serviceId,
|
||||
message: `🔄 Auto-restart attempt ${attemptNum}/${maxRetries} succeeded for "${serviceId}".`,
|
||||
...successData,
|
||||
});
|
||||
} catch (notifErr) {
|
||||
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||
}
|
||||
|
||||
this.log.info('auto-restart', 'Container restarted', {
|
||||
serviceId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
});
|
||||
|
||||
return { action: 'restarted', ...successData };
|
||||
} catch (restartErr) {
|
||||
policy.lastRestartAt = new Date().toISOString();
|
||||
await this._savePolicies();
|
||||
|
||||
const failData = {
|
||||
serviceId,
|
||||
containerId,
|
||||
attempt: attemptNum,
|
||||
maxRetries,
|
||||
error: restartErr.message,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
/**
|
||||
* @event AutoRestartManager#auto-restart-failed
|
||||
* @type {Object}
|
||||
*/
|
||||
this.emit('auto-restart-failed', failData);
|
||||
|
||||
// Notify
|
||||
try {
|
||||
await this._notify('auto-restart', {
|
||||
containerName: serviceId,
|
||||
message: `❌ Auto-restart attempt ${attemptNum}/${maxRetries} failed for "${serviceId}": ${restartErr.message}`,
|
||||
...failData,
|
||||
});
|
||||
} catch (notifErr) {
|
||||
this.log.error('auto-restart', 'Notification failed', { error: notifErr.message });
|
||||
}
|
||||
|
||||
this.log.error('auto-restart', 'Restart failed', {
|
||||
serviceId,
|
||||
attempt: attemptNum,
|
||||
error: restartErr.message,
|
||||
});
|
||||
|
||||
return { action: 'failed', ...failData };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when a container recovers to healthy state.
|
||||
* Resets the retry counter for the associated service.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async handleContainerUp(serviceId) {
|
||||
const policy = this.policies.get(serviceId);
|
||||
if (!policy) return;
|
||||
|
||||
if (policy.currentRetries > 0) {
|
||||
policy.currentRetries = 0;
|
||||
policy.cooldownUntil = null;
|
||||
await this._savePolicies();
|
||||
|
||||
this.log.info('auto-restart', 'Retries reset after recovery', { serviceId });
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Health Event Bridge ─────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Internal handler for HealthChecker `status-check` events.
|
||||
* Detects healthy→unhealthy and unhealthy→healthy transitions for tracked services.
|
||||
*
|
||||
* @param {Object} status - HealthChecker status object
|
||||
* @param {string} status.serviceId
|
||||
* @param {string} status.status - "up" or "down"
|
||||
* @private
|
||||
*/
|
||||
async _handleStatusCheck(status) {
|
||||
const { serviceId, status: currentStatus } = status;
|
||||
if (!serviceId) return;
|
||||
|
||||
// Only process services that have a restart policy
|
||||
if (!this.policies.has(serviceId)) return;
|
||||
|
||||
const previousStatus = this._previousHealth.get(serviceId);
|
||||
this._previousHealth.set(serviceId, currentStatus);
|
||||
|
||||
// Transition: healthy → unhealthy
|
||||
if (previousStatus === 'up' && currentStatus === 'down') {
|
||||
// Find the containerId from the health checker config or status details
|
||||
const containerId = this._resolveContainerId(serviceId, status);
|
||||
if (containerId) {
|
||||
try {
|
||||
await this.handleContainerDown(serviceId, containerId);
|
||||
} catch (err) {
|
||||
this.logError('auto-restart-health-bridge', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Transition: unhealthy → healthy (recovery)
|
||||
if (previousStatus === 'down' && currentStatus === 'up') {
|
||||
try {
|
||||
await this.handleContainerUp(serviceId);
|
||||
} catch (err) {
|
||||
this.logError('auto-restart-health-bridge', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to find the containerId for a service from various sources.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @param {Object} status - The status-check event data
|
||||
* @returns {string|null}
|
||||
* @private
|
||||
*/
|
||||
_resolveContainerId(serviceId, status) {
|
||||
// Check if it's in the status details (some health checks embed it)
|
||||
if (status.details?.containerId) return status.details.containerId;
|
||||
|
||||
// Look in the health checker config
|
||||
const hcService = this.healthChecker?.config?.services?.[serviceId];
|
||||
if (hcService?.containerId) return hcService.containerId;
|
||||
|
||||
// Try to look it up from the services state manager
|
||||
try {
|
||||
const servicesStateManager = this.ctx.servicesStateManager;
|
||||
if (servicesStateManager) {
|
||||
const readResult = servicesStateManager.read();
|
||||
if (readResult && typeof readResult.then === 'function') {
|
||||
// It returns a promise — fire-and-forget lookup
|
||||
readResult.then(list => {
|
||||
const found = (list || []).find(s => s.id === serviceId);
|
||||
return found?.containerId || null;
|
||||
}).catch(() => null);
|
||||
} else {
|
||||
const found = (readResult || []).find(s => s.id === serviceId);
|
||||
if (found?.containerId) return found.containerId;
|
||||
}
|
||||
}
|
||||
} catch (_) { /* best effort */ }
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── Persistence ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Persist current policies to disk.
|
||||
* @returns {Promise<void>}
|
||||
* @private
|
||||
*/
|
||||
async _savePolicies() {
|
||||
try {
|
||||
const obj = {};
|
||||
for (const [serviceId, policy] of this.policies.entries()) {
|
||||
obj[serviceId] = { ...policy };
|
||||
}
|
||||
await writeJsonFile(this.policiesFile, obj);
|
||||
} catch (err) {
|
||||
this.log.error('auto-restart', 'Failed to save policies', { error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Send a notification via the notification manager.
|
||||
*
|
||||
* @param {string} event - Event type (e.g. 'auto-restart')
|
||||
* @param {Object} data - Notification payload
|
||||
* @returns {Promise<Object>}
|
||||
* @private
|
||||
*/
|
||||
async _notify(event, data) {
|
||||
if (this.notification?.send) {
|
||||
return this.notification.send(event, data);
|
||||
}
|
||||
return { success: false, reason: 'no-notification-manager' };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { AutoRestartManager, DEFAULT_POLICY };
|
||||
@@ -0,0 +1,376 @@
|
||||
/**
|
||||
* Config Drift Detector - Compares services.json with live Docker state
|
||||
*
|
||||
* Detects discrepancies between the configured service list and what is
|
||||
* actually running in Docker, including missing containers, unknown
|
||||
* containers, port mismatches, state mismatches, and stale records.
|
||||
*
|
||||
* @module config-drift-detector
|
||||
*/
|
||||
|
||||
const EventEmitter = require('events');
|
||||
|
||||
/**
|
||||
* @typedef {Object} DriftReport
|
||||
* @property {string} checkedAt - ISO timestamp of the check
|
||||
* @property {Object[]} missingContainers - Services with containerId but container absent in Docker
|
||||
* @property {Object[]} unknownContainers - Running Docker containers with sami.managed label but not in services.json
|
||||
* @property {Object[]} portMismatch - Service port != container mapped port
|
||||
* @property {Object[]} stateMismatch - Service expected up but container stopped/absent
|
||||
* @property {Object[]} staleRecords - Services with containerId pointing to removed containers
|
||||
* @property {boolean} hasDrift - Whether any drift category is non-empty
|
||||
*/
|
||||
|
||||
/**
|
||||
* Detects and reports configuration drift between services.json and Docker.
|
||||
*
|
||||
* @extends EventEmitter
|
||||
*
|
||||
* @fires ConfigDriftDetector#drift-detected
|
||||
*/
|
||||
class ConfigDriftDetector extends EventEmitter {
|
||||
/**
|
||||
* @param {Object} ctx - Shared application context
|
||||
* @param {Object} ctx.docker - Docker client wrapper ({ client: Dockerode })
|
||||
* @param {Object} ctx.servicesStateManager - StateManager for services.json
|
||||
* @param {Object} ctx.notification - NotificationManager instance
|
||||
* @param {Object} ctx.log - Logger instance
|
||||
* @param {Function} ctx.logError - Error logging function
|
||||
*/
|
||||
constructor(ctx) {
|
||||
super();
|
||||
this.ctx = ctx;
|
||||
this.log = ctx.log || console;
|
||||
this.logError = ctx.logError || ((_c, err) => console.error(err));
|
||||
this.docker = ctx.docker;
|
||||
this.servicesStateManager = ctx.servicesStateManager;
|
||||
this.notification = ctx.notification;
|
||||
|
||||
/** @type {DriftReport|null} Cached report from last detection */
|
||||
this.lastReport = null;
|
||||
|
||||
/** @type {NodeJS.Timeout|null} Polling timer reference */
|
||||
this._pollTimer = null;
|
||||
|
||||
/** Whether polling is currently active */
|
||||
this._polling = false;
|
||||
}
|
||||
|
||||
// ─── Detection ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Run a full drift detection and return the report.
|
||||
*
|
||||
* Reads services from servicesStateManager and live containers from Docker,
|
||||
* then compares them across five drift categories.
|
||||
*
|
||||
* @returns {Promise<DriftReport>}
|
||||
*/
|
||||
async detect() {
|
||||
const checkedAt = new Date().toISOString();
|
||||
|
||||
// Gather configured services
|
||||
let services = [];
|
||||
try {
|
||||
const data = await this.servicesStateManager.read();
|
||||
services = Array.isArray(data) ? data : (data.services || []);
|
||||
} catch (err) {
|
||||
this.log.error('drift', 'Failed to read services', { error: err.message });
|
||||
}
|
||||
|
||||
// Gather live Docker containers
|
||||
let containers = [];
|
||||
try {
|
||||
containers = await this.docker.client.listContainers({ all: true });
|
||||
} catch (err) {
|
||||
this.log.error('drift', 'Failed to list containers', { error: err.message });
|
||||
}
|
||||
|
||||
// Build lookup maps
|
||||
const containerById = new Map(); // containerId (short or long) → container info
|
||||
const containerByName = new Map(); // container name → container info
|
||||
|
||||
for (const c of containers) {
|
||||
// Store by full ID
|
||||
containerById.set(c.Id, c);
|
||||
// Store by short ID (first 12 chars)
|
||||
if (c.Id && c.Id.length >= 12) {
|
||||
containerById.set(c.Id.substring(0, 12), c);
|
||||
}
|
||||
// Store by name (strip leading /)
|
||||
for (const name of (c.Names || [])) {
|
||||
containerByName.set(name.replace(/^\//, ''), c);
|
||||
}
|
||||
}
|
||||
|
||||
// Build set of service containerIds for reverse lookup
|
||||
const serviceContainerIds = new Set();
|
||||
const serviceByContainerId = new Map();
|
||||
|
||||
for (const svc of services) {
|
||||
if (svc.containerId) {
|
||||
serviceContainerIds.add(svc.containerId);
|
||||
// Index by both full and short ID
|
||||
serviceByContainerId.set(svc.containerId, svc);
|
||||
if (svc.containerId.length >= 12) {
|
||||
serviceByContainerId.set(svc.containerId.substring(0, 12), svc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const missingContainers = [];
|
||||
const portMismatch = [];
|
||||
const stateMismatch = [];
|
||||
const staleRecords = [];
|
||||
|
||||
for (const svc of services) {
|
||||
if (!svc.containerId) continue;
|
||||
|
||||
// Look up the container
|
||||
const container = containerById.get(svc.containerId)
|
||||
|| containerById.get(svc.containerId.substring(0, 12));
|
||||
|
||||
if (!container) {
|
||||
// Container ID referenced but not found in Docker at all
|
||||
staleRecords.push({
|
||||
serviceId: svc.id,
|
||||
name: svc.name,
|
||||
containerId: svc.containerId,
|
||||
reason: 'Container not found in Docker',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Missing container — service expects it but it's not running
|
||||
if (container.State !== 'running') {
|
||||
missingContainers.push({
|
||||
serviceId: svc.id,
|
||||
name: svc.name,
|
||||
containerId: svc.containerId,
|
||||
containerState: container.State,
|
||||
containerStatus: container.Status,
|
||||
});
|
||||
|
||||
// Also a state mismatch if the service is expected to be up
|
||||
stateMismatch.push({
|
||||
serviceId: svc.id,
|
||||
name: svc.name,
|
||||
expectedState: 'running',
|
||||
actualState: container.State,
|
||||
containerId: svc.containerId,
|
||||
});
|
||||
}
|
||||
|
||||
// Port mismatch detection
|
||||
if (svc.port && container.State === 'running') {
|
||||
const actualPorts = this._extractContainerPorts(container);
|
||||
if (actualPorts.length > 0 && !actualPorts.includes(svc.port)) {
|
||||
portMismatch.push({
|
||||
serviceId: svc.id,
|
||||
name: svc.name,
|
||||
configuredPort: svc.port,
|
||||
actualPorts,
|
||||
containerId: svc.containerId,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Unknown managed containers: Docker containers with sami.managed label
|
||||
// that are NOT in services.json
|
||||
const unknownContainers = [];
|
||||
for (const c of containers) {
|
||||
const isManaged = c.Labels && c.Labels['sami.managed'] === 'true';
|
||||
if (!isManaged) continue;
|
||||
|
||||
const isInServices = serviceByContainerId.has(c.Id)
|
||||
|| serviceByContainerId.has(c.Id.substring(0, 12));
|
||||
|
||||
if (!isInServices) {
|
||||
unknownContainers.push({
|
||||
containerId: c.Id,
|
||||
name: (c.Names && c.Names[0] || '').replace(/^\//, ''),
|
||||
image: c.Image,
|
||||
state: c.State,
|
||||
status: c.Status,
|
||||
app: c.Labels?.['sami.app'] || null,
|
||||
subdomain: c.Labels?.['sami.subdomain'] || null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const report = {
|
||||
checkedAt,
|
||||
missingContainers,
|
||||
unknownContainers,
|
||||
portMismatch,
|
||||
stateMismatch,
|
||||
staleRecords,
|
||||
hasDrift: missingContainers.length > 0
|
||||
|| unknownContainers.length > 0
|
||||
|| portMismatch.length > 0
|
||||
|| stateMismatch.length > 0
|
||||
|| staleRecords.length > 0,
|
||||
};
|
||||
|
||||
// Cache for quick API access
|
||||
this.lastReport = report;
|
||||
|
||||
// Emit and notify if drift detected
|
||||
if (report.hasDrift) {
|
||||
/**
|
||||
* @event ConfigDriftDetector#drift-detected
|
||||
* @type {DriftReport}
|
||||
*/
|
||||
this.emit('drift-detected', report);
|
||||
|
||||
try {
|
||||
await this._sendDriftNotification(report);
|
||||
} catch (notifErr) {
|
||||
this.log.error('drift', 'Failed to send drift notification', {
|
||||
error: notifErr.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
this.log.info('drift', 'Detection complete', {
|
||||
hasDrift: report.hasDrift,
|
||||
missing: report.missingContainers.length,
|
||||
unknown: report.unknownContainers.length,
|
||||
portMismatch: report.portMismatch.length,
|
||||
stateMismatch: report.stateMismatch.length,
|
||||
stale: report.staleRecords.length,
|
||||
});
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
// ─── Auto-fix ────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Attempt to auto-fix drift:
|
||||
* - Remove stale records (services referencing removed containers)
|
||||
* - Flag unknown containers for review
|
||||
*
|
||||
* @returns {Promise<{ staleRemoved: number, unknownFlagged: number }>}
|
||||
*/
|
||||
async autoFix() {
|
||||
const report = await this.detect();
|
||||
let staleRemoved = 0;
|
||||
|
||||
// Remove stale records from services.json
|
||||
if (report.staleRecords.length > 0) {
|
||||
const staleIds = new Set(report.staleRecords.map(r => r.serviceId));
|
||||
await this.servicesStateManager.update(services => {
|
||||
const before = services.length;
|
||||
const cleaned = services.filter(s => !staleIds.has(s.id));
|
||||
staleRemoved = before - cleaned.length;
|
||||
return cleaned;
|
||||
});
|
||||
}
|
||||
|
||||
const unknownFlagged = report.unknownContainers.length;
|
||||
|
||||
this.log.info('drift', 'Auto-fix applied', { staleRemoved, unknownFlagged });
|
||||
|
||||
return { staleRemoved, unknownFlagged };
|
||||
}
|
||||
|
||||
// ─── Polling ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Start periodic drift detection.
|
||||
*
|
||||
* @param {number} [intervalMs=300000] - Polling interval in milliseconds (default 5 min)
|
||||
*/
|
||||
startPolling(intervalMs = 300000) {
|
||||
this.stopPolling();
|
||||
|
||||
this._polling = true;
|
||||
this._pollTimer = setInterval(async () => {
|
||||
try {
|
||||
await this.detect();
|
||||
} catch (err) {
|
||||
this.logError('drift-poll', err);
|
||||
}
|
||||
}, intervalMs);
|
||||
|
||||
this.log.info('drift', 'Polling started', { intervalMs });
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop periodic drift detection.
|
||||
*/
|
||||
stopPolling() {
|
||||
if (this._pollTimer) {
|
||||
clearInterval(this._pollTimer);
|
||||
this._pollTimer = null;
|
||||
}
|
||||
this._polling = false;
|
||||
this.log.info('drift', 'Polling stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether polling is currently active.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isPolling() {
|
||||
return this._polling;
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Extract mapped host ports from a Docker container info object.
|
||||
*
|
||||
* @param {Object} container - Dockerode container info
|
||||
* @returns {number[]} Array of host port numbers
|
||||
* @private
|
||||
*/
|
||||
_extractContainerPorts(container) {
|
||||
const ports = [];
|
||||
if (!container.Ports) return ports;
|
||||
|
||||
for (const p of container.Ports) {
|
||||
if (p.PublicPort) {
|
||||
ports.push(p.PublicPort);
|
||||
}
|
||||
}
|
||||
|
||||
return ports;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a notification about detected drift.
|
||||
*
|
||||
* @param {DriftReport} report
|
||||
* @returns {Promise<Object>}
|
||||
* @private
|
||||
*/
|
||||
async _sendDriftNotification(report) {
|
||||
if (!this.notification?.send) {
|
||||
return { success: false, reason: 'no-notification-manager' };
|
||||
}
|
||||
|
||||
const parts = [];
|
||||
if (report.missingContainers.length > 0) {
|
||||
parts.push(`Missing containers: ${report.missingContainers.map(c => c.name).join(', ')}`);
|
||||
}
|
||||
if (report.unknownContainers.length > 0) {
|
||||
parts.push(`Unknown managed containers: ${report.unknownContainers.map(c => c.name).join(', ')}`);
|
||||
}
|
||||
if (report.portMismatch.length > 0) {
|
||||
parts.push(`Port mismatches: ${report.portMismatch.map(c => c.name).join(', ')}`);
|
||||
}
|
||||
if (report.staleRecords.length > 0) {
|
||||
parts.push(`Stale records: ${report.staleRecords.map(c => c.name).join(', ')}`);
|
||||
}
|
||||
|
||||
return this.notification.send('drift-detected', {
|
||||
text: `⚠️ Configuration drift detected:\n${parts.join('\n')}`,
|
||||
report,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { ConfigDriftDetector };
|
||||
@@ -0,0 +1,414 @@
|
||||
/**
|
||||
* Credential Manager for DashCaddy
|
||||
* Unified interface for secure credential storage
|
||||
* Uses OS keychain when available, falls back to encrypted file storage
|
||||
*/
|
||||
|
||||
const keychainManager = require('../security/keychain-manager');
|
||||
const cryptoUtils = require('../security/crypto-utils');
|
||||
const lockfile = require('proper-lockfile');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Resolve credentials file path — supports both standard install (/app/credentials.json)
|
||||
// and custom deployments with consolidated data directory (/app/data/credentials.json)
|
||||
function resolveCredentialsFile() {
|
||||
if (process.env.CREDENTIALS_FILE) {
|
||||
return process.env.CREDENTIALS_FILE;
|
||||
}
|
||||
const candidates = [
|
||||
path.join(__dirname, 'credentials.json'),
|
||||
path.join(__dirname, 'data', 'credentials.json'),
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (fs.existsSync(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
// No existing file — return standard path so first store() creates it there
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
const CREDENTIALS_FILE = resolveCredentialsFile();
|
||||
|
||||
class CredentialManager {
|
||||
constructor() {
|
||||
this.useKeychain = keychainManager.available;
|
||||
this.cache = new Map(); // In-memory cache with TTL
|
||||
this.CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
||||
this.lockOptions = {
|
||||
retries: { retries: 10, minTimeout: 100, maxTimeout: 300 },
|
||||
stale: 30000
|
||||
};
|
||||
|
||||
console.log(`[CredentialManager] Initialized with ${this.useKeychain ? 'OS keychain' : 'encrypted file'} storage`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a credential securely
|
||||
* @param {string} key - Credential identifier (e.g., 'dns.token', 'cloudflare.apikey')
|
||||
* @param {string} value - Credential value
|
||||
* @param {Object} metadata - Optional metadata (non-sensitive)
|
||||
* @returns {Promise<boolean>} Success status
|
||||
*/
|
||||
async store(key, value, metadata = {}) {
|
||||
try {
|
||||
// Validate inputs
|
||||
if (!key || typeof key !== 'string') {
|
||||
throw new Error('Credential key is required');
|
||||
}
|
||||
if (!value || typeof value !== 'string') {
|
||||
throw new Error('Credential value is required');
|
||||
}
|
||||
|
||||
// Try OS keychain first
|
||||
if (this.useKeychain) {
|
||||
const success = await keychainManager.store(key, value);
|
||||
if (success) {
|
||||
// Store metadata separately in file
|
||||
await this.storeMetadata(key, metadata);
|
||||
this.cache.set(key, { value, exp: Date.now() + this.CACHE_TTL_MS });
|
||||
console.log(`[CredentialManager] Stored '${key}' in OS keychain`);
|
||||
return true;
|
||||
}
|
||||
console.warn(`[CredentialManager] Keychain storage failed for '${key}', falling back to encrypted file`);
|
||||
}
|
||||
|
||||
// Fallback to encrypted file storage
|
||||
await this.storeInFile(key, value, metadata);
|
||||
this.cache.set(key, { value, exp: Date.now() + this.CACHE_TTL_MS });
|
||||
console.log(`[CredentialManager] Stored '${key}' in encrypted file`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error(`[CredentialManager] Failed to store '${key}':`, error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a credential
|
||||
* @param {string} key - Credential identifier
|
||||
* @returns {Promise<string|null>} Credential value or null
|
||||
*/
|
||||
async retrieve(key) {
|
||||
try {
|
||||
// Check cache first (with TTL expiration)
|
||||
if (this.cache.has(key)) {
|
||||
const cached = this.cache.get(key);
|
||||
if (Date.now() < cached.exp) {
|
||||
return cached.value;
|
||||
}
|
||||
this.cache.delete(key);
|
||||
}
|
||||
|
||||
// Try OS keychain first
|
||||
if (this.useKeychain) {
|
||||
const value = await keychainManager.retrieve(key);
|
||||
if (value) {
|
||||
this.cache.set(key, { value, exp: Date.now() + this.CACHE_TTL_MS });
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to encrypted file storage
|
||||
const value = await this.retrieveFromFile(key);
|
||||
if (value) {
|
||||
this.cache.set(key, { value, exp: Date.now() + this.CACHE_TTL_MS });
|
||||
}
|
||||
return value;
|
||||
} catch (error) {
|
||||
console.error(`[CredentialManager] Failed to retrieve '${key}':`, error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a credential
|
||||
* @param {string} key - Credential identifier
|
||||
* @returns {Promise<boolean>} Success status
|
||||
*/
|
||||
async delete(key) {
|
||||
try {
|
||||
// Remove from cache
|
||||
this.cache.delete(key);
|
||||
|
||||
// Try OS keychain
|
||||
if (this.useKeychain) {
|
||||
await keychainManager.delete(key);
|
||||
}
|
||||
|
||||
// Remove from file storage
|
||||
await this.deleteFromFile(key);
|
||||
|
||||
console.log(`[CredentialManager] Deleted '${key}'`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error(`[CredentialManager] Failed to delete '${key}':`, error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List all stored credential keys (not values)
|
||||
* @returns {Promise<Array<string>>} Array of credential keys
|
||||
*/
|
||||
async list() {
|
||||
try {
|
||||
const credentials = await this.loadCredentialsFile();
|
||||
return Object.keys(credentials);
|
||||
} catch (error) {
|
||||
console.error('[CredentialManager] Failed to list credentials:', error.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata for a credential
|
||||
* @param {string} key - Credential identifier
|
||||
* @returns {Promise<Object|null>} Metadata object or null
|
||||
*/
|
||||
async getMetadata(key) {
|
||||
try {
|
||||
const credentials = await this.loadCredentialsFile();
|
||||
return credentials[key]?.metadata || null;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rotate encryption key (re-encrypt all credentials with new key)
|
||||
* @returns {Promise<boolean>} Success status
|
||||
*/
|
||||
async rotateEncryptionKey() {
|
||||
let release;
|
||||
try {
|
||||
console.log('[CredentialManager] Starting encryption key rotation...');
|
||||
|
||||
// Ensure file exists before locking
|
||||
this._ensureFileExists();
|
||||
release = await lockfile.lock(CREDENTIALS_FILE, this.lockOptions);
|
||||
|
||||
const data = fs.readFileSync(CREDENTIALS_FILE, 'utf8');
|
||||
const credentials = JSON.parse(data);
|
||||
const keys = Object.keys(credentials);
|
||||
|
||||
if (keys.length === 0) {
|
||||
console.log('[CredentialManager] No credentials to rotate');
|
||||
return true;
|
||||
}
|
||||
|
||||
// Decrypt all values with the CURRENT key first
|
||||
const decryptedEntries = {};
|
||||
for (const key of keys) {
|
||||
const value = credentials[key].value;
|
||||
decryptedEntries[key] = {
|
||||
plaintext: cryptoUtils.isEncrypted(value) ? cryptoUtils.decrypt(value) : value,
|
||||
metadata: credentials[key].metadata
|
||||
};
|
||||
}
|
||||
|
||||
// Generate new key (this replaces the cached key and saves to disk)
|
||||
const { oldKey } = cryptoUtils.rotateKey();
|
||||
|
||||
// Re-encrypt all credentials with the new key
|
||||
const rotated = {};
|
||||
for (const key of keys) {
|
||||
rotated[key] = {
|
||||
value: cryptoUtils.encrypt(decryptedEntries[key].plaintext),
|
||||
metadata: decryptedEntries[key].metadata,
|
||||
rotatedAt: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
// Save with new encryption
|
||||
fs.writeFileSync(CREDENTIALS_FILE, JSON.stringify(rotated, null, 2), { mode: 0o600 });
|
||||
|
||||
// Clear cache to force reload
|
||||
this.cache.clear();
|
||||
|
||||
console.log(`[CredentialManager] Successfully rotated ${keys.length} credentials`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('[CredentialManager] Key rotation failed:', error.message);
|
||||
return false;
|
||||
} finally {
|
||||
if (release) {
|
||||
try { await release(); } catch (e) { /* lock will expire via stale timeout */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate plaintext credentials to encrypted format
|
||||
* @returns {Promise<Object>} Migration results
|
||||
*/
|
||||
async migrateToEncrypted() {
|
||||
try {
|
||||
let migrated = 0;
|
||||
let skipped = 0;
|
||||
|
||||
await this._lockedUpdate(credentials => {
|
||||
for (const [key, data] of Object.entries(credentials)) {
|
||||
if (!cryptoUtils.isEncrypted(data.value)) {
|
||||
credentials[key].value = cryptoUtils.encrypt(data.value);
|
||||
credentials[key].migratedAt = new Date().toISOString();
|
||||
migrated++;
|
||||
} else {
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
return credentials;
|
||||
});
|
||||
|
||||
if (migrated > 0) {
|
||||
this.cache.clear();
|
||||
console.log(`[CredentialManager] Migrated ${migrated} plaintext credentials to encrypted format`);
|
||||
}
|
||||
|
||||
return { migrated, skipped, total: migrated + skipped };
|
||||
} catch (error) {
|
||||
console.error('[CredentialManager] Migration failed:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Private methods
|
||||
|
||||
/**
|
||||
* Ensure credentials file exists (needed before locking)
|
||||
* @private
|
||||
*/
|
||||
_ensureFileExists() {
|
||||
if (!fs.existsSync(CREDENTIALS_FILE)) {
|
||||
const dir = path.dirname(CREDENTIALS_FILE);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(CREDENTIALS_FILE, '{}', { mode: 0o600 });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomic read-modify-write with file locking
|
||||
* @param {Function} updateFn - Receives current credentials object, returns updated object
|
||||
* @returns {Promise<Object>} Updated credentials
|
||||
* @private
|
||||
*/
|
||||
async _lockedUpdate(updateFn) {
|
||||
this._ensureFileExists();
|
||||
let release;
|
||||
try {
|
||||
release = await lockfile.lock(CREDENTIALS_FILE, this.lockOptions);
|
||||
const data = fs.readFileSync(CREDENTIALS_FILE, 'utf8');
|
||||
const credentials = JSON.parse(data);
|
||||
const updated = await updateFn(credentials);
|
||||
fs.writeFileSync(CREDENTIALS_FILE, JSON.stringify(updated, null, 2), { mode: 0o600 });
|
||||
return updated;
|
||||
} catch (error) {
|
||||
if (error.code === 'ELOCKED') {
|
||||
throw new Error('Credentials file is locked by another process. Try again.');
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
if (release) {
|
||||
try { await release(); } catch (e) { /* lock will expire via stale timeout */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async storeInFile(key, value, metadata) {
|
||||
await this._lockedUpdate(credentials => {
|
||||
credentials[key] = {
|
||||
value: cryptoUtils.encrypt(value),
|
||||
metadata,
|
||||
updatedAt: new Date().toISOString()
|
||||
};
|
||||
return credentials;
|
||||
});
|
||||
}
|
||||
|
||||
async retrieveFromFile(key) {
|
||||
const credentials = await this.loadCredentialsFile();
|
||||
const data = credentials[key];
|
||||
if (!data) return null;
|
||||
|
||||
return cryptoUtils.isEncrypted(data.value)
|
||||
? cryptoUtils.decrypt(data.value)
|
||||
: data.value;
|
||||
}
|
||||
|
||||
async deleteFromFile(key) {
|
||||
await this._lockedUpdate(credentials => {
|
||||
delete credentials[key];
|
||||
return credentials;
|
||||
});
|
||||
}
|
||||
|
||||
async storeMetadata(key, metadata) {
|
||||
await this._lockedUpdate(credentials => {
|
||||
if (!credentials[key]) {
|
||||
credentials[key] = { metadata };
|
||||
} else {
|
||||
credentials[key].metadata = metadata;
|
||||
}
|
||||
credentials[key].updatedAt = new Date().toISOString();
|
||||
return credentials;
|
||||
});
|
||||
}
|
||||
|
||||
async loadCredentialsFile() {
|
||||
try {
|
||||
if (!fs.existsSync(CREDENTIALS_FILE)) {
|
||||
return {};
|
||||
}
|
||||
const data = fs.readFileSync(CREDENTIALS_FILE, 'utf8');
|
||||
return JSON.parse(data);
|
||||
} catch (error) {
|
||||
console.error('[CredentialManager] Failed to load credentials file:', error.message);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export credentials for backup (encrypted)
|
||||
* @returns {Promise<string>} Encrypted backup data
|
||||
*/
|
||||
async exportBackup() {
|
||||
const credentials = await this.loadCredentialsFile();
|
||||
const backup = {
|
||||
version: '1.0',
|
||||
exportedAt: new Date().toISOString(),
|
||||
credentials
|
||||
};
|
||||
return cryptoUtils.encrypt(JSON.stringify(backup));
|
||||
}
|
||||
|
||||
/**
|
||||
* Import credentials from backup
|
||||
* @param {string} encryptedBackup - Encrypted backup data
|
||||
* @returns {Promise<boolean>} Success status
|
||||
*/
|
||||
async importBackup(encryptedBackup) {
|
||||
try {
|
||||
const decrypted = cryptoUtils.decrypt(encryptedBackup);
|
||||
const backup = JSON.parse(decrypted);
|
||||
|
||||
if (backup.version !== '1.0') {
|
||||
throw new Error('Unsupported backup version');
|
||||
}
|
||||
|
||||
await this._lockedUpdate(() => backup.credentials);
|
||||
this.cache.clear();
|
||||
|
||||
console.log('[CredentialManager] Successfully imported backup');
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('[CredentialManager] Failed to import backup:', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
module.exports = new CredentialManager();
|
||||
@@ -0,0 +1,605 @@
|
||||
/**
|
||||
* Dependency Manager - Service dependency tracking with ordered restart chains
|
||||
*
|
||||
* Manages directed acyclic graph (DAG) of service dependencies. Services can
|
||||
* declare which other services they depend on, and this manager provides:
|
||||
* - Full dependency graph inspection
|
||||
* - Topological ordering for safe restart chains
|
||||
* - Circular dependency detection
|
||||
* - Health-aware restart with per-service polling
|
||||
*
|
||||
* Dependencies are stored directly on service objects in services.json:
|
||||
* { id, name, ..., dependsOn: ['service-id-1', 'service-id-2'] }
|
||||
*
|
||||
* @module dependency-manager
|
||||
*/
|
||||
|
||||
const EventEmitter = require('events');
|
||||
|
||||
/** Maximum seconds to wait for a single container to become healthy after restart */
|
||||
const HEALTH_CHECK_TIMEOUT_MS = 30_000;
|
||||
|
||||
/** Interval between container health polls */
|
||||
const HEALTH_CHECK_INTERVAL_MS = 1_000;
|
||||
|
||||
/**
|
||||
* @typedef {Object} ServiceNode
|
||||
* @property {string} serviceId
|
||||
* @property {string} name
|
||||
* @property {string|null} containerId
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} DependencyEdge
|
||||
* @property {string} from - The service that depends
|
||||
* @property {string} to - The service being depended upon
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} DependencyGraph
|
||||
* @property {ServiceNode[]} nodes
|
||||
* @property {DependencyEdge[]} edges
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} DependencyStatusEntry
|
||||
* @property {string} serviceId
|
||||
* @property {string} name
|
||||
* @property {boolean} isUp
|
||||
* @property {string} [error]
|
||||
*/
|
||||
|
||||
/**
|
||||
* DependencyManager — tracks service dependencies and orchestrates ordered restarts.
|
||||
*
|
||||
* Events emitted:
|
||||
* - `dependency-restart-start` ({ serviceId, chain: string[] })
|
||||
* - `dependency-restart-progress` ({ serviceId, currentServiceId, index, total })
|
||||
* - `dependency-restart-complete` ({ serviceId, chain: string[], results: Array })
|
||||
* - `dependency-restart-failed` ({ serviceId, failedServiceId, error, chain: string[] })
|
||||
*
|
||||
* @extends EventEmitter
|
||||
*/
|
||||
class DependencyManager extends EventEmitter {
|
||||
/**
|
||||
* @param {Object} ctx - Application context
|
||||
* @param {Object} ctx.servicesStateManager - StateManager for services.json
|
||||
* @param {Object} ctx.docker - Docker context ({ client: Dockerode })
|
||||
* @param {Object} ctx.notification - NotificationManager instance
|
||||
* @param {Object} ctx.log - Logger instance
|
||||
*/
|
||||
constructor(ctx) {
|
||||
super();
|
||||
/** @private */
|
||||
this.ctx = ctx;
|
||||
/** @private */
|
||||
this._servicesStateManager = ctx.servicesStateManager;
|
||||
/** @private */
|
||||
this._docker = ctx.docker;
|
||||
/** @private */
|
||||
this._notification = ctx.notification;
|
||||
/** @private */
|
||||
this._log = ctx.log || console;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Core helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Load all services from the state manager.
|
||||
* @private
|
||||
* @returns {Promise<Object[]>}
|
||||
*/
|
||||
async _loadServices() {
|
||||
const data = await this._servicesStateManager.read();
|
||||
return Array.isArray(data) ? data : (data.services || []);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a single service by ID.
|
||||
* @private
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<Object|null>}
|
||||
*/
|
||||
async _findService(serviceId) {
|
||||
const services = await this._loadServices();
|
||||
return services.find(s => s.id === serviceId) || null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Graph queries
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Return the full dependency graph for visualisation.
|
||||
*
|
||||
* @returns {Promise<DependencyGraph>}
|
||||
*/
|
||||
async getDependencyGraph() {
|
||||
const services = await this._loadServices();
|
||||
|
||||
const nodes = services.map(s => ({
|
||||
serviceId: s.id,
|
||||
name: s.name,
|
||||
containerId: s.containerId || null,
|
||||
}));
|
||||
|
||||
const edges = [];
|
||||
for (const service of services) {
|
||||
const deps = service.dependsOn || [];
|
||||
for (const depId of deps) {
|
||||
edges.push({ from: service.id, to: depId });
|
||||
}
|
||||
}
|
||||
|
||||
return { nodes, edges };
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the services that depend on the given service (reverse deps).
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<Object[]>} Services whose `dependsOn` includes `serviceId`.
|
||||
*/
|
||||
async getDependents(serviceId) {
|
||||
const services = await this._loadServices();
|
||||
return services.filter(s => (s.dependsOn || []).includes(serviceId));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the direct dependencies for a service.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<Object[]>} Services that `serviceId` depends on.
|
||||
*/
|
||||
async getDependencies(serviceId) {
|
||||
const services = await this._loadServices();
|
||||
const service = services.find(s => s.id === serviceId);
|
||||
if (!service) return [];
|
||||
const depIds = service.dependsOn || [];
|
||||
return services.filter(s => depIds.includes(s.id));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Topological sort
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Build an adjacency list for the current dependency graph.
|
||||
* Edge direction: service → its dependencies (i.e. what it depends on).
|
||||
*
|
||||
* @private
|
||||
* @param {Object[]} services
|
||||
* @returns {Map<string, string[]>}
|
||||
*/
|
||||
_buildAdjacencyList(services) {
|
||||
const adj = new Map();
|
||||
for (const service of services) {
|
||||
adj.set(service.id, (service.dependsOn || []).slice());
|
||||
}
|
||||
return adj;
|
||||
}
|
||||
|
||||
/**
|
||||
* DFS-based topological sort with cycle detection (white/gray/black coloring).
|
||||
*
|
||||
* Returns services in restart order: dependencies first, dependents last.
|
||||
* The target service is included at the end.
|
||||
*
|
||||
* @private
|
||||
* @param {string} serviceId - Target service (will be last in the result).
|
||||
* @param {Object[]} services - All services.
|
||||
* @param {Map<string, string[]>} adj - Adjacency list (service → deps).
|
||||
* @returns {string[]} Ordered service IDs for restart.
|
||||
* @throws {Error} If a circular dependency is detected.
|
||||
*/
|
||||
_topologicalSort(serviceId, services, adj) {
|
||||
// Collect only the reachable sub-graph from serviceId
|
||||
const visited = new Set();
|
||||
const reachable = new Set();
|
||||
|
||||
const collectReachable = (id) => {
|
||||
if (reachable.has(id)) return;
|
||||
reachable.add(id);
|
||||
for (const dep of (adj.get(id) || [])) {
|
||||
collectReachable(dep);
|
||||
}
|
||||
};
|
||||
collectReachable(serviceId);
|
||||
|
||||
// DFS topological sort on the reachable sub-graph
|
||||
const WHITE = 0, GRAY = 1, BLACK = 2;
|
||||
const color = new Map();
|
||||
for (const id of reachable) color.set(id, WHITE);
|
||||
|
||||
const result = [];
|
||||
|
||||
const dfs = (id) => {
|
||||
if (color.get(id) === BLACK) return;
|
||||
if (color.get(id) === GRAY) {
|
||||
throw new Error(`Circular dependency detected involving service "${id}"`);
|
||||
}
|
||||
color.set(id, GRAY);
|
||||
for (const dep of (adj.get(id) || [])) {
|
||||
dfs(dep);
|
||||
}
|
||||
color.set(id, BLACK);
|
||||
result.push(id);
|
||||
};
|
||||
|
||||
// Visit the target last so it ends up at the end of the result
|
||||
// Actually, we want deps *first* then the target.
|
||||
// The DFS naturally puts deps before dependents, so starting from
|
||||
// serviceId will place it last (which is correct for restart order).
|
||||
dfs(serviceId);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the topologically ordered restart chain for a service.
|
||||
*
|
||||
* The returned array lists all services that must be restarted,
|
||||
* starting with leaf dependencies and ending with the target service.
|
||||
*
|
||||
* @param {string} serviceId - The service to build the chain for.
|
||||
* @returns {Promise<string[]>} Ordered service IDs.
|
||||
* @throws {Error} If `serviceId` doesn't exist or a circular dependency is found.
|
||||
*/
|
||||
async getOrderedRestartChain(serviceId) {
|
||||
const services = await this._loadServices();
|
||||
const service = services.find(s => s.id === serviceId);
|
||||
if (!service) {
|
||||
throw new Error(`Service "${serviceId}" not found`);
|
||||
}
|
||||
|
||||
const adj = this._buildAdjacencyList(services);
|
||||
return this._topologicalSort(serviceId, services, adj);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Validation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Validate a proposed set of dependencies for a service.
|
||||
*
|
||||
* Checks:
|
||||
* - All referenced service IDs exist.
|
||||
* - Adding these dependencies would not create a circular dependency.
|
||||
* - A service cannot depend on itself.
|
||||
*
|
||||
* @param {string} serviceId - The service to set dependencies on.
|
||||
* @param {string[]} dependsOn - Proposed dependency IDs.
|
||||
* @returns {Promise<{ valid: boolean, errors: string[] }>}
|
||||
*/
|
||||
async validateDependencies(serviceId, dependsOn) {
|
||||
const errors = [];
|
||||
|
||||
if (!Array.isArray(dependsOn)) {
|
||||
return { valid: false, errors: ['dependsOn must be an array'] };
|
||||
}
|
||||
|
||||
const services = await this._loadServices();
|
||||
const allIds = new Set(services.map(s => s.id));
|
||||
|
||||
// Service must exist
|
||||
if (!allIds.has(serviceId)) {
|
||||
return { valid: false, errors: [`Service "${serviceId}" not found`] };
|
||||
}
|
||||
|
||||
// Self-dependency
|
||||
if (dependsOn.includes(serviceId)) {
|
||||
errors.push(`Service "${serviceId}" cannot depend on itself`);
|
||||
}
|
||||
|
||||
// Existence check
|
||||
for (const depId of dependsOn) {
|
||||
if (!allIds.has(depId)) {
|
||||
errors.push(`Dependency service "${depId}" does not exist`);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
return { valid: false, errors };
|
||||
}
|
||||
|
||||
// Circular dependency check: temporarily set the proposed dependsOn
|
||||
// and attempt a topological sort.
|
||||
const tempServices = services.map(s => {
|
||||
if (s.id === serviceId) {
|
||||
return { ...s, dependsOn: dependsOn.slice() };
|
||||
}
|
||||
return { ...s };
|
||||
});
|
||||
|
||||
const adj = this._buildAdjacencyList(tempServices);
|
||||
|
||||
// Check every node for cycles with the new edges
|
||||
try {
|
||||
const WHITE = 0, GRAY = 1, BLACK = 2;
|
||||
const color = new Map();
|
||||
for (const s of tempServices) color.set(s.id, WHITE);
|
||||
|
||||
const dfs = (id) => {
|
||||
if (color.get(id) === BLACK) return;
|
||||
if (color.get(id) === GRAY) {
|
||||
throw new Error(`Circular dependency detected involving service "${id}"`);
|
||||
}
|
||||
color.set(id, GRAY);
|
||||
for (const dep of (adj.get(id) || [])) {
|
||||
dfs(dep);
|
||||
}
|
||||
color.set(id, BLACK);
|
||||
};
|
||||
|
||||
for (const s of tempServices) {
|
||||
if (color.get(s.id) === WHITE) {
|
||||
dfs(s.id);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push(err.message);
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Health status
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Get the current container status for a service and all its transitive dependencies.
|
||||
*
|
||||
* @param {string} serviceId
|
||||
* @returns {Promise<DependencyStatusEntry[]>}
|
||||
* @throws {Error} If `serviceId` doesn't exist.
|
||||
*/
|
||||
async getDependencyStatus(serviceId) {
|
||||
const services = await this._loadServices();
|
||||
const service = services.find(s => s.id === serviceId);
|
||||
if (!service) {
|
||||
throw new Error(`Service "${serviceId}" not found`);
|
||||
}
|
||||
|
||||
// Collect all transitive dependencies via BFS
|
||||
const serviceMap = new Map(services.map(s => [s.id, s]));
|
||||
const visited = new Set();
|
||||
const queue = [serviceId];
|
||||
const allRelated = [];
|
||||
|
||||
while (queue.length > 0) {
|
||||
const currentId = queue.shift();
|
||||
if (visited.has(currentId)) continue;
|
||||
visited.add(currentId);
|
||||
|
||||
const svc = serviceMap.get(currentId);
|
||||
if (!svc) continue;
|
||||
|
||||
allRelated.push(svc);
|
||||
|
||||
for (const depId of (svc.dependsOn || [])) {
|
||||
if (!visited.has(depId)) {
|
||||
queue.push(depId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Query container status for each
|
||||
const results = [];
|
||||
for (const svc of allRelated) {
|
||||
const entry = {
|
||||
serviceId: svc.id,
|
||||
name: svc.name,
|
||||
isUp: false,
|
||||
};
|
||||
|
||||
if (!svc.containerId) {
|
||||
entry.error = 'No container associated with this service';
|
||||
results.push(entry);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const container = this._docker.client.getContainer(svc.containerId);
|
||||
const info = await container.inspect();
|
||||
entry.isUp = info.State?.Running === true;
|
||||
} catch (err) {
|
||||
entry.error = err.message || 'Unable to inspect container';
|
||||
}
|
||||
|
||||
results.push(entry);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Restart with dependencies
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Wait for a container to report as running after a restart.
|
||||
*
|
||||
* @private
|
||||
* @param {string} containerId
|
||||
* @param {number} [timeoutMs=30000]
|
||||
* @returns {Promise<boolean>} `true` if healthy, `false` if timed out.
|
||||
*/
|
||||
async _waitForContainerHealthy(containerId, timeoutMs = HEALTH_CHECK_TIMEOUT_MS) {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
try {
|
||||
const container = this._docker.client.getContainer(containerId);
|
||||
const info = await container.inspect();
|
||||
if (info.State?.Running === true) {
|
||||
return true;
|
||||
}
|
||||
} catch {
|
||||
// Container might not be inspectable during restart — keep polling
|
||||
}
|
||||
await new Promise(r => setTimeout(r, HEALTH_CHECK_INTERVAL_MS));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Restart a service and all its dependencies in topological order.
|
||||
*
|
||||
* Emits progress events and sends a notification on completion/failure.
|
||||
* This method is designed to be called from the route handler and
|
||||
* **does not throw** — errors are reported via events and notifications.
|
||||
*
|
||||
* @param {string} serviceId - Target service to restart (with deps).
|
||||
* @returns {Promise<{ success: boolean, chain: string[], results: Array }>}
|
||||
*/
|
||||
async restartWithDependencies(serviceId) {
|
||||
const service = await this._findService(serviceId);
|
||||
if (!service) {
|
||||
const err = new Error(`Service "${serviceId}" not found`);
|
||||
this.emit('dependency-restart-failed', {
|
||||
serviceId,
|
||||
failedServiceId: serviceId,
|
||||
error: err.message,
|
||||
chain: [],
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
|
||||
let chain;
|
||||
try {
|
||||
chain = await this.getOrderedRestartChain(serviceId);
|
||||
} catch (err) {
|
||||
this.emit('dependency-restart-failed', {
|
||||
serviceId,
|
||||
failedServiceId: serviceId,
|
||||
error: err.message,
|
||||
chain: [],
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
|
||||
const services = await this._loadServices();
|
||||
const serviceMap = new Map(services.map(s => [s.id, s]));
|
||||
|
||||
this._log.info('dependency', 'Starting dependency restart chain', {
|
||||
serviceId,
|
||||
chain,
|
||||
});
|
||||
|
||||
this.emit('dependency-restart-start', { serviceId, chain });
|
||||
|
||||
const results = [];
|
||||
const total = chain.length;
|
||||
|
||||
for (let i = 0; i < total; i++) {
|
||||
const currentId = chain[i];
|
||||
const svc = serviceMap.get(currentId);
|
||||
|
||||
this.emit('dependency-restart-progress', {
|
||||
serviceId,
|
||||
currentServiceId: currentId,
|
||||
index: i,
|
||||
total,
|
||||
});
|
||||
|
||||
if (!svc || !svc.containerId) {
|
||||
const msg = !svc
|
||||
? `Service "${currentId}" not found in state`
|
||||
: `Service "${currentId}" has no container — skipping restart`;
|
||||
this._log.warn('dependency', msg);
|
||||
results.push({ serviceId: currentId, restarted: false, skipped: true, reason: msg });
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const container = this._docker.client.getContainer(svc.containerId);
|
||||
this._log.info('dependency', `Restarting container for service "${currentId}"`, {
|
||||
containerId: svc.containerId,
|
||||
});
|
||||
await container.restart();
|
||||
|
||||
// Wait for it to come back up
|
||||
const healthy = await this._waitForContainerHealthy(svc.containerId);
|
||||
if (!healthy) {
|
||||
const msg = `Container for service "${currentId}" did not become healthy within ${HEALTH_CHECK_TIMEOUT_MS / 1000}s`;
|
||||
this._log.warn('dependency', msg);
|
||||
results.push({ serviceId: currentId, restarted: true, healthy: false, error: msg });
|
||||
|
||||
// Abort chain — dependency didn't come back
|
||||
this.emit('dependency-restart-failed', {
|
||||
serviceId,
|
||||
failedServiceId: currentId,
|
||||
error: msg,
|
||||
chain,
|
||||
});
|
||||
await this._notifyRestartResult(serviceId, false, chain, results, currentId);
|
||||
return { success: false, chain, results };
|
||||
}
|
||||
|
||||
this._log.info('dependency', `Service "${currentId}" is healthy after restart`);
|
||||
results.push({ serviceId: currentId, restarted: true, healthy: true });
|
||||
} catch (err) {
|
||||
const msg = err.message || 'Unknown error during restart';
|
||||
this._log.error('dependency', `Failed to restart service "${currentId}"`, {
|
||||
error: msg,
|
||||
});
|
||||
results.push({ serviceId: currentId, restarted: false, error: msg });
|
||||
|
||||
this.emit('dependency-restart-failed', {
|
||||
serviceId,
|
||||
failedServiceId: currentId,
|
||||
error: msg,
|
||||
chain,
|
||||
});
|
||||
await this._notifyRestartResult(serviceId, false, chain, results, currentId);
|
||||
return { success: false, chain, results };
|
||||
}
|
||||
}
|
||||
|
||||
this.emit('dependency-restart-complete', { serviceId, chain, results });
|
||||
await this._notifyRestartResult(serviceId, true, chain, results);
|
||||
return { success: true, chain, results };
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a notification about the restart result.
|
||||
*
|
||||
* @private
|
||||
* @param {string} serviceId
|
||||
* @param {boolean} success
|
||||
* @param {string[]} chain
|
||||
* @param {Array} results
|
||||
* @param {string} [failedServiceId]
|
||||
*/
|
||||
async _notifyRestartResult(serviceId, success, chain, results, failedServiceId) {
|
||||
if (!this._notification) return;
|
||||
|
||||
try {
|
||||
if (success) {
|
||||
await this._notification.send('dependency-restart-complete', {
|
||||
text: `✅ Dependency restart chain completed for "${serviceId}". Restarted: ${chain.join(' → ')}`,
|
||||
serviceId,
|
||||
chain,
|
||||
results,
|
||||
});
|
||||
} else {
|
||||
await this._notification.send('dependency-restart-failed', {
|
||||
text: `❌ Dependency restart chain failed for "${serviceId}" at "${failedServiceId}". Chain: ${chain.join(' → ')}`,
|
||||
serviceId,
|
||||
failedServiceId,
|
||||
chain,
|
||||
results,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
this._log.error('dependency', 'Failed to send restart notification', {
|
||||
error: err.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = DependencyManager;
|
||||
@@ -0,0 +1,494 @@
|
||||
/**
|
||||
* DashCaddy License Manager
|
||||
*
|
||||
* Runtime license validation, activation, and feature gating.
|
||||
* Uses credential-manager for secure storage of activation tokens.
|
||||
*
|
||||
* Hybrid model:
|
||||
* - First activation: online validation against license server (if reachable)
|
||||
* - Fallback: offline HMAC validation using embedded master secret hash
|
||||
* - Ongoing: locally stored activation token checked on each premium request
|
||||
*/
|
||||
|
||||
const crypto = require('crypto');
|
||||
const os = require('os');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { verifyCode, parseCode, VALID_DURATIONS } = require('./license-keygen');
|
||||
const { errorResponse } = require('../utils/responses');
|
||||
|
||||
const LICENSE_CRED_KEY = 'license.activation';
|
||||
const LICENSE_SERVER_URL = process.env.LICENSE_SERVER_URL || null; // Set when license server exists
|
||||
|
||||
// Features gated behind premium
|
||||
const PREMIUM_FEATURES = {
|
||||
sso: { name: 'Auto-Login SSO', description: 'Automatic single sign-on for deployed apps' },
|
||||
recipes: { name: 'Recipes', description: 'Multi-container stack deployment' },
|
||||
swarm: { name: 'Docker Swarm', description: 'Multi-node cluster orchestration' }
|
||||
};
|
||||
|
||||
class LicenseManager {
|
||||
constructor(credentialManager, configFile, log) {
|
||||
this.credentialManager = credentialManager;
|
||||
this.configFile = configFile;
|
||||
this.log = log || console;
|
||||
this.activation = null; // Cached activation state
|
||||
this.masterSecretHash = null; // Loaded from shipped secret hash (not the secret itself)
|
||||
this._loaded = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load license state from storage on startup.
|
||||
* Primary: encrypted credential store. Fallback: config.json backup.
|
||||
* If the credential store fails (e.g. encryption key changed after rebuild),
|
||||
* restores from the config.json backup automatically.
|
||||
*/
|
||||
async load() {
|
||||
try {
|
||||
const stored = await this.credentialManager.retrieve(LICENSE_CRED_KEY);
|
||||
if (stored) {
|
||||
this.activation = JSON.parse(stored);
|
||||
if (this.isExpired()) {
|
||||
this.log.info?.('license', 'License has expired', {
|
||||
code: this._maskCode(this.activation.code),
|
||||
expiredAt: this.activation.expiresAt
|
||||
});
|
||||
} else {
|
||||
this.log.info?.('license', 'License loaded', {
|
||||
code: this._maskCode(this.activation.code),
|
||||
expiresAt: this.activation.expiresAt,
|
||||
daysRemaining: this.daysRemaining()
|
||||
});
|
||||
}
|
||||
this._loaded = true;
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
this.log.warn?.('license', 'Failed to load from credential store, trying config backup', { error: error.message });
|
||||
}
|
||||
|
||||
// Fallback: restore from config.json backup
|
||||
try {
|
||||
const fsp = require('fs').promises;
|
||||
const data = await fsp.readFile(this.configFile, 'utf8');
|
||||
const config = JSON.parse(data);
|
||||
if (config.licenseBackup) {
|
||||
this.activation = config.licenseBackup;
|
||||
this.log.info?.('license', 'License restored from config backup', {
|
||||
code: this._maskCode(this.activation.code),
|
||||
lifetime: this.activation.lifetime
|
||||
});
|
||||
// Re-store in credential manager so future loads succeed
|
||||
try {
|
||||
await this.credentialManager.store(LICENSE_CRED_KEY, JSON.stringify(this.activation));
|
||||
this.log.info?.('license', 'License re-stored in credential manager');
|
||||
} catch (storeErr) {
|
||||
this.log.warn?.('license', 'Could not re-store license in credential manager', { error: storeErr.message });
|
||||
}
|
||||
this._loaded = true;
|
||||
return;
|
||||
}
|
||||
} catch (_) {
|
||||
// Config doesn't exist or no backup — continue
|
||||
}
|
||||
|
||||
this.log.info?.('license', 'No active license');
|
||||
this.activation = null;
|
||||
this._loaded = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the shipped master secret hash for offline validation.
|
||||
* The actual master secret is NEVER shipped — only a hash of it is embedded
|
||||
* in the product, and the keygen embeds HMAC signatures in codes using the real secret.
|
||||
* For offline validation, we verify the code's internal HMAC consistency.
|
||||
*
|
||||
* @param {string} secretFile - Path to .license-secret file (dev only) or .license-secret-hash (shipped)
|
||||
*/
|
||||
loadSecret(secretFile) {
|
||||
try {
|
||||
if (fs.existsSync(secretFile)) {
|
||||
const secret = fs.readFileSync(secretFile, 'utf8').trim();
|
||||
this.masterSecretHash = secret;
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
this.log.warn?.('license', 'Could not load license secret', { error: error.message });
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a machine fingerprint for activation binding
|
||||
*/
|
||||
getMachineFingerprint() {
|
||||
const components = [
|
||||
os.hostname(),
|
||||
os.platform(),
|
||||
os.arch(),
|
||||
os.cpus()[0]?.model || 'unknown'
|
||||
];
|
||||
// Get primary MAC address
|
||||
const interfaces = os.networkInterfaces();
|
||||
for (const name of Object.keys(interfaces)) {
|
||||
for (const iface of interfaces[name]) {
|
||||
if (!iface.internal && iface.mac && iface.mac !== '00:00:00:00:00:00') {
|
||||
components.push(iface.mac);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return crypto.createHash('sha256').update(components.join('|')).digest('hex').substring(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
* Activate a license code
|
||||
* @param {string} code - License code (DC-XXXXX-XXXXX-XXXXX-XXXXX-XXXXX)
|
||||
* @returns {Object} { success, message, activation? }
|
||||
*/
|
||||
async activate(code) {
|
||||
if (!code || typeof code !== 'string') {
|
||||
return { success: false, message: 'License code is required' };
|
||||
}
|
||||
|
||||
// Normalize code format
|
||||
code = code.trim().toUpperCase();
|
||||
if (!code.startsWith('DC-')) {
|
||||
return { success: false, message: 'Invalid code format. Codes start with DC-' };
|
||||
}
|
||||
|
||||
// Check if already activated with this code
|
||||
if (this.activation && this.activation.code === code && !this.isExpired()) {
|
||||
return {
|
||||
success: true,
|
||||
message: 'This code is already activated',
|
||||
activation: this.getStatus()
|
||||
};
|
||||
}
|
||||
|
||||
// Try online validation first
|
||||
let onlineResult = null;
|
||||
if (LICENSE_SERVER_URL) {
|
||||
onlineResult = await this._validateOnline(code);
|
||||
if (onlineResult && !onlineResult.success) {
|
||||
// Server explicitly rejected — don't fallback to offline
|
||||
return onlineResult;
|
||||
}
|
||||
}
|
||||
|
||||
// Offline validation (HMAC check)
|
||||
if (!onlineResult) {
|
||||
const offlineResult = this._validateOffline(code);
|
||||
if (!offlineResult.valid) {
|
||||
return { success: false, message: offlineResult.reason || 'Invalid license code' };
|
||||
}
|
||||
|
||||
// Code is cryptographically valid
|
||||
const machineId = this.getMachineFingerprint();
|
||||
const now = new Date();
|
||||
const isLifetime = offlineResult.durationDays === 0;
|
||||
const expiresAt = isLifetime
|
||||
? new Date('2099-12-31T23:59:59.999Z')
|
||||
: new Date(now.getTime() + offlineResult.durationDays * 86400000);
|
||||
|
||||
this.activation = {
|
||||
code,
|
||||
codeId: offlineResult.codeId,
|
||||
durationDays: offlineResult.durationDays,
|
||||
lifetime: isLifetime,
|
||||
activatedAt: now.toISOString(),
|
||||
expiresAt: expiresAt.toISOString(),
|
||||
machineId,
|
||||
validationMethod: 'offline',
|
||||
features: Object.keys(PREMIUM_FEATURES)
|
||||
};
|
||||
} else {
|
||||
// Online validation succeeded — use server response
|
||||
this.activation = onlineResult.activation;
|
||||
this.activation.validationMethod = 'online';
|
||||
}
|
||||
|
||||
// Store activation token
|
||||
try {
|
||||
await this.credentialManager.store(LICENSE_CRED_KEY, JSON.stringify(this.activation), {
|
||||
activatedAt: this.activation.activatedAt,
|
||||
expiresAt: this.activation.expiresAt
|
||||
});
|
||||
} catch (error) {
|
||||
this.log.error?.('license', 'Failed to store activation', { error: error.message });
|
||||
return { success: false, message: 'License validated but failed to save activation' };
|
||||
}
|
||||
|
||||
// Update config.json with license info (non-sensitive)
|
||||
await this._updateConfig();
|
||||
|
||||
this.log.info?.('license', 'License activated', {
|
||||
code: this._maskCode(code),
|
||||
durationDays: this.activation.durationDays,
|
||||
expiresAt: this.activation.expiresAt,
|
||||
method: this.activation.validationMethod
|
||||
});
|
||||
|
||||
const durationLabel = this.activation.lifetime ? 'lifetime' : `${this.activation.durationDays} days`;
|
||||
return {
|
||||
success: true,
|
||||
message: `License activated for ${durationLabel}`,
|
||||
activation: this.getStatus()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Deactivate the current license
|
||||
* @returns {Object} { success, message }
|
||||
*/
|
||||
async deactivate() {
|
||||
if (!this.activation) {
|
||||
return { success: false, message: 'No active license to deactivate' };
|
||||
}
|
||||
|
||||
const code = this._maskCode(this.activation.code);
|
||||
|
||||
// If online server exists, notify it of deactivation
|
||||
if (LICENSE_SERVER_URL) {
|
||||
try {
|
||||
await this._notifyDeactivation();
|
||||
} catch (error) {
|
||||
this.log.warn?.('license', 'Could not notify license server of deactivation', { error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
// Clear local activation
|
||||
await this.credentialManager.delete(LICENSE_CRED_KEY);
|
||||
this.activation = null;
|
||||
await this._updateConfig();
|
||||
|
||||
this.log.info?.('license', 'License deactivated', { code });
|
||||
|
||||
return { success: true, message: 'License deactivated. You can reuse this code on another machine.' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current license status
|
||||
* @returns {Object} Status object
|
||||
*/
|
||||
getStatus() {
|
||||
if (!this.activation) {
|
||||
return {
|
||||
active: false,
|
||||
tier: 'free',
|
||||
features: [],
|
||||
premiumFeatures: PREMIUM_FEATURES
|
||||
};
|
||||
}
|
||||
|
||||
const expired = this.isExpired();
|
||||
const isLifetime = !!(this.activation.lifetime || this.activation.durationDays === 0);
|
||||
const daysRemaining = isLifetime ? null : this.daysRemaining();
|
||||
|
||||
return {
|
||||
active: !expired,
|
||||
tier: expired ? 'free' : 'premium',
|
||||
lifetime: isLifetime,
|
||||
code: this._maskCode(this.activation.code),
|
||||
durationDays: this.activation.durationDays,
|
||||
activatedAt: this.activation.activatedAt,
|
||||
expiresAt: isLifetime ? null : this.activation.expiresAt,
|
||||
daysRemaining: isLifetime ? null : Math.max(0, daysRemaining),
|
||||
expired,
|
||||
features: expired ? [] : (this.activation.features || Object.keys(PREMIUM_FEATURES)),
|
||||
premiumFeatures: PREMIUM_FEATURES,
|
||||
validationMethod: this.activation.validationMethod
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a specific premium feature is available
|
||||
* @param {string} feature - Feature key (e.g., 'sso', 'recipes', 'swarm')
|
||||
* @returns {boolean}
|
||||
*/
|
||||
hasFeature(feature) {
|
||||
if (!this.activation) return false;
|
||||
if (this.isExpired()) return false;
|
||||
const features = this.activation.features || Object.keys(PREMIUM_FEATURES);
|
||||
return features.includes(feature);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the license has expired
|
||||
*/
|
||||
isExpired() {
|
||||
if (!this.activation) return true;
|
||||
// Lifetime licenses never expire
|
||||
if (this.activation.lifetime || this.activation.durationDays === 0) return false;
|
||||
if (!this.activation.expiresAt) return false; // No expiry set = lifetime
|
||||
return Date.now() > new Date(this.activation.expiresAt).getTime();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get days remaining on the license
|
||||
*/
|
||||
daysRemaining() {
|
||||
if (!this.activation) return 0;
|
||||
const remaining = new Date(this.activation.expiresAt).getTime() - Date.now();
|
||||
return Math.ceil(remaining / 86400000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Express middleware: gate a route behind a premium feature
|
||||
* @param {string} feature - Feature key
|
||||
* @returns {Function} Express middleware
|
||||
*/
|
||||
requirePremium(feature) {
|
||||
return (req, res, next) => {
|
||||
if (this.hasFeature(feature)) {
|
||||
return next();
|
||||
}
|
||||
|
||||
const featureInfo = PREMIUM_FEATURES[feature] || { name: feature };
|
||||
return errorResponse(res, 403, `${featureInfo.name} requires a DashCaddy Premium subscription.`, {
|
||||
premiumRequired: true,
|
||||
feature,
|
||||
featureName: featureInfo.name,
|
||||
featureDescription: featureInfo.description,
|
||||
currentTier: this.isExpired() ? 'free' : 'expired',
|
||||
upgradeUrl: '/settings#license'
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
// Private methods
|
||||
|
||||
/**
|
||||
* Validate code offline using HMAC
|
||||
*/
|
||||
_validateOffline(code) {
|
||||
if (!this.masterSecretHash) {
|
||||
// No secret available — try structural validation only
|
||||
try {
|
||||
const parsed = parseCode(code);
|
||||
// Without the secret we can't verify HMAC, but we can check structure
|
||||
if (parsed.version !== 1) return { valid: false, reason: 'Unsupported code version' };
|
||||
if (parsed.durationDays !== 0 && !VALID_DURATIONS.includes(parsed.durationDays)) return { valid: false, reason: 'Invalid duration' };
|
||||
// Can't verify signature without secret — reject
|
||||
return { valid: false, reason: 'License validation unavailable. Please try again when connected to the internet.' };
|
||||
} catch (e) {
|
||||
return { valid: false, reason: e.message };
|
||||
}
|
||||
}
|
||||
|
||||
// Full verification with secret
|
||||
return verifyCode(this.masterSecretHash, code);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate code against online license server
|
||||
*/
|
||||
async _validateOnline(code) {
|
||||
try {
|
||||
const machineId = this.getMachineFingerprint();
|
||||
const response = await fetch(`${LICENSE_SERVER_URL}/api/license/validate`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ code, machineId }),
|
||||
signal: AbortSignal.timeout(10000) // 10s timeout
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const data = await response.json().catch(() => ({}));
|
||||
return { success: false, message: data.error || `Server returned ${response.status}` };
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
if (data.success) {
|
||||
return {
|
||||
success: true,
|
||||
activation: {
|
||||
code,
|
||||
codeId: data.codeId,
|
||||
durationDays: data.durationDays,
|
||||
activatedAt: new Date().toISOString(),
|
||||
expiresAt: data.expiresAt,
|
||||
machineId,
|
||||
features: data.features || Object.keys(PREMIUM_FEATURES),
|
||||
serverToken: data.token
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
return { success: false, message: data.message || 'License server rejected the code' };
|
||||
} catch (error) {
|
||||
// Server unreachable — return null to fallback to offline
|
||||
this.log.warn?.('license', 'License server unreachable, falling back to offline validation', {
|
||||
error: error.message
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify license server of deactivation
|
||||
*/
|
||||
async _notifyDeactivation() {
|
||||
if (!LICENSE_SERVER_URL || !this.activation) return;
|
||||
await fetch(`${LICENSE_SERVER_URL}/api/license/deactivate`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
code: this.activation.code,
|
||||
machineId: this.activation.machineId,
|
||||
serverToken: this.activation.serverToken
|
||||
}),
|
||||
signal: AbortSignal.timeout(10000)
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Update config.json with license info and full activation backup.
|
||||
* The backup ensures the license survives encryption key changes
|
||||
* (e.g. container rebuilds that generate new keys).
|
||||
*/
|
||||
async _updateConfig() {
|
||||
try {
|
||||
const fsp = require('fs').promises;
|
||||
let config = {};
|
||||
try {
|
||||
const data = await fsp.readFile(this.configFile, 'utf8');
|
||||
config = JSON.parse(data);
|
||||
} catch (e) {
|
||||
// Config doesn't exist yet
|
||||
}
|
||||
|
||||
if (this.activation && !this.isExpired()) {
|
||||
config.license = {
|
||||
active: true,
|
||||
tier: 'premium',
|
||||
expiresAt: this.activation.expiresAt,
|
||||
daysRemaining: this.daysRemaining(),
|
||||
features: this.activation.features || Object.keys(PREMIUM_FEATURES)
|
||||
};
|
||||
// Full backup of activation data (config.json is volume-mounted and persists)
|
||||
config.licenseBackup = this.activation;
|
||||
} else {
|
||||
config.license = { active: false, tier: 'free' };
|
||||
delete config.licenseBackup;
|
||||
}
|
||||
|
||||
config.updatedAt = new Date().toISOString();
|
||||
await fsp.writeFile(this.configFile, JSON.stringify(config, null, 2), 'utf8');
|
||||
} catch (error) {
|
||||
this.log.error?.('license', 'Failed to update config with license info', { error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mask a license code for display (show first and last groups only)
|
||||
*/
|
||||
_maskCode(code) {
|
||||
if (!code) return 'none';
|
||||
const parts = code.split('-');
|
||||
if (parts.length < 4) return 'DC-*****';
|
||||
return `${parts[0]}-${parts[1]}-*****-*****-${parts[parts.length - 1]}`;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { LicenseManager, PREMIUM_FEATURES };
|
||||
@@ -0,0 +1,501 @@
|
||||
/**
|
||||
* Notification Manager - Multi-provider notification delivery
|
||||
* Supports Discord, Telegram, ntfy, and Email notifications
|
||||
*/
|
||||
const EventEmitter = require('events');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const nodemailer = require('nodemailer');
|
||||
|
||||
const DEFAULT_CONFIG = {
|
||||
enabled: true,
|
||||
providers: {
|
||||
discord: { enabled: false, webhookUrl: '' },
|
||||
telegram: { enabled: false, botToken: '', chatId: '' },
|
||||
ntfy: { enabled: false, topic: '', serverUrl: 'https://ntfy.sh' },
|
||||
email: { enabled: false, host: '', port: 587, to: '', from: '', username: '', password: '' }
|
||||
},
|
||||
events: {
|
||||
'container-down': true,
|
||||
'container-up': false,
|
||||
'alert': true,
|
||||
'backup-complete': true,
|
||||
'backup-failed': true,
|
||||
'update-available': true
|
||||
}
|
||||
};
|
||||
|
||||
class NotificationManager extends EventEmitter {
|
||||
constructor(ctx) {
|
||||
super();
|
||||
this.ctx = ctx;
|
||||
this.NOTIFICATIONS_FILE = ctx.NOTIFICATIONS_FILE;
|
||||
this.log = ctx.log || console;
|
||||
this.config = { ...DEFAULT_CONFIG };
|
||||
this.lastSent = null;
|
||||
this.history = [];
|
||||
this.maxHistory = 100;
|
||||
this.healthDaemonInterval = null;
|
||||
this.healthState = new Map();
|
||||
|
||||
this._loadConfig();
|
||||
}
|
||||
|
||||
/**
|
||||
* Load config from file
|
||||
*/
|
||||
_loadConfig() {
|
||||
try {
|
||||
if (fs.existsSync(this.NOTIFICATIONS_FILE)) {
|
||||
const data = JSON.parse(fs.readFileSync(this.NOTIFICATIONS_FILE, 'utf8'));
|
||||
this.config = this._mergeConfig(DEFAULT_CONFIG, data);
|
||||
}
|
||||
} catch (error) {
|
||||
this.log.error('notification', 'Failed to load config', { error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge loaded config with defaults
|
||||
*/
|
||||
_mergeConfig(defaults, loaded) {
|
||||
const result = { ...defaults };
|
||||
for (const key of Object.keys(defaults)) {
|
||||
if (loaded && typeof defaults[key] === 'object' && !Array.isArray(defaults[key])) {
|
||||
result[key] = { ...defaults[key], ...loaded[key] };
|
||||
} else if (loaded && loaded[key] !== undefined) {
|
||||
result[key] = loaded[key];
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current config (for API)
|
||||
*/
|
||||
getConfig() {
|
||||
return this.config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save config to file
|
||||
*/
|
||||
async saveConfig() {
|
||||
try {
|
||||
const dir = path.dirname(this.NOTIFICATIONS_FILE);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(this.NOTIFICATIONS_FILE, JSON.stringify(this.config, null, 2));
|
||||
return true;
|
||||
} catch (error) {
|
||||
this.log.error('notification', 'Failed to save config', { error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get notification history
|
||||
*/
|
||||
getHistory() {
|
||||
return this.history.slice();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear notification history
|
||||
*/
|
||||
clearHistory() {
|
||||
this.history = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Add entry to history
|
||||
*/
|
||||
_addToHistory(entry) {
|
||||
this.history.unshift({
|
||||
...entry,
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
if (this.history.length > this.maxHistory) {
|
||||
this.history = this.history.slice(0, this.maxHistory);
|
||||
}
|
||||
this.lastSent = new Date().toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Send notification via all enabled providers
|
||||
*/
|
||||
async send(event, data, type = 'info') {
|
||||
if (!this.config.enabled) {
|
||||
return { success: false, error: 'Notifications disabled' };
|
||||
}
|
||||
|
||||
// Check if event is enabled
|
||||
if (event && this.config.events && !this.config.events[event]) {
|
||||
return { success: false, error: `Event ${event} not enabled` };
|
||||
}
|
||||
|
||||
const results = [];
|
||||
const providers = this.config.providers;
|
||||
|
||||
// Discord
|
||||
if (providers.discord?.enabled && providers.discord?.webhookUrl) {
|
||||
try {
|
||||
const result = await this.sendDiscord(this._formatText(data, event), this._formatEmbed(data, event, type));
|
||||
results.push({ provider: 'discord', ...result });
|
||||
} catch (error) {
|
||||
results.push({ provider: 'discord', success: false, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
// Telegram
|
||||
if (providers.telegram?.enabled && providers.telegram?.botToken && providers.telegram?.chatId) {
|
||||
try {
|
||||
const result = await this.sendTelegram(this._formatText(data, event));
|
||||
results.push({ provider: 'telegram', ...result });
|
||||
} catch (error) {
|
||||
results.push({ provider: 'telegram', success: false, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
// ntfy
|
||||
if (providers.ntfy?.enabled && providers.ntfy?.topic) {
|
||||
try {
|
||||
const result = await this.sendNtfy(this._formatText(data, event), this._formatTitle(event));
|
||||
results.push({ provider: 'ntfy', ...result });
|
||||
} catch (error) {
|
||||
results.push({ provider: 'ntfy', success: false, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
// Email
|
||||
if (providers.email?.enabled && providers.email?.host && providers.email?.to) {
|
||||
try {
|
||||
const result = await this.sendEmail(
|
||||
this._formatTitle(event),
|
||||
this._formatText(data, event)
|
||||
);
|
||||
results.push({ provider: 'email', ...result });
|
||||
} catch (error) {
|
||||
results.push({ provider: 'email', success: false, error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
const allSucceeded = results.every(r => r.success);
|
||||
this._addToHistory({
|
||||
title: this._formatTitle(event),
|
||||
type,
|
||||
event,
|
||||
results
|
||||
});
|
||||
|
||||
return { success: allSucceeded, results };
|
||||
}
|
||||
|
||||
/**
|
||||
* Send Discord webhook notification
|
||||
*/
|
||||
async sendDiscord(text, embed) {
|
||||
const { webhookUrl } = this.config.providers.discord;
|
||||
if (!webhookUrl) {
|
||||
throw new Error('Discord webhook not configured');
|
||||
}
|
||||
|
||||
const payload = {
|
||||
content: text,
|
||||
embeds: embed ? [embed] : []
|
||||
};
|
||||
|
||||
const response = await this.ctx.fetchT(webhookUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Discord API error: ${response.status}`);
|
||||
}
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Send Telegram message
|
||||
*/
|
||||
async sendTelegram(text) {
|
||||
const { botToken, chatId } = this.config.providers.telegram;
|
||||
if (!botToken || !chatId) {
|
||||
throw new Error('Telegram not configured');
|
||||
}
|
||||
|
||||
const url = `https://api.telegram.org/bot${botToken}/sendMessage`;
|
||||
const payload = {
|
||||
chat_id: chatId,
|
||||
text,
|
||||
parse_mode: 'Markdown'
|
||||
};
|
||||
|
||||
const response = await this.ctx.fetchT(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
if (!data.ok) {
|
||||
throw new Error(`Telegram error: ${data.description}`);
|
||||
}
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Send ntfy notification
|
||||
*/
|
||||
async sendNtfy(text, title) {
|
||||
const { serverUrl, topic } = this.config.providers.ntfy;
|
||||
if (!topic) {
|
||||
throw new Error('ntfy topic not configured');
|
||||
}
|
||||
|
||||
const url = `${serverUrl.replace(/\/$/, '')}/${topic}`;
|
||||
|
||||
const response = await this.ctx.fetchT(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'text/plain',
|
||||
'Title': title || 'DashCaddy',
|
||||
'Priority': '3'
|
||||
},
|
||||
body: text
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`ntfy error: ${response.status}`);
|
||||
}
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Send email notification
|
||||
*/
|
||||
async sendEmail(subject, body) {
|
||||
const { host, port, to, from, username, password, secure } = this.config.providers.email;
|
||||
if (!host || !to) {
|
||||
throw new Error('Email not configured');
|
||||
}
|
||||
|
||||
// Create transporter
|
||||
const transporter = nodemailer.createTransport({
|
||||
host,
|
||||
port: parseInt(port) || 587,
|
||||
secure: !!secure,
|
||||
auth: username ? {
|
||||
user: username,
|
||||
pass: password
|
||||
} : undefined
|
||||
});
|
||||
|
||||
// Send mail
|
||||
await transporter.sendMail({
|
||||
from: from || username,
|
||||
to,
|
||||
subject,
|
||||
text: body,
|
||||
html: `<pre style="font-family: monospace;">${body}</pre>`
|
||||
});
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Send resource alert
|
||||
*/
|
||||
async sendAlert(alert) {
|
||||
const text = this._formatAlertText(alert);
|
||||
const embed = {
|
||||
title: `⚠️ Resource Alert: ${alert.containerName}`,
|
||||
color: this._getAlertColor(alert.alerts),
|
||||
fields: alert.alerts.map(a => ({
|
||||
name: a.type.toUpperCase(),
|
||||
value: a.message,
|
||||
inline: true
|
||||
})),
|
||||
footer: {
|
||||
text: 'DashCaddy Resource Monitor'
|
||||
},
|
||||
timestamp: alert.timestamp
|
||||
};
|
||||
|
||||
return this.send('alert', { ...alert, text, embed }, 'warning');
|
||||
}
|
||||
|
||||
/**
|
||||
* Send backup complete notification
|
||||
*/
|
||||
async sendBackupComplete(backup) {
|
||||
const event = backup.status === 'success' ? 'backup-complete' : 'backup-failed';
|
||||
const type = backup.status === 'success' ? 'success' : 'error';
|
||||
|
||||
const text = backup.status === 'success'
|
||||
? `✅ Backup "${backup.name}" completed successfully`
|
||||
: `❌ Backup "${backup.name}" failed: ${backup.error}`;
|
||||
|
||||
return this.send(event, { ...backup, text }, type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send service event notification (container up/down, deploy success/fail)
|
||||
*/
|
||||
async sendServiceEvent(event, service) {
|
||||
const eventMap = {
|
||||
'container-up': { type: 'success', text: `✅ Container "${service.containerName || service.name}" is now UP` },
|
||||
'container-down': { type: 'error', text: `🔴 Container "${service.containerName || service.name}" is DOWN` },
|
||||
'deploy-success': { type: 'success', text: `✅ "${service.name}" deployed successfully` },
|
||||
'deploy-failed': { type: 'error', text: `❌ "${service.name}" deployment failed` },
|
||||
'auto-restart': { type: 'warning', text: `🔄 Container "${service.containerName || service.name}" auto-restarted` }
|
||||
};
|
||||
|
||||
const info = eventMap[event] || { type: 'info', text: `Service event: ${event}` };
|
||||
return this.send(event, { ...service, text: info.text }, info.type);
|
||||
}
|
||||
|
||||
// ===== Helper Methods =====
|
||||
|
||||
_formatTitle(event) {
|
||||
const titles = {
|
||||
'container-down': 'Container Down',
|
||||
'container-up': 'Container Recovered',
|
||||
'alert': 'Resource Alert',
|
||||
'backup-complete': 'Backup Complete',
|
||||
'backup-failed': 'Backup Failed',
|
||||
'update-available': 'Update Available',
|
||||
'test': 'Test Notification',
|
||||
'auto-restart': 'Auto-Restart',
|
||||
'deploy-success': 'Deployment Success',
|
||||
'deploy-failed': 'Deployment Failed'
|
||||
};
|
||||
return titles[event] || 'DashCaddy Notification';
|
||||
}
|
||||
|
||||
_formatText(data, event) {
|
||||
if (typeof data === 'string') return data;
|
||||
return data.text || data.message || this._formatTitle(event);
|
||||
}
|
||||
|
||||
_formatEmbed(data, event, type) {
|
||||
if (typeof data === 'string') return null;
|
||||
if (data.embed) return data.embed;
|
||||
|
||||
return {
|
||||
title: this._formatTitle(event),
|
||||
description: data.text || data.message || '',
|
||||
color: this._getTypeColor(type),
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
_formatAlertText(alert) {
|
||||
const lines = [
|
||||
`**${alert.containerName}**`,
|
||||
'',
|
||||
...alert.alerts.map(a => `• ${a.message}`)
|
||||
];
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
_getAlertColor(alerts) {
|
||||
if (alerts.some(a => a.severity === 'critical')) return 15158332; // Red
|
||||
if (alerts.some(a => a.severity === 'warning')) return 16776960; // Yellow
|
||||
return 3447003; // Blue
|
||||
}
|
||||
|
||||
_getTypeColor(type) {
|
||||
const colors = {
|
||||
success: 3066993, // Green
|
||||
error: 15158332, // Red
|
||||
warning: 16776960, // Yellow
|
||||
info: 3447003 // Blue
|
||||
};
|
||||
return colors[type] || colors.info;
|
||||
}
|
||||
|
||||
// ===== Health Check Daemon =====
|
||||
|
||||
startHealthDaemon() {
|
||||
if (this.healthDaemonInterval) return;
|
||||
|
||||
const interval = (this.config.healthCheck?.intervalMinutes || 5) * 60 * 1000;
|
||||
this.healthDaemonInterval = setInterval(() => {
|
||||
this.checkHealth().catch(err => {
|
||||
this.log.error('notification', 'Health check failed', { error: err.message });
|
||||
});
|
||||
}, interval);
|
||||
|
||||
this.log.info('notification', 'Health daemon started', { intervalMinutes: this.config.healthCheck?.intervalMinutes });
|
||||
}
|
||||
|
||||
stopHealthDaemon() {
|
||||
if (this.healthDaemonInterval) {
|
||||
clearInterval(this.healthDaemonInterval);
|
||||
this.healthDaemonInterval = null;
|
||||
this.log.info('notification', 'Health daemon stopped');
|
||||
}
|
||||
}
|
||||
|
||||
async checkHealth() {
|
||||
if (!this.config.healthCheck?.enabled || !this.ctx.docker) {
|
||||
return { checked: false };
|
||||
}
|
||||
|
||||
try {
|
||||
const containers = await this.ctx.docker.listContainers({ all: true });
|
||||
const previousState = new Map(this.healthState);
|
||||
|
||||
for (const container of containers) {
|
||||
const name = container.Names[0]?.replace(/^\//, '') || container.Id.substring(0, 12);
|
||||
const wasDown = previousState.get(container.Id) === false;
|
||||
const isDown = container.State !== 'running';
|
||||
|
||||
this.healthState.set(container.Id, isDown);
|
||||
|
||||
if (wasDown && !isDown) {
|
||||
// Container recovered
|
||||
await this.sendServiceEvent('container-up', {
|
||||
containerId: container.Id,
|
||||
containerName: name,
|
||||
state: container.State
|
||||
});
|
||||
} else if (!wasDown && isDown) {
|
||||
// Container went down
|
||||
await this.sendServiceEvent('container-down', {
|
||||
containerId: container.Id,
|
||||
containerName: name,
|
||||
state: container.State
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Update last check time
|
||||
this.config.healthCheck = this.config.healthCheck || {};
|
||||
this.config.healthCheck.lastCheck = new Date().toISOString();
|
||||
await this.saveConfig();
|
||||
|
||||
return {
|
||||
checked: true,
|
||||
containersMonitored: containers.length,
|
||||
lastCheck: this.config.healthCheck.lastCheck
|
||||
};
|
||||
} catch (error) {
|
||||
this.log.error('notification', 'Health check error', { error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
getHealthState() {
|
||||
return new Map(this.healthState);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = NotificationManager;
|
||||
@@ -0,0 +1,235 @@
|
||||
/**
|
||||
* Port Lock Manager
|
||||
* Provides atomic port allocation using file-based locks to prevent race conditions
|
||||
* during concurrent container deployments
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const lockfile = require('proper-lockfile');
|
||||
|
||||
const LOCK_DIR = path.join(__dirname, '.port-locks');
|
||||
const LOCK_TIMEOUT = 120000; // 2 minutes
|
||||
const LOCK_STALE_THRESHOLD = 120000; // 2 minutes
|
||||
const LOCK_RETRY_OPTIONS = {
|
||||
retries: {
|
||||
retries: 10,
|
||||
minTimeout: 100,
|
||||
maxTimeout: 1000,
|
||||
randomize: true
|
||||
},
|
||||
stale: LOCK_STALE_THRESHOLD,
|
||||
realpath: false
|
||||
};
|
||||
|
||||
class PortLockManager {
|
||||
constructor() {
|
||||
this.activeLocks = new Map(); // Map of lockId -> { ports: [], release: fn }
|
||||
this.ensureLockDirectory();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure lock directory exists
|
||||
*/
|
||||
ensureLockDirectory() {
|
||||
if (!fs.existsSync(LOCK_DIR)) {
|
||||
fs.mkdirSync(LOCK_DIR, { recursive: true });
|
||||
console.log('[PortLockManager] Created lock directory:', LOCK_DIR);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get lock file path for a port
|
||||
*/
|
||||
getLockFilePath(port) {
|
||||
return path.join(LOCK_DIR, `port-${port}.lock`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire locks for multiple ports atomically
|
||||
* Ports are sorted to prevent deadlocks
|
||||
* @param {string[]} ports - Array of port numbers as strings
|
||||
* @returns {Promise<string>} Lock ID for releasing locks later
|
||||
*/
|
||||
async acquirePorts(ports) {
|
||||
if (!Array.isArray(ports) || ports.length === 0) {
|
||||
throw new Error('Ports must be a non-empty array');
|
||||
}
|
||||
|
||||
const lockId = `lock-${Date.now()}-${Math.random().toString(36).substring(7)}`;
|
||||
const sortedPorts = [...new Set(ports)].sort((a, b) => parseInt(a) - parseInt(b));
|
||||
const acquiredLocks = [];
|
||||
const releaseFunctions = [];
|
||||
|
||||
try {
|
||||
console.log(`[PortLockManager] Acquiring locks for ports: ${sortedPorts.join(', ')}`);
|
||||
|
||||
// Acquire locks in sorted order to prevent deadlocks
|
||||
for (const port of sortedPorts) {
|
||||
const lockFilePath = this.getLockFilePath(port);
|
||||
|
||||
// Create lock file if it doesn't exist
|
||||
if (!fs.existsSync(lockFilePath)) {
|
||||
fs.writeFileSync(lockFilePath, JSON.stringify({
|
||||
created: new Date().toISOString(),
|
||||
port
|
||||
}));
|
||||
}
|
||||
|
||||
// Acquire lock with retry
|
||||
const release = await lockfile.lock(lockFilePath, LOCK_RETRY_OPTIONS);
|
||||
|
||||
acquiredLocks.push(port);
|
||||
releaseFunctions.push(release);
|
||||
|
||||
console.log(`[PortLockManager] Locked port ${port}`);
|
||||
}
|
||||
|
||||
// Store lock information
|
||||
this.activeLocks.set(lockId, {
|
||||
ports: sortedPorts,
|
||||
releases: releaseFunctions,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
console.log(`[PortLockManager] Successfully acquired all locks (ID: ${lockId})`);
|
||||
return lockId;
|
||||
|
||||
} catch (error) {
|
||||
// Release any locks we managed to acquire
|
||||
console.error(`[PortLockManager] Failed to acquire all locks:`, error.message);
|
||||
|
||||
for (const release of releaseFunctions) {
|
||||
try {
|
||||
await release();
|
||||
} catch (releaseError) {
|
||||
console.error(`[PortLockManager] Error releasing lock during cleanup:`, releaseError.message);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Failed to acquire port locks: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Release locks for a lock ID
|
||||
* @param {string} lockId - Lock ID returned from acquirePorts
|
||||
*/
|
||||
async releasePorts(lockId) {
|
||||
const lockInfo = this.activeLocks.get(lockId);
|
||||
|
||||
if (!lockInfo) {
|
||||
console.warn(`[PortLockManager] Lock ID ${lockId} not found (may have been released already)`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[PortLockManager] Releasing locks for ports: ${lockInfo.ports.join(', ')}`);
|
||||
|
||||
const errors = [];
|
||||
|
||||
for (const release of lockInfo.releases) {
|
||||
try {
|
||||
await release();
|
||||
} catch (error) {
|
||||
errors.push(error.message);
|
||||
console.error(`[PortLockManager] Error releasing lock:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
this.activeLocks.delete(lockId);
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.warn(`[PortLockManager] Released locks with ${errors.length} errors`);
|
||||
} else {
|
||||
console.log(`[PortLockManager] Successfully released all locks (ID: ${lockId})`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up stale lock files
|
||||
* Removes locks older than LOCK_STALE_THRESHOLD
|
||||
*/
|
||||
async cleanupStaleLocks() {
|
||||
console.log('[PortLockManager] Cleaning up stale locks...');
|
||||
|
||||
this.ensureLockDirectory();
|
||||
|
||||
let cleaned = 0;
|
||||
let errors = 0;
|
||||
|
||||
try {
|
||||
const files = fs.readdirSync(LOCK_DIR);
|
||||
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.lock')) continue;
|
||||
|
||||
const lockFilePath = path.join(LOCK_DIR, file);
|
||||
|
||||
try {
|
||||
// Check if lock is stale using proper-lockfile's built-in check
|
||||
const isLocked = await lockfile.check(lockFilePath, { realpath: false, stale: LOCK_STALE_THRESHOLD });
|
||||
|
||||
if (!isLocked) {
|
||||
// Lock is stale or not locked, safe to remove
|
||||
fs.unlinkSync(lockFilePath);
|
||||
cleaned++;
|
||||
console.log(`[PortLockManager] Removed stale lock: ${file}`);
|
||||
}
|
||||
} catch (error) {
|
||||
// File might not exist or might have been removed by another process
|
||||
if (error.code !== 'ENOENT') {
|
||||
errors++;
|
||||
console.warn(`[PortLockManager] Error checking lock ${file}:`, error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[PortLockManager] Cleanup complete: ${cleaned} stale locks removed, ${errors} errors`);
|
||||
} catch (error) {
|
||||
console.error('[PortLockManager] Error during cleanup:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current lock status
|
||||
*/
|
||||
getStatus() {
|
||||
const activeLocks = Array.from(this.activeLocks.entries()).map(([lockId, info]) => ({
|
||||
lockId,
|
||||
ports: info.ports,
|
||||
age: Date.now() - info.timestamp,
|
||||
timestamp: new Date(info.timestamp).toISOString()
|
||||
}));
|
||||
|
||||
return {
|
||||
activeLocks: activeLocks.length,
|
||||
locks: activeLocks,
|
||||
lockDirectory: LOCK_DIR
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a port is currently locked
|
||||
* @param {string} port - Port number as string
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async isPortLocked(port) {
|
||||
const lockFilePath = this.getLockFilePath(port);
|
||||
|
||||
if (!fs.existsSync(lockFilePath)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
return await lockfile.check(lockFilePath, { realpath: false, stale: LOCK_STALE_THRESHOLD });
|
||||
} catch (error) {
|
||||
// If we can't check, assume it's not locked
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton instance
|
||||
const portLockManager = new PortLockManager();
|
||||
|
||||
module.exports = portLockManager;
|
||||
@@ -0,0 +1,987 @@
|
||||
/**
|
||||
* Container Resource Monitoring Module
|
||||
* Tracks CPU, memory, disk, and network usage for Docker containers
|
||||
* Provides alerts and historical data
|
||||
*/
|
||||
|
||||
const Docker = require('dockerode');
|
||||
const EventEmitter = require('events');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const docker = new Docker();
|
||||
|
||||
// Configuration
|
||||
const STATS_FILE = process.env.STATS_FILE || path.join(__dirname, 'container-stats.json');
|
||||
const STATS_HOURLY_FILE = process.env.STATS_HOURLY_FILE || path.join(__dirname, 'container-stats-hourly.json');
|
||||
const STATS_DAILY_FILE = process.env.STATS_DAILY_FILE || path.join(__dirname, 'container-stats-daily.json');
|
||||
const ALERT_CONFIG_FILE = process.env.ALERT_CONFIG_FILE || path.join(__dirname, 'alert-config.json');
|
||||
const ALERT_HISTORY_FILE = process.env.ALERT_HISTORY_FILE || path.join(__dirname, 'alert-history.json');
|
||||
const STATS_RETENTION_HOURS = parseInt(process.env.STATS_RETENTION_HOURS || '168', 10); // 7 days raw
|
||||
const STATS_HOURLY_RETENTION_DAYS = parseInt(process.env.STATS_HOURLY_RETENTION_DAYS || '30', 10); // 30 days hourly
|
||||
const STATS_DAILY_RETENTION_DAYS = parseInt(process.env.STATS_DAILY_RETENTION_DAYS || '365', 10); // 365 days daily
|
||||
const MONITORING_INTERVAL = parseInt(process.env.MONITORING_INTERVAL || '10000', 10); // 10 seconds
|
||||
const ROLLUP_HOURLY_INTERVAL = parseInt(process.env.ROLLUP_HOURLY_INTERVAL || String(60 * 60 * 1000), 10); // 1h
|
||||
const ROLLUP_DAILY_INTERVAL = parseInt(process.env.ROLLUP_DAILY_INTERVAL || String(24 * 60 * 60 * 1000), 10); // 24h
|
||||
|
||||
class ResourceMonitor extends EventEmitter {
|
||||
constructor() {
|
||||
super();
|
||||
this.monitoring = false;
|
||||
this.monitoringInterval = null;
|
||||
this.hourlyRollupTimer = null;
|
||||
this.dailyRollupTimer = null;
|
||||
this.stats = new Map(); // containerId -> { name, history: [...] } (raw 10s samples, 7d)
|
||||
this.hourlyHistory = new Map(); // containerId -> { name, samples: [...] } (hourly avg, 30d)
|
||||
this.dailyHistory = new Map(); // containerId -> { name, samples: [...] } (daily avg, 365d)
|
||||
this.alerts = new Map(); // containerId -> alert config
|
||||
this.lastAlerts = new Map(); // containerId -> last alert timestamp
|
||||
this.alertHistory = []; // alert history entries
|
||||
this.notificationManager = null;
|
||||
|
||||
this.loadStats();
|
||||
this.loadHourlyStats();
|
||||
this.loadDailyStats();
|
||||
this.loadAlertConfig();
|
||||
this.loadAlertHistory();
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the notification manager for sending alerts
|
||||
*/
|
||||
setNotificationManager(nm) {
|
||||
this.notificationManager = nm;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start monitoring all containers
|
||||
*/
|
||||
start() {
|
||||
if (this.monitoring) {
|
||||
console.log('[ResourceMonitor] Already monitoring');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('[ResourceMonitor] Starting container monitoring');
|
||||
this.monitoring = true;
|
||||
this.monitoringInterval = setInterval(() => this.collectStats(), MONITORING_INTERVAL);
|
||||
|
||||
// Hourly rollup — fires once an hour, computes the previous full hour
|
||||
this.hourlyRollupTimer = setInterval(() => {
|
||||
try { this.rollupHourly(); } catch (e) { console.error('[ResourceMonitor] hourly rollup error:', e.message); }
|
||||
}, ROLLUP_HOURLY_INTERVAL);
|
||||
|
||||
// Daily rollup — schedule first run at the next midnight, then fire every 24h
|
||||
const now = new Date();
|
||||
const nextMidnight = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, 0, 0, 5);
|
||||
const msUntilMidnight = nextMidnight.getTime() - now.getTime();
|
||||
setTimeout(() => {
|
||||
try { this.rollupDaily(); } catch (e) { console.error('[ResourceMonitor] daily rollup error:', e.message); }
|
||||
this.dailyRollupTimer = setInterval(() => {
|
||||
try { this.rollupDaily(); } catch (e) { console.error('[ResourceMonitor] daily rollup error:', e.message); }
|
||||
}, ROLLUP_DAILY_INTERVAL);
|
||||
}, msUntilMidnight);
|
||||
|
||||
// Initial collection
|
||||
this.collectStats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop monitoring
|
||||
*/
|
||||
stop() {
|
||||
if (!this.monitoring) return;
|
||||
|
||||
console.log('[ResourceMonitor] Stopping container monitoring');
|
||||
this.monitoring = false;
|
||||
|
||||
if (this.monitoringInterval) {
|
||||
clearInterval(this.monitoringInterval);
|
||||
this.monitoringInterval = null;
|
||||
}
|
||||
if (this.hourlyRollupTimer) {
|
||||
clearInterval(this.hourlyRollupTimer);
|
||||
this.hourlyRollupTimer = null;
|
||||
}
|
||||
if (this.dailyRollupTimer) {
|
||||
clearInterval(this.dailyRollupTimer);
|
||||
this.dailyRollupTimer = null;
|
||||
}
|
||||
|
||||
this.saveStats();
|
||||
this.saveHourlyStats();
|
||||
this.saveDailyStats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect stats from all running containers
|
||||
*/
|
||||
async collectStats() {
|
||||
try {
|
||||
const containers = await docker.listContainers({ all: false });
|
||||
|
||||
for (const containerInfo of containers) {
|
||||
try {
|
||||
const container = docker.getContainer(containerInfo.Id);
|
||||
const stats = await this.getContainerStats(container);
|
||||
|
||||
if (stats) {
|
||||
this.recordStats(containerInfo.Id, containerInfo.Names[0], stats);
|
||||
this.checkAlerts(containerInfo.Id, containerInfo.Names[0], stats);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[ResourceMonitor] Error collecting stats for ${containerInfo.Names[0]}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup old stats
|
||||
this.cleanupOldStats();
|
||||
|
||||
// Persist stats periodically
|
||||
if (Math.random() < 0.1) { // 10% chance to save (every ~100 seconds)
|
||||
this.saveStats();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error collecting container stats:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get stats for a single container
|
||||
*/
|
||||
async getContainerStats(container) {
|
||||
return new Promise((resolve, reject) => {
|
||||
container.stats({ stream: false }, (err, stats) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
|
||||
// Calculate CPU percentage
|
||||
const cpuDelta = stats.cpu_stats.cpu_usage.total_usage -
|
||||
(stats.precpu_stats.cpu_usage?.total_usage || 0);
|
||||
const systemDelta = stats.cpu_stats.system_cpu_usage -
|
||||
(stats.precpu_stats.system_cpu_usage || 0);
|
||||
const cpuPercent = systemDelta > 0 ? (cpuDelta / systemDelta) * 100 : 0;
|
||||
|
||||
// Calculate memory usage
|
||||
const memoryUsage = stats.memory_stats.usage || 0;
|
||||
const memoryLimit = stats.memory_stats.limit || 0;
|
||||
const memoryPercent = memoryLimit > 0 ? (memoryUsage / memoryLimit) * 100 : 0;
|
||||
|
||||
// Calculate network I/O
|
||||
let networkRx = 0;
|
||||
let networkTx = 0;
|
||||
if (stats.networks) {
|
||||
Object.values(stats.networks).forEach(net => {
|
||||
networkRx += net.rx_bytes || 0;
|
||||
networkTx += net.tx_bytes || 0;
|
||||
});
|
||||
}
|
||||
|
||||
// Calculate block I/O
|
||||
let blockRead = 0;
|
||||
let blockWrite = 0;
|
||||
if (stats.blkio_stats?.io_service_bytes_recursive) {
|
||||
stats.blkio_stats.io_service_bytes_recursive.forEach(io => {
|
||||
if (io.op === 'Read') blockRead += io.value;
|
||||
if (io.op === 'Write') blockWrite += io.value;
|
||||
});
|
||||
}
|
||||
|
||||
resolve({
|
||||
timestamp: new Date().toISOString(),
|
||||
cpu: {
|
||||
percent: Math.round(cpuPercent * 100) / 100,
|
||||
usage: stats.cpu_stats.cpu_usage.total_usage
|
||||
},
|
||||
memory: {
|
||||
usage: memoryUsage,
|
||||
limit: memoryLimit,
|
||||
percent: Math.round(memoryPercent * 100) / 100,
|
||||
usageMB: Math.round(memoryUsage / 1024 / 1024),
|
||||
limitMB: Math.round(memoryLimit / 1024 / 1024)
|
||||
},
|
||||
network: {
|
||||
rxBytes: networkRx,
|
||||
txBytes: networkTx,
|
||||
rxMB: Math.round(networkRx / 1024 / 1024 * 100) / 100,
|
||||
txMB: Math.round(networkTx / 1024 / 1024 * 100) / 100
|
||||
},
|
||||
disk: {
|
||||
readBytes: blockRead,
|
||||
writeBytes: blockWrite,
|
||||
readMB: Math.round(blockRead / 1024 / 1024 * 100) / 100,
|
||||
writeMB: Math.round(blockWrite / 1024 / 1024 * 100) / 100
|
||||
},
|
||||
pids: stats.pids_stats?.current || 0
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Record stats for a container
|
||||
*/
|
||||
recordStats(containerId, containerName, stats) {
|
||||
if (!this.stats.has(containerId)) {
|
||||
this.stats.set(containerId, {
|
||||
name: containerName,
|
||||
history: []
|
||||
});
|
||||
}
|
||||
|
||||
const containerStats = this.stats.get(containerId);
|
||||
containerStats.name = containerName; // Update name in case it changed
|
||||
containerStats.history.push(stats);
|
||||
|
||||
// Keep only recent stats (based on retention policy)
|
||||
const cutoffTime = Date.now() - (STATS_RETENTION_HOURS * 60 * 60 * 1000);
|
||||
containerStats.history = containerStats.history.filter(s =>
|
||||
new Date(s.timestamp).getTime() > cutoffTime
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if any alerts should be triggered
|
||||
*/
|
||||
checkAlerts(containerId, containerName, stats) {
|
||||
const alertConfig = this.alerts.get(containerId);
|
||||
if (!alertConfig || !alertConfig.enabled) return;
|
||||
|
||||
const now = Date.now();
|
||||
const lastAlert = this.lastAlerts.get(containerId) || 0;
|
||||
const cooldown = (alertConfig.cooldownMinutes || 15) * 60 * 1000;
|
||||
|
||||
// Don't spam alerts - respect cooldown period
|
||||
if (now - lastAlert < cooldown) return;
|
||||
|
||||
const alerts = [];
|
||||
|
||||
// Check CPU threshold
|
||||
if (alertConfig.cpuThreshold && stats.cpu.percent > alertConfig.cpuThreshold) {
|
||||
alerts.push({
|
||||
type: 'cpu',
|
||||
severity: 'warning',
|
||||
message: `CPU usage ${stats.cpu.percent.toFixed(1)}% exceeds threshold ${alertConfig.cpuThreshold}%`,
|
||||
value: stats.cpu.percent,
|
||||
threshold: alertConfig.cpuThreshold
|
||||
});
|
||||
}
|
||||
|
||||
// Check memory threshold
|
||||
if (alertConfig.memoryThreshold && stats.memory.percent > alertConfig.memoryThreshold) {
|
||||
alerts.push({
|
||||
type: 'memory',
|
||||
severity: 'warning',
|
||||
message: `Memory usage ${stats.memory.percent.toFixed(1)}% exceeds threshold ${alertConfig.memoryThreshold}%`,
|
||||
value: stats.memory.percent,
|
||||
threshold: alertConfig.memoryThreshold
|
||||
});
|
||||
}
|
||||
|
||||
// Check disk I/O threshold (MB/s)
|
||||
if (alertConfig.diskIOThreshold) {
|
||||
const diskIO = stats.disk.readMB + stats.disk.writeMB;
|
||||
if (diskIO > alertConfig.diskIOThreshold) {
|
||||
alerts.push({
|
||||
type: 'disk',
|
||||
severity: 'warning',
|
||||
message: `Disk I/O ${diskIO.toFixed(1)} MB/s exceeds threshold ${alertConfig.diskIOThreshold} MB/s`,
|
||||
value: diskIO,
|
||||
threshold: alertConfig.diskIOThreshold
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (alerts.length > 0) {
|
||||
this.lastAlerts.set(containerId, now);
|
||||
|
||||
// Add alert history entries
|
||||
for (const alert of alerts) {
|
||||
this.addAlertHistoryEntry({
|
||||
id: `${containerId}-${Date.now()}-${alert.type}`,
|
||||
timestamp: new Date().toISOString(),
|
||||
containerId,
|
||||
containerName,
|
||||
type: alert.type,
|
||||
metric: alert.type,
|
||||
value: alert.value,
|
||||
threshold: alert.threshold,
|
||||
severity: alert.severity,
|
||||
notified: !!this.notificationManager,
|
||||
autoRestartTriggered: !!alertConfig.autoRestart
|
||||
});
|
||||
}
|
||||
|
||||
const alertPayload = {
|
||||
containerId,
|
||||
containerName,
|
||||
timestamp: new Date().toISOString(),
|
||||
alerts,
|
||||
stats,
|
||||
config: alertConfig
|
||||
};
|
||||
|
||||
this.emit('alert', alertPayload);
|
||||
|
||||
// Send notification if manager is configured
|
||||
if (this.notificationManager) {
|
||||
this.notificationManager.sendAlert(alertPayload).catch(err => {
|
||||
console.error('[ResourceMonitor] Failed to send alert notification:', err.message);
|
||||
});
|
||||
}
|
||||
|
||||
// Auto-restart if configured
|
||||
if (alertConfig.autoRestart) {
|
||||
this.restartContainer(containerId, containerName, alerts);
|
||||
}
|
||||
|
||||
// Trigger bundled workflows for resource-alert
|
||||
this.triggerWorkflows('resource-alert', {
|
||||
containerId,
|
||||
containerName,
|
||||
alerts,
|
||||
stats,
|
||||
diskPercent: (stats.disk?.readBytes + stats.disk?.writeBytes) > 0
|
||||
? Math.round((stats.disk.readBytes / (stats.disk.readBytes + stats.disk.writeBytes)) * 100)
|
||||
: 0,
|
||||
host: require('os').hostname()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Restart a container due to resource alerts
|
||||
*/
|
||||
async restartContainer(containerId, containerName, alerts) {
|
||||
try {
|
||||
console.log(`[ResourceMonitor] Auto-restarting ${containerName} due to alerts:`, alerts.map(a => a.type).join(', '));
|
||||
|
||||
const container = docker.getContainer(containerId);
|
||||
await container.restart();
|
||||
|
||||
this.emit('auto-restart', {
|
||||
containerId,
|
||||
containerName,
|
||||
timestamp: new Date().toISOString(),
|
||||
reason: alerts
|
||||
});
|
||||
|
||||
// Send notification if manager is configured
|
||||
if (this.notificationManager) {
|
||||
this.notificationManager.send('auto-restart', {
|
||||
containerId,
|
||||
containerName,
|
||||
timestamp: new Date().toISOString(),
|
||||
reason: alerts
|
||||
}).catch(err => {
|
||||
console.error('[ResourceMonitor] Failed to send auto-restart notification:', err.message);
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`[ResourceMonitor] Failed to restart ${containerName}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger bundled workflows for an event
|
||||
*/
|
||||
triggerWorkflows(eventType, eventData) {
|
||||
if (!this.workflowEngine) {
|
||||
console.log('[ResourceMonitor] Workflow engine not set, skipping workflow trigger');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
this.workflowEngine.triggerForEvent(eventType, eventData)
|
||||
.then(results => {
|
||||
if (results && results.length > 0) {
|
||||
console.log(`[ResourceMonitor] Triggered ${results.length} workflow(s) for ${eventType}`);
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
console.error('[ResourceMonitor] Workflow trigger error:', err.message);
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error triggering workflows:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the workflow engine for triggering workflows
|
||||
*/
|
||||
setWorkflowEngine(workflowEngine) {
|
||||
this.workflowEngine = workflowEngine;
|
||||
console.log('[ResourceMonitor] Workflow engine configured');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current stats for a container
|
||||
*/
|
||||
getCurrentStats(containerId) {
|
||||
const containerStats = this.stats.get(containerId);
|
||||
if (!containerStats || containerStats.history.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return containerStats.history[containerStats.history.length - 1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get historical stats for a container
|
||||
*/
|
||||
getHistoricalStats(containerId, hours = 24) {
|
||||
const containerStats = this.stats.get(containerId);
|
||||
if (!containerStats) return [];
|
||||
|
||||
const cutoffTime = Date.now() - (hours * 60 * 60 * 1000);
|
||||
return containerStats.history.filter(s =>
|
||||
new Date(s.timestamp).getTime() > cutoffTime
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get aggregated stats for a container
|
||||
*/
|
||||
getAggregatedStats(containerId, hours = 24) {
|
||||
const history = this.getHistoricalStats(containerId, hours);
|
||||
if (history.length === 0) return null;
|
||||
|
||||
const cpuValues = history.map(s => s.cpu.percent);
|
||||
const memoryValues = history.map(s => s.memory.percent);
|
||||
|
||||
return {
|
||||
cpu: {
|
||||
current: cpuValues[cpuValues.length - 1],
|
||||
avg: cpuValues.reduce((a, b) => a + b, 0) / cpuValues.length,
|
||||
max: Math.max(...cpuValues),
|
||||
min: Math.min(...cpuValues)
|
||||
},
|
||||
memory: {
|
||||
current: memoryValues[memoryValues.length - 1],
|
||||
avg: memoryValues.reduce((a, b) => a + b, 0) / memoryValues.length,
|
||||
max: Math.max(...memoryValues),
|
||||
min: Math.min(...memoryValues)
|
||||
},
|
||||
dataPoints: history.length,
|
||||
timeRange: hours
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get stats for all containers
|
||||
*/
|
||||
getAllStats() {
|
||||
const result = {};
|
||||
|
||||
for (const [containerId, data] of this.stats.entries()) {
|
||||
const current = this.getCurrentStats(containerId);
|
||||
const aggregated = this.getAggregatedStats(containerId, 24);
|
||||
|
||||
result[containerId] = {
|
||||
name: data.name,
|
||||
current,
|
||||
aggregated,
|
||||
alertConfig: this.alerts.get(containerId)
|
||||
};
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure alerts for a container
|
||||
*/
|
||||
setAlertConfig(containerId, config) {
|
||||
this.alerts.set(containerId, {
|
||||
enabled: config.enabled !== false,
|
||||
cpuThreshold: config.cpuThreshold || null,
|
||||
memoryThreshold: config.memoryThreshold || null,
|
||||
diskIOThreshold: config.diskIOThreshold || null,
|
||||
cooldownMinutes: config.cooldownMinutes || 15,
|
||||
autoRestart: config.autoRestart || false,
|
||||
notificationChannels: config.notificationChannels || []
|
||||
});
|
||||
|
||||
this.saveAlertConfig();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get alert configuration for a container
|
||||
*/
|
||||
getAlertConfig(containerId) {
|
||||
return this.alerts.get(containerId) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove alert configuration
|
||||
*/
|
||||
removeAlertConfig(containerId) {
|
||||
this.alerts.delete(containerId);
|
||||
this.lastAlerts.delete(containerId);
|
||||
this.saveAlertConfig();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all alert configurations
|
||||
*/
|
||||
getAllAlertConfigs() {
|
||||
const configs = {};
|
||||
for (const [containerId, config] of this.alerts.entries()) {
|
||||
configs[containerId] = config;
|
||||
}
|
||||
return configs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add entry to alert history
|
||||
*/
|
||||
addAlertHistoryEntry(entry) {
|
||||
this.alertHistory.unshift(entry);
|
||||
// Keep only last 1000 entries
|
||||
if (this.alertHistory.length > 1000) {
|
||||
this.alertHistory = this.alertHistory.slice(0, 1000);
|
||||
}
|
||||
this.saveAlertHistory();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get alert history
|
||||
*/
|
||||
getAlertHistory(limit = 50) {
|
||||
return this.alertHistory.slice(0, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load alert history from disk
|
||||
*/
|
||||
loadAlertHistory() {
|
||||
try {
|
||||
if (fs.existsSync(ALERT_HISTORY_FILE)) {
|
||||
const data = JSON.parse(fs.readFileSync(ALERT_HISTORY_FILE, 'utf8'));
|
||||
this.alertHistory = Array.isArray(data) ? data : [];
|
||||
console.log(`[ResourceMonitor] Loaded ${this.alertHistory.length} alert history entries`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error loading alert history:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save alert history to disk
|
||||
*/
|
||||
saveAlertHistory() {
|
||||
try {
|
||||
fs.writeFileSync(ALERT_HISTORY_FILE, JSON.stringify(this.alertHistory, null, 2));
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error saving alert history:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup old stats beyond retention period
|
||||
*/
|
||||
cleanupOldStats() {
|
||||
const cutoffTime = Date.now() - (STATS_RETENTION_HOURS * 60 * 60 * 1000);
|
||||
|
||||
for (const [containerId, data] of this.stats.entries()) {
|
||||
data.history = data.history.filter(s =>
|
||||
new Date(s.timestamp).getTime() > cutoffTime
|
||||
);
|
||||
|
||||
// Remove container stats if no recent data
|
||||
if (data.history.length === 0) {
|
||||
this.stats.delete(containerId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load stats from disk
|
||||
*/
|
||||
loadStats() {
|
||||
try {
|
||||
if (fs.existsSync(STATS_FILE)) {
|
||||
const data = JSON.parse(fs.readFileSync(STATS_FILE, 'utf8'));
|
||||
this.stats = new Map(Object.entries(data));
|
||||
console.log(`[ResourceMonitor] Loaded stats for ${this.stats.size} containers`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error loading stats:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save stats to disk
|
||||
*/
|
||||
saveStats() {
|
||||
try {
|
||||
const data = Object.fromEntries(this.stats);
|
||||
fs.writeFileSync(STATS_FILE, JSON.stringify(data, null, 2));
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error saving stats:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load alert configuration from disk
|
||||
*/
|
||||
loadAlertConfig() {
|
||||
try {
|
||||
if (fs.existsSync(ALERT_CONFIG_FILE)) {
|
||||
const data = JSON.parse(fs.readFileSync(ALERT_CONFIG_FILE, 'utf8'));
|
||||
this.alerts = new Map(Object.entries(data));
|
||||
console.log(`[ResourceMonitor] Loaded alert config for ${this.alerts.size} containers`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error loading alert config:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save alert configuration to disk
|
||||
*/
|
||||
saveAlertConfig() {
|
||||
try {
|
||||
const data = Object.fromEntries(this.alerts);
|
||||
fs.writeFileSync(ALERT_CONFIG_FILE, JSON.stringify(data, null, 2));
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error saving alert config:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate a list of raw samples into a single rollup sample
|
||||
* @param {Array} samples - Raw stats samples
|
||||
* @param {string} timestamp - ISO timestamp to use for the rollup bucket
|
||||
* @returns {Object|null} Aggregated sample, or null if input is empty
|
||||
*/
|
||||
_aggregateSamples(samples, timestamp) {
|
||||
if (!samples || samples.length === 0) return null;
|
||||
|
||||
let cpuSum = 0, cpuMax = 0;
|
||||
let memSum = 0, memMax = 0;
|
||||
let memPctSum = 0, memPctMax = 0;
|
||||
let netRxSum = 0, netTxSum = 0;
|
||||
let diskRSum = 0, diskWSum = 0;
|
||||
|
||||
for (const s of samples) {
|
||||
const cpu = s.cpu?.percent || 0;
|
||||
const memUsage = s.memory?.usage || 0;
|
||||
const memPct = s.memory?.percent || 0;
|
||||
cpuSum += cpu; if (cpu > cpuMax) cpuMax = cpu;
|
||||
memSum += memUsage; if (memUsage > memMax) memMax = memUsage;
|
||||
memPctSum += memPct; if (memPct > memPctMax) memPctMax = memPct;
|
||||
netRxSum += s.network?.rxBytes || 0;
|
||||
netTxSum += s.network?.txBytes || 0;
|
||||
diskRSum += s.disk?.readBytes || 0;
|
||||
diskWSum += s.disk?.writeBytes || 0;
|
||||
}
|
||||
|
||||
const n = samples.length;
|
||||
return {
|
||||
timestamp,
|
||||
sampleCount: n,
|
||||
cpu: {
|
||||
avg: Math.round((cpuSum / n) * 100) / 100,
|
||||
max: Math.round(cpuMax * 100) / 100,
|
||||
},
|
||||
memory: {
|
||||
avgUsage: Math.round(memSum / n),
|
||||
maxUsage: memMax,
|
||||
avgPercent: Math.round((memPctSum / n) * 100) / 100,
|
||||
maxPercent: Math.round(memPctMax * 100) / 100,
|
||||
avgUsageMB: Math.round(memSum / n / 1024 / 1024),
|
||||
maxUsageMB: Math.round(memMax / 1024 / 1024),
|
||||
},
|
||||
network: {
|
||||
rxBytes: netRxSum,
|
||||
txBytes: netTxSum,
|
||||
rxMB: Math.round(netRxSum / 1024 / 1024 * 100) / 100,
|
||||
txMB: Math.round(netTxSum / 1024 / 1024 * 100) / 100,
|
||||
},
|
||||
disk: {
|
||||
readBytes: diskRSum,
|
||||
writeBytes: diskWSum,
|
||||
readMB: Math.round(diskRSum / 1024 / 1024 * 100) / 100,
|
||||
writeMB: Math.round(diskWSum / 1024 / 1024 * 100) / 100,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Combine already-aggregated samples (e.g. hourly buckets) into a single coarser bucket
|
||||
* @param {Array} samples - Aggregated samples (output of _aggregateSamples)
|
||||
* @param {string} timestamp - ISO timestamp to use for the rollup bucket
|
||||
* @returns {Object|null}
|
||||
*/
|
||||
_combineAggregated(samples, timestamp) {
|
||||
if (!samples || samples.length === 0) return null;
|
||||
|
||||
let totalCount = 0;
|
||||
let cpuWeightedSum = 0, cpuMax = 0;
|
||||
let memWeightedSum = 0, memMax = 0;
|
||||
let memPctWeightedSum = 0, memPctMax = 0;
|
||||
let netRxSum = 0, netTxSum = 0;
|
||||
let diskRSum = 0, diskWSum = 0;
|
||||
|
||||
for (const s of samples) {
|
||||
const w = s.sampleCount || 1;
|
||||
totalCount += w;
|
||||
cpuWeightedSum += (s.cpu?.avg || 0) * w;
|
||||
if ((s.cpu?.max || 0) > cpuMax) cpuMax = s.cpu.max;
|
||||
memWeightedSum += (s.memory?.avgUsage || 0) * w;
|
||||
if ((s.memory?.maxUsage || 0) > memMax) memMax = s.memory.maxUsage;
|
||||
memPctWeightedSum += (s.memory?.avgPercent || 0) * w;
|
||||
if ((s.memory?.maxPercent || 0) > memPctMax) memPctMax = s.memory.maxPercent;
|
||||
netRxSum += s.network?.rxBytes || 0;
|
||||
netTxSum += s.network?.txBytes || 0;
|
||||
diskRSum += s.disk?.readBytes || 0;
|
||||
diskWSum += s.disk?.writeBytes || 0;
|
||||
}
|
||||
|
||||
return {
|
||||
timestamp,
|
||||
sampleCount: totalCount,
|
||||
cpu: {
|
||||
avg: Math.round((cpuWeightedSum / totalCount) * 100) / 100,
|
||||
max: Math.round(cpuMax * 100) / 100,
|
||||
},
|
||||
memory: {
|
||||
avgUsage: Math.round(memWeightedSum / totalCount),
|
||||
maxUsage: memMax,
|
||||
avgPercent: Math.round((memPctWeightedSum / totalCount) * 100) / 100,
|
||||
maxPercent: Math.round(memPctMax * 100) / 100,
|
||||
avgUsageMB: Math.round(memWeightedSum / totalCount / 1024 / 1024),
|
||||
maxUsageMB: Math.round(memMax / 1024 / 1024),
|
||||
},
|
||||
network: {
|
||||
rxBytes: netRxSum,
|
||||
txBytes: netTxSum,
|
||||
rxMB: Math.round(netRxSum / 1024 / 1024 * 100) / 100,
|
||||
txMB: Math.round(netTxSum / 1024 / 1024 * 100) / 100,
|
||||
},
|
||||
disk: {
|
||||
readBytes: diskRSum,
|
||||
writeBytes: diskWSum,
|
||||
readMB: Math.round(diskRSum / 1024 / 1024 * 100) / 100,
|
||||
writeMB: Math.round(diskWSum / 1024 / 1024 * 100) / 100,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Roll up the previous complete hour of raw samples into a single hourly point.
|
||||
* Trims hourlyHistory entries older than STATS_HOURLY_RETENTION_DAYS.
|
||||
*/
|
||||
rollupHourly() {
|
||||
const now = new Date();
|
||||
// The "previous complete hour" — bucket starts at top of (current_hour - 1)
|
||||
const bucketStart = new Date(now.getFullYear(), now.getMonth(), now.getDate(), now.getHours() - 1, 0, 0);
|
||||
const bucketEnd = new Date(bucketStart.getTime() + 60 * 60 * 1000);
|
||||
const bucketStartMs = bucketStart.getTime();
|
||||
const bucketEndMs = bucketEnd.getTime();
|
||||
const bucketTimestamp = bucketStart.toISOString();
|
||||
|
||||
for (const [containerId, data] of this.stats.entries()) {
|
||||
const samples = data.history.filter(s => {
|
||||
const t = new Date(s.timestamp).getTime();
|
||||
return t >= bucketStartMs && t < bucketEndMs;
|
||||
});
|
||||
if (samples.length === 0) continue;
|
||||
|
||||
const rollup = this._aggregateSamples(samples, bucketTimestamp);
|
||||
if (!rollup) continue;
|
||||
|
||||
if (!this.hourlyHistory.has(containerId)) {
|
||||
this.hourlyHistory.set(containerId, { name: data.name, samples: [] });
|
||||
}
|
||||
const entry = this.hourlyHistory.get(containerId);
|
||||
entry.name = data.name;
|
||||
// Avoid duplicate buckets if rollup ran twice
|
||||
if (!entry.samples.find(s => s.timestamp === bucketTimestamp)) {
|
||||
entry.samples.push(rollup);
|
||||
}
|
||||
|
||||
// Trim old entries
|
||||
const cutoff = Date.now() - (STATS_HOURLY_RETENTION_DAYS * 24 * 60 * 60 * 1000);
|
||||
entry.samples = entry.samples.filter(s => new Date(s.timestamp).getTime() > cutoff);
|
||||
}
|
||||
|
||||
this.saveHourlyStats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Roll up the previous complete day of hourly samples into a single daily point.
|
||||
* Trims dailyHistory entries older than STATS_DAILY_RETENTION_DAYS.
|
||||
*/
|
||||
rollupDaily() {
|
||||
const now = new Date();
|
||||
// Previous calendar day, midnight to midnight
|
||||
const bucketStart = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1, 0, 0, 0);
|
||||
const bucketEnd = new Date(now.getFullYear(), now.getMonth(), now.getDate(), 0, 0, 0);
|
||||
const bucketStartMs = bucketStart.getTime();
|
||||
const bucketEndMs = bucketEnd.getTime();
|
||||
const bucketTimestamp = bucketStart.toISOString();
|
||||
|
||||
for (const [containerId, data] of this.hourlyHistory.entries()) {
|
||||
const samples = data.samples.filter(s => {
|
||||
const t = new Date(s.timestamp).getTime();
|
||||
return t >= bucketStartMs && t < bucketEndMs;
|
||||
});
|
||||
if (samples.length === 0) continue;
|
||||
|
||||
const rollup = this._combineAggregated(samples, bucketTimestamp);
|
||||
if (!rollup) continue;
|
||||
|
||||
if (!this.dailyHistory.has(containerId)) {
|
||||
this.dailyHistory.set(containerId, { name: data.name, samples: [] });
|
||||
}
|
||||
const entry = this.dailyHistory.get(containerId);
|
||||
entry.name = data.name;
|
||||
if (!entry.samples.find(s => s.timestamp === bucketTimestamp)) {
|
||||
entry.samples.push(rollup);
|
||||
}
|
||||
|
||||
const cutoff = Date.now() - (STATS_DAILY_RETENTION_DAYS * 24 * 60 * 60 * 1000);
|
||||
entry.samples = entry.samples.filter(s => new Date(s.timestamp).getTime() > cutoff);
|
||||
}
|
||||
|
||||
this.saveDailyStats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get history for a container by time range, auto-selecting the appropriate tier.
|
||||
* - <= 24h → raw 10s samples
|
||||
* - 1-30 days → hourly rollups
|
||||
* - > 30 days → daily rollups
|
||||
* @param {string} containerId
|
||||
* @param {number} startTime - epoch ms
|
||||
* @param {number} endTime - epoch ms
|
||||
* @returns {{ tier: 'raw'|'hourly'|'daily', samples: Array, unit: string }}
|
||||
*/
|
||||
getHistoryByRange(containerId, startTime, endTime) {
|
||||
const rangeMs = endTime - startTime;
|
||||
const oneDay = 24 * 60 * 60 * 1000;
|
||||
const thirtyDays = 30 * oneDay;
|
||||
|
||||
let tier, samples;
|
||||
if (rangeMs <= oneDay) {
|
||||
tier = 'raw';
|
||||
const data = this.stats.get(containerId);
|
||||
samples = data ? data.history.filter(s => {
|
||||
const t = new Date(s.timestamp).getTime();
|
||||
return t >= startTime && t <= endTime;
|
||||
}) : [];
|
||||
} else if (rangeMs <= thirtyDays) {
|
||||
tier = 'hourly';
|
||||
const data = this.hourlyHistory.get(containerId);
|
||||
samples = data ? data.samples.filter(s => {
|
||||
const t = new Date(s.timestamp).getTime();
|
||||
return t >= startTime && t <= endTime;
|
||||
}) : [];
|
||||
} else {
|
||||
tier = 'daily';
|
||||
const data = this.dailyHistory.get(containerId);
|
||||
samples = data ? data.samples.filter(s => {
|
||||
const t = new Date(s.timestamp).getTime();
|
||||
return t >= startTime && t <= endTime;
|
||||
}) : [];
|
||||
}
|
||||
|
||||
return { tier, samples, unit: tier === 'raw' ? '10s' : tier === 'hourly' ? '1h' : '1d' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Load hourly rollups from disk
|
||||
*/
|
||||
loadHourlyStats() {
|
||||
try {
|
||||
if (fs.existsSync(STATS_HOURLY_FILE)) {
|
||||
const data = JSON.parse(fs.readFileSync(STATS_HOURLY_FILE, 'utf8'));
|
||||
this.hourlyHistory = new Map(Object.entries(data));
|
||||
console.log(`[ResourceMonitor] Loaded hourly rollups for ${this.hourlyHistory.size} containers`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error loading hourly stats:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save hourly rollups to disk
|
||||
*/
|
||||
saveHourlyStats() {
|
||||
try {
|
||||
const data = Object.fromEntries(this.hourlyHistory);
|
||||
fs.writeFileSync(STATS_HOURLY_FILE, JSON.stringify(data, null, 2));
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error saving hourly stats:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load daily rollups from disk
|
||||
*/
|
||||
loadDailyStats() {
|
||||
try {
|
||||
if (fs.existsSync(STATS_DAILY_FILE)) {
|
||||
const data = JSON.parse(fs.readFileSync(STATS_DAILY_FILE, 'utf8'));
|
||||
this.dailyHistory = new Map(Object.entries(data));
|
||||
console.log(`[ResourceMonitor] Loaded daily rollups for ${this.dailyHistory.size} containers`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error loading daily stats:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save daily rollups to disk
|
||||
*/
|
||||
saveDailyStats() {
|
||||
try {
|
||||
const data = Object.fromEntries(this.dailyHistory);
|
||||
fs.writeFileSync(STATS_DAILY_FILE, JSON.stringify(data, null, 2));
|
||||
} catch (error) {
|
||||
console.error('[ResourceMonitor] Error saving daily stats:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export stats for backup
|
||||
*/
|
||||
exportStats() {
|
||||
return {
|
||||
stats: Object.fromEntries(this.stats),
|
||||
hourlyHistory: Object.fromEntries(this.hourlyHistory),
|
||||
dailyHistory: Object.fromEntries(this.dailyHistory),
|
||||
alerts: Object.fromEntries(this.alerts),
|
||||
exportedAt: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Import stats from backup
|
||||
*/
|
||||
importStats(data) {
|
||||
if (data.stats) {
|
||||
this.stats = new Map(Object.entries(data.stats));
|
||||
}
|
||||
if (data.hourlyHistory) {
|
||||
this.hourlyHistory = new Map(Object.entries(data.hourlyHistory));
|
||||
}
|
||||
if (data.dailyHistory) {
|
||||
this.dailyHistory = new Map(Object.entries(data.dailyHistory));
|
||||
}
|
||||
if (data.alerts) {
|
||||
this.alerts = new Map(Object.entries(data.alerts));
|
||||
}
|
||||
this.saveStats();
|
||||
this.saveHourlyStats();
|
||||
this.saveDailyStats();
|
||||
this.saveAlertConfig();
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
module.exports = new ResourceMonitor();
|
||||
@@ -0,0 +1,237 @@
|
||||
/**
|
||||
* State Manager - Thread-safe file operations with locking
|
||||
*
|
||||
* Prevents data corruption when multiple API requests modify state files concurrently.
|
||||
* Uses file-based locking with automatic retry and timeout handling.
|
||||
*
|
||||
* @module state-manager
|
||||
*/
|
||||
|
||||
const lockfile = require('proper-lockfile');
|
||||
const fs = require('fs').promises;
|
||||
const fsSync = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
class StateManager {
|
||||
/**
|
||||
* Create a StateManager instance
|
||||
* @param {string} filePath - Path to the state file (e.g., services.json)
|
||||
* @param {Object} options - Configuration options
|
||||
* @param {number} options.lockTimeout - Max time to wait for lock (ms)
|
||||
* @param {number} options.lockRetries - Number of lock acquisition retries
|
||||
* @param {number} options.lockRetryInterval - Time between retries (ms)
|
||||
*/
|
||||
constructor(filePath, options = {}) {
|
||||
this.filePath = filePath;
|
||||
this.lockOptions = {
|
||||
retries: {
|
||||
retries: options.lockRetries || 10,
|
||||
minTimeout: options.lockRetryInterval || 100,
|
||||
maxTimeout: (options.lockRetryInterval || 100) * 3
|
||||
},
|
||||
stale: options.lockTimeout || 30000 // 30 seconds
|
||||
};
|
||||
|
||||
// Ensure file exists
|
||||
this._ensureFileExists();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the state file exists, create with empty array if not
|
||||
* @private
|
||||
*/
|
||||
_ensureFileExists() {
|
||||
if (!fsSync.existsSync(this.filePath)) {
|
||||
const dir = path.dirname(this.filePath);
|
||||
if (!fsSync.existsSync(dir)) {
|
||||
fsSync.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
fsSync.writeFileSync(this.filePath, '[]', 'utf8');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the state file (no locking required for read-only operations)
|
||||
* @returns {Promise<any>} Parsed JSON data
|
||||
* @throws {Error} If file doesn't exist or JSON is invalid
|
||||
*/
|
||||
async read() {
|
||||
try {
|
||||
const content = await fs.readFile(this.filePath, 'utf8');
|
||||
return JSON.parse(content);
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
// File doesn't exist — recreate without locking (no file to lock)
|
||||
this._ensureFileExists();
|
||||
return [];
|
||||
}
|
||||
throw new Error(`Failed to read state file: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write data to the state file (with locking)
|
||||
* @param {any} data - Data to write (will be JSON.stringify'd)
|
||||
* @returns {Promise<void>}
|
||||
* @throws {Error} If lock cannot be acquired or write fails
|
||||
*/
|
||||
async write(data) {
|
||||
let release;
|
||||
try {
|
||||
// Acquire lock
|
||||
release = await lockfile.lock(this.filePath, this.lockOptions);
|
||||
|
||||
// Write data with pretty formatting
|
||||
await fs.writeFile(this.filePath, JSON.stringify(data, null, 2), 'utf8');
|
||||
} catch (error) {
|
||||
if (error.code === 'ELOCKED') {
|
||||
throw new Error('State file is locked by another process. Try again.');
|
||||
}
|
||||
throw new Error(`Failed to write state file: ${error.message}`);
|
||||
} finally {
|
||||
// Always release lock
|
||||
if (release) {
|
||||
try {
|
||||
await release();
|
||||
} catch (e) {
|
||||
// Lock release failure (non-critical, lock will expire via stale timeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the state file using a callback function (atomic operation)
|
||||
* This is the recommended method for most operations.
|
||||
*
|
||||
* @param {Function} updateFn - Function that receives current data and returns updated data
|
||||
* @returns {Promise<any>} The updated data
|
||||
* @throws {Error} If lock cannot be acquired or update fails
|
||||
*
|
||||
* @example
|
||||
* // Add a new service
|
||||
* await stateManager.update(services => {
|
||||
* services.push({ id: 'new-service', name: 'New Service' });
|
||||
* return services;
|
||||
* });
|
||||
*
|
||||
* @example
|
||||
* // Remove a service
|
||||
* await stateManager.update(services => {
|
||||
* return services.filter(s => s.id !== 'old-service');
|
||||
* });
|
||||
*/
|
||||
async update(updateFn) {
|
||||
let release;
|
||||
try {
|
||||
// Acquire lock
|
||||
release = await lockfile.lock(this.filePath, this.lockOptions);
|
||||
|
||||
// Read current data
|
||||
const content = await fs.readFile(this.filePath, 'utf8');
|
||||
const currentData = JSON.parse(content);
|
||||
|
||||
// Apply update function
|
||||
const updatedData = await updateFn(currentData);
|
||||
|
||||
// Write updated data
|
||||
await fs.writeFile(this.filePath, JSON.stringify(updatedData, null, 2), 'utf8');
|
||||
|
||||
return updatedData;
|
||||
} catch (error) {
|
||||
if (error.code === 'ELOCKED') {
|
||||
throw new Error('State file is locked by another process. Try again.');
|
||||
}
|
||||
throw new Error(`Failed to update state file: ${error.message}`);
|
||||
} finally {
|
||||
// Always release lock
|
||||
if (release) {
|
||||
try {
|
||||
await release();
|
||||
} catch (e) {
|
||||
// Lock release failure (non-critical, lock will expire via stale timeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the state file is currently locked
|
||||
* @returns {Promise<boolean>} True if locked, false otherwise
|
||||
*/
|
||||
async isLocked() {
|
||||
try {
|
||||
return await lockfile.check(this.filePath);
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Forcefully unlock the state file (use with caution!)
|
||||
* Only use this if a lock is stuck due to a crashed process.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async forceUnlock() {
|
||||
try {
|
||||
await lockfile.unlock(this.filePath);
|
||||
} catch (error) {
|
||||
// Ignore errors if file wasn't locked
|
||||
if (error.code !== 'ENOTACQUIRED') {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an item to the state array (convenience method)
|
||||
* @param {any} item - Item to add
|
||||
* @returns {Promise<any>} Updated array
|
||||
*/
|
||||
async addItem(item) {
|
||||
return await this.update(items => {
|
||||
items.push(item);
|
||||
return items;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove an item from the state array by ID (convenience method)
|
||||
* @param {string} id - ID of item to remove
|
||||
* @returns {Promise<any>} Updated array
|
||||
*/
|
||||
async removeItem(id) {
|
||||
return await this.update(items => {
|
||||
return items.filter(item => item.id !== id);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an item in the state array by ID (convenience method)
|
||||
* @param {string} id - ID of item to update
|
||||
* @param {Object} updates - Properties to update
|
||||
* @returns {Promise<any>} Updated array
|
||||
*/
|
||||
async updateItem(id, updates) {
|
||||
return await this.update(items => {
|
||||
return items.map(item => {
|
||||
if (item.id === id) {
|
||||
return { ...item, ...updates };
|
||||
}
|
||||
return item;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Find an item in the state array by ID (convenience method)
|
||||
* @param {string} id - ID of item to find
|
||||
* @returns {Promise<any|null>} Found item or null
|
||||
*/
|
||||
async findItem(id) {
|
||||
const items = await this.read();
|
||||
return items.find(item => item.id === id) || null;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = StateManager;
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user